diff --git a/.github/workflows/ci-cmake_tests.yml b/.github/workflows/ci-cmake_tests.yml index d99767377..3187dc6e5 100644 --- a/.github/workflows/ci-cmake_tests.yml +++ b/.github/workflows/ci-cmake_tests.yml @@ -63,7 +63,8 @@ jobs: # pyproject.toml as the single source of truth for everything # Python. run: | - mamba install _openmp_mutex=*=*_llvm cmake make boost git compilers blas=*=openblas liblapacke arpack gtest + mamba install _openmp_mutex=*=*_llvm cmake make boost git compilers blas=*=openblas liblapacke arpack gtest hdf5 + - name: CPU info run: lscpu diff --git a/.github/workflows/coverity-scan.yml b/.github/workflows/coverity-scan.yml index 3e6ba77c7..295ca06e1 100644 --- a/.github/workflows/coverity-scan.yml +++ b/.github/workflows/coverity-scan.yml @@ -35,10 +35,15 @@ jobs: - name: Install dependencies shell: bash -l {0} run: | - mamba install cmake make boost compilers blas=*=mkl pybind11 arpack + mamba config append channels conda-forge - cmake -S ${{github.workspace}} -B ${{github.workspace}}/build -DCMAKE_INSTALL_PREFIX=/home/runner/works/Cytnx_lib --preset=mkl-cpu + mamba install cmake make boost compilers hdf5 mkl mkl-include arpack + cmake -S ${{github.workspace}} -B ${{github.workspace}}/build \ + -DCMAKE_INSTALL_PREFIX=/home/runner/works/Cytnx_lib \ + -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \ + -DHDF5_ROOT=$CONDA_PREFIX \ + --preset=mkl-cpu -DBUILD_PYTHON=OFF - name: Download Coverity Build Tool shell: bash -l {0} working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index cea6adf58..60f982146 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -286,6 +286,11 @@ find_library(ARPACK_LIB arpack REQUIRED) message(STATUS "Found ARPACK_LIB at: ${ARPACK_LIB}") target_link_libraries(cytnx PRIVATE ${ARPACK_LIB}) +#HDF5 +find_package(HDF5 1.10.0 REQUIRED COMPONENTS CXX) +target_include_directories(cytnx PRIVATE ${HDF5_INCLUDE_DIRS}) +target_link_libraries(cytnx PRIVATE ${HDF5_LIBRARIES}) + # ##################################################################### # ## Get Gtest & benchmark @@ -375,6 +380,7 @@ IF(BUILD_PYTHON) endif() pybind11_add_module(pycytnx MODULE pybind/cytnx.cpp + pybind/io_py.cpp pybind/generator_py.cpp pybind/storage_py.cpp pybind/tensor_py.cpp diff --git a/benchmarks/linalg/Svd_bm.cpp b/benchmarks/linalg/Svd_bm.cpp index 57c4147e7..6709e172d 100644 --- a/benchmarks/linalg/Svd_bm.cpp +++ b/benchmarks/linalg/Svd_bm.cpp @@ -39,7 +39,7 @@ namespace BMTest_Svd { BENCHMARK(BM_Tensor_Svd_C128)->Args({1, 1})->Args({10, 10})->Args({100, 100})->Args({1000, 1000}); // DenseUniTensor - UniTensor ConstructDenseUT(const int D, const int dtype, const unsigned int device = Device.cpu) { + UniTensor ConstructDenseUT(const int D, const int dtype, const int device = Device.cpu) { auto bd_vi = Bond(D, BD_IN); auto bd_pi = Bond(2, BD_IN); auto bd_po = Bond(2, BD_OUT); diff --git a/benchmarks/linalg/Svd_truncate_bm.cpp b/benchmarks/linalg/Svd_truncate_bm.cpp index dffa756c6..9b95640db 100644 --- a/benchmarks/linalg/Svd_truncate_bm.cpp +++ b/benchmarks/linalg/Svd_truncate_bm.cpp @@ -5,7 +5,7 @@ using namespace cytnx; namespace BMTest_Svd_truncate { // DenseUniTensor - UniTensor ConstructDenseUT(const int D, const int dtype, const unsigned int device = Device.cpu) { + UniTensor ConstructDenseUT(const int D, const int dtype, const int device = Device.cpu) { auto bd_vi = Bond(D, BD_IN); auto bd_pi = Bond(2, BD_IN); auto bd_po = Bond(2, BD_OUT); diff --git a/conda_build/meta.yaml b/conda_build/meta.yaml index 71794a6fb..d178cdd03 100644 --- a/conda_build/meta.yaml +++ b/conda_build/meta.yaml @@ -10,6 +10,9 @@ source: build: number: 0 + channels: + - conda-forge + - defaults script_env: # Refer to : https://github.com/ccache/ccache/discussions/821#discussioncomment-521209 - CCACHE_NOHASHDIR=1 @@ -39,6 +42,7 @@ requirements: - boost >=1.82.0 - blas=*=mkl # [x86] - blas=*=openblas # [not x86] + - hdf5 >=1.10.0 - gtest - arpack run: @@ -49,6 +53,7 @@ requirements: - graphviz - blas=*=mkl # [x86] - blas=*=openblas # [not x86] + - hdf5 >=1.10.0 - beartype - arpack diff --git a/cytnx/io_conti.py b/cytnx/io_conti.py new file mode 100644 index 000000000..81ac007a2 --- /dev/null +++ b/cytnx/io_conti.py @@ -0,0 +1,9 @@ +from typing import Any +from cytnx import * + +def Load(obj: Any, container, name: str, path: str = ""): + io.c_Load(obj, container, name, path) + +# inject into the submodule +obj = io +setattr(obj,"Load",Load) diff --git a/docs/source/adv_install.rst b/docs/source/adv_install.rst index ecfe0ed2a..0603364ce 100644 --- a/docs/source/adv_install.rst +++ b/docs/source/adv_install.rst @@ -93,7 +93,7 @@ There are two methods how you can set-up all the dependencies before starting th .. code-block:: shell - $conda install cmake make boost boost-cpp git compilers numpy openblas arpack pybind11 beartype arpack + $conda install cmake make boost boost-cpp git compilers hdf5 numpy openblas arpack pybind11 beartype arpack .. Note:: @@ -103,7 +103,7 @@ There are two methods how you can set-up all the dependencies before starting th .. code-block:: shell - $conda install cmake make boost boost-cpp git compilers numpy mkl mkl-include mkl-service arpack pybind11 libblas=*=*mkl beartype arpack + $conda install cmake make boost boost-cpp git compilers hdf5 numpy mkl mkl-include mkl-service arpack pybind11 libblas=*=*mkl beartype arpack 3. After the installation, an automated test based on gtest and benchmark can be run. This option needs to be activated in the install script. In this case, gtest needs to be installed as well: diff --git a/include/Bond.hpp b/include/Bond.hpp index 4e2417901..0e9c87797 100644 --- a/include/Bond.hpp +++ b/include/Bond.hpp @@ -1,14 +1,18 @@ #ifndef CYTNX_BOND_H_ #define CYTNX_BOND_H_ -#include "Type.hpp" -#include "cytnx_error.hpp" -#include "Symmetry.hpp" -#include -#include +#include +#include #include +#include #include -#include +#include + +#include "H5Cpp.h" + +#include "Symmetry.hpp" +#include "Type.hpp" +#include "cytnx_error.hpp" #include "intrusive_ptr_base.hpp" #include "utils/vec_clone.hpp" @@ -36,6 +40,10 @@ namespace cytnx { BD_IN = -1, /*!< -1, same as BD_KET */ BD_OUT = 1 /*!< 1, same as BD_BRA */ }; + static const std::map bondtype_to_string = { + {BD_REG, "REG"}, {BD_IN, "IN"}, {BD_OUT, "OUT"}}; + static const std::map string_to_bondtype = { + {"REG", BD_REG}, {"IN", BD_IN}, {"OUT", BD_OUT}}; /// @cond class Bond_impl : public intrusive_ptr_base { @@ -848,52 +856,116 @@ namespace cytnx { } /** - @brief Save the Bond object to the file. - @details Save the Bond object to the file. The file extension will be automatically - added as ".cybd". - @param[in] fname the file name of the Bond object (exclude the file extension). - @see Load(const std::string &fname) - */ - void Save(const std::string &fname) const; - + * @brief Save Bond to file + * @details Save the Bond to a file. The file ending should be one of ".h5", ".hdf5", ".H5", + * ".HDF5", ".hdf" to save in HDF5 file format. Otherwise, a binary file format is used. + * @param[in] fname file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/' will write + * the Bond to the group '/foo/bar' in the file. + * @param[in] mode the write mode:\n + * `w` Creates a new file. If the given file exists, its contents are destroyed.\n + * `x` Creates a new file. Fails if the given file exists already.\n + * `a` Opens for writing without overwriting any existing content. Creates the file if it + * doesn't exist. Only available for HDF5 files.\n + * `u` Opens for writing. Existing content will be updated(overwritten). + * Creates the file if it doesn't exist. Only available for HDF5 files. + * @note The common file ending for saving a Bond in binary format is ".cybd". + * @warning HDF5 file format is strongly recommended for compatibility with other libraries, + * readability, and future-proofing. + * @see Load() + */ + void Save(const std::filesystem::path &fname, const std::string &path = "/Bond/", + const char mode = 'w') const; /** - @see Save(const std::string &fname) const; - */ - void Save(const char *fname) const; + * @see Save(const std::filesystem::path &fname, const std::string &path, const char mode) + * const; + */ + void Save(const char *fname, const std::string &path = "/Bond/", const char mode = 'w') const; + + /** + * @brief Load Bond from file and create new instance + * @details This function creates a new Bond and keeps the original Bond unchanged. See Load_() + * for loading the Bond to the current Bond. + * @param fname[in] file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/' will read + * the Bond from the group '/foo/bar' in the file. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @pre The file must be a Bond object which is saved by Save(). + * @note For HDF5 file format, one of the file endings ".h5", ".hdf5", ".H5", ".HDF5", ".hdf" is + * expected. For binary format, the common file ending for a Bond is ".cybd". + */ + static cytnx::Bond Load(const std::filesystem::path &fname, const std::string &path = "/Bond/"); + /** + * @see Load(const std::filesystem::path &fname, const std::string &path) + */ + static cytnx::Bond Load(const char *fname, const std::string &path = "/Bond/"); /** - @brief Load the Bond object from the file. - @param[in] fname the file name of the Bond object. - @pre The file need to be the file of Bond object, which is saved by the - function Bond::Save(const std::string &fname) const. - */ - static cytnx::Bond Load(const std::string &fname); + * @brief Load Bond from file and overwrite current instance + * @details This function overwrites the existing Bond. See Load() for creating a new Bond. + * @see Load() + */ + void Load_(const std::filesystem::path &fname, const std::string &path = "/Bond/"); + /** + * @see Load_(const std::filesystem::path &fname, const std::string &path) + */ + void Load_(const char *fname, const std::string &path = "/Bond/"); /** - @see Load(const std::string &fname) - */ - static cytnx::Bond Load(const char *fname); + * @brief Save Bond to HDF5 file + * @param[in] container the HDF5 parent group. + * @param[in] name the subgroup in which the Bond will be saved. + * @param[in] overwrite overwrite previous Bond information in the container. + * @warning This function is only available in C++. Use Save() for saving to file in C++ or + * Python. + * @see from_hdf5() + */ + void to_hdf5(H5::Group &container, const std::string &name = "Bond", + const bool overwrite = false) const; + /** + * @brief Load Bond from HDF5 file (inline) + * @param[in] container the HDF5 parent group. + * @param[in] name the subgroup from which the Bond will be loaded. + * @warning This function is only available in C++. Use Load() for loading from file in C++ or + * Python. + * @see to_hdf5() const + */ + void from_hdf5(H5::Group &container, const std::string &name = "Bond"); - /// @cond - void _Save(std::fstream &f) const; - void _Load(std::fstream &f); - /// @endcond + /** + * @brief Save Bond to binary file + * @param[in] f the output stream where the Bond will be saved. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Save() for saving to file in C++ or Python. + * @see from_binary() + */ + void to_binary(std::ostream &f) const; + /** + * @brief Load Bond from binary file + * @param[in] f the input stream from which the Bond will be loaded. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Load() for loading from file in C++ or Python. + * @see to_binary() const + */ + void from_binary(std::istream &f); /** @brief The comparison operator 'equal to'. - @details The comparison operators to compare two Bonds. If two Bond object are - same, return true. Otherwise, return false. This equal to operator will - compare all the "value" of the Bond object. Even the Bond object are different - object (different address), but they are same "value", it will return true. - @see operator!=(const Bond &rhs) const + @details The comparison operators to compare two Bonds. If two Bond object are same, return + true. Otherwise, return false. This equal to operator will compare all the "value" of the Bond + object. Even the Bond object are different object (different address), but they are same + "value", it will return true. + @see operator!=(const Bond &rhs) const */ bool operator==(const Bond &rhs) const; /** @brief The comparison operator 'not equal to'. - @details The comparison operators to compare two Bonds. More precisely, it is - the opposite result of the operator==(const Bond &rhs) const. - @see operator==(const Bond &rhs) const + @details The comparison operators to compare two Bonds. More precisely, it is the opposite + result of the operator==(const Bond &rhs) const. + @see operator==(const Bond &rhs) const */ bool operator!=(const Bond &rhs) const; diff --git a/include/Gncon.hpp b/include/Gncon.hpp index d83c1c14f..438f8aee6 100644 --- a/include/Gncon.hpp +++ b/include/Gncon.hpp @@ -1,16 +1,18 @@ #ifndef CYTNX_GNCON_H_ #define CYTNX_GNCON_H_ -#include "Type.hpp" -#include "cytnx_error.hpp" +#include +#include #include -#include #include -#include -#include "intrusive_ptr_base.hpp" -#include "utils/utils.hpp" +#include + +#include "Type.hpp" #include "UniTensor.hpp" #include "contraction_tree.hpp" +#include "cytnx_error.hpp" +#include "intrusive_ptr_base.hpp" +#include "utils/utils.hpp" #ifdef BACKEND_TORCH #else @@ -88,14 +90,14 @@ namespace cytnx { const std::vector &alias, const std::string &contract_order); - virtual void Fromfile(const std::string &fname); + virtual void Fromfile(const std::filesystem::path &fname); virtual void FromString(const std::vector &content); virtual void clear(); virtual std::string getOptimalOrder(); virtual UniTensor Launch(const bool &optimal = false, const std::string &contract_order = ""); virtual void PrintNet(std::ostream &os); virtual boost::intrusive_ptr clone(); - virtual void Savefile(const std::string &fname); + virtual void Savefile(const std::filesystem::path &fname); virtual ~Gncon_base(){}; }; // Gncon_base @@ -103,7 +105,7 @@ namespace cytnx { class RegularGncon : public Gncon_base { public: RegularGncon() { this->nwrktype_id = GNType.Regular; }; - void Fromfile(const std::string &fname); + void Fromfile(const std::filesystem::path &fname); void FromString(const std::vector &contents); void PutUniTensor(const std::string &name, const UniTensor &utensor); void PutUniTensor(const cytnx_uint64 &idx, const UniTensor &utensor); @@ -138,7 +140,7 @@ namespace cytnx { return out; } void PrintNet(std::ostream &os); - void Savefile(const std::string &fname); + void Savefile(const std::filesystem::path &fname); ~RegularGncon(){}; }; @@ -149,7 +151,7 @@ namespace cytnx { public: FermionGncon() { this->nwrktype_id = GNType.Fermion; }; - void Fromfile(const std::string &fname){}; + void Fromfile(const std::filesystem::path &fname){}; void FromString(const std::vector &contents){}; void PutUniTensor(const std::string &name, const UniTensor &utensor){}; void PutUniTensor(const cytnx_uint64 &idx, const UniTensor &utensor){}; @@ -183,7 +185,7 @@ namespace cytnx { return out; } void PrintNet(std::ostream &os){}; - void Savefile(const std::string &fname){}; + void Savefile(const std::filesystem::path &fname){}; ~FermionGncon(){}; }; @@ -254,7 +256,7 @@ namespace cytnx { */ - void Fromfile(const std::string &fname, const int &Gncon_type = GNType.Regular) { + void Fromfile(const std::filesystem::path &fname, const int &Gncon_type = GNType.Regular) { if (Gncon_type == GNType.Regular) { boost::intrusive_ptr tmp(new RegularGncon()); this->_impl = tmp; @@ -301,7 +303,7 @@ namespace cytnx { } this->_impl->FromString(contents); } - // void Savefile(const std::string &fname); + // void Savefile(const std::filesystem::path &fname); static Gncon Contract(const std::vector &tensors, const std::string &Tout, const std::vector &alias = {}, @@ -313,7 +315,7 @@ namespace cytnx { return out; } - Gncon(const std::string &fname, const int &Gncon_type = GNType.Regular) { + Gncon(const std::filesystem::path &fname, const int &Gncon_type = GNType.Regular) { this->Fromfile(fname, Gncon_type); } @@ -354,7 +356,7 @@ namespace cytnx { } void PrintNet() { this->_impl->PrintNet(std::cout); } - void Savefile(const std::string &fname) { this->_impl->Savefile(fname); } + void Savefile(const std::filesystem::path &fname) { this->_impl->Savefile(fname); } }; ///@cond diff --git a/include/Network.hpp b/include/Network.hpp index 75aa82db1..dc31f9129 100644 --- a/include/Network.hpp +++ b/include/Network.hpp @@ -1,16 +1,18 @@ #ifndef CYTNX_NETWORK_H_ #define CYTNX_NETWORK_H_ -#include "Type.hpp" -#include "cytnx_error.hpp" +#include +#include #include -#include #include -#include -#include "intrusive_ptr_base.hpp" -#include "utils/utils.hpp" +#include + +#include "Type.hpp" #include "UniTensor.hpp" #include "contraction_tree.hpp" +#include "cytnx_error.hpp" +#include "intrusive_ptr_base.hpp" +#include "utils/utils.hpp" #ifdef BACKEND_TORCH #else @@ -119,7 +121,7 @@ namespace cytnx { const std::vector &alias, const std::string &contract_order); - virtual void Fromfile(const std::string &fname); + virtual void Fromfile(const std::filesystem::path &fname); virtual void FromString(const std::vector &content); virtual void clear(); virtual std::string getOptimalOrder(); @@ -137,7 +139,7 @@ namespace cytnx { const std::string &order, const bool optim); virtual void PrintNet(std::ostream &os); virtual boost::intrusive_ptr clone(); - virtual void Savefile(const std::string &fname); + virtual void Savefile(const std::filesystem::path &fname); virtual ~Network_base(){}; }; // Network_base @@ -145,7 +147,7 @@ namespace cytnx { class RegularNetwork : public Network_base { public: RegularNetwork() { this->nwrktype_id = NtType.Regular; }; - void Fromfile(const std::string &fname); + void Fromfile(const std::filesystem::path &fname); void FromString(const std::vector &contents); void PutUniTensor(const std::string &name, const UniTensor &utensor); void PutUniTensor(const cytnx_uint64 &idx, const UniTensor &utensor); @@ -195,7 +197,7 @@ namespace cytnx { return out; } void PrintNet(std::ostream &os); - void Savefile(const std::string &fname); + void Savefile(const std::filesystem::path &fname); ~RegularNetwork(){}; }; @@ -206,7 +208,7 @@ namespace cytnx { public: FermionNetwork() { this->nwrktype_id = NtType.Fermion; }; - void Fromfile(const std::string &fname){}; + void Fromfile(const std::filesystem::path &fname){}; void FromString(const std::vector &contents){}; void RmUniTensor(const cytnx_uint64 &idx){}; void RmUniTensor(const std::string &name){}; @@ -246,7 +248,7 @@ namespace cytnx { return out; } void PrintNet(std::ostream &os){}; - void Savefile(const std::string &fname){}; + void Savefile(const std::filesystem::path &fname){}; ~FermionNetwork(){}; }; @@ -317,7 +319,7 @@ namespace cytnx { */ - void Fromfile(const std::string &fname, const int &network_type = NtType.Regular) { + void Fromfile(const std::filesystem::path &fname, const int &network_type = NtType.Regular) { if (network_type == NtType.Regular) { boost::intrusive_ptr tmp(new RegularNetwork()); this->_impl = tmp; @@ -364,7 +366,7 @@ namespace cytnx { } this->_impl->FromString(contents); } - // void Savefile(const std::string &fname); + // void Savefile(const std::filesystem::path &fname); static Network Contract(const std::vector &tensors, const std::string &Tout, const std::vector &alias = {}, @@ -376,7 +378,7 @@ namespace cytnx { return out; } - Network(const std::string &fname, const int &network_type = NtType.Regular) { + Network(const std::filesystem::path &fname, const int &network_type = NtType.Regular) { this->Fromfile(fname, network_type); } @@ -452,7 +454,7 @@ namespace cytnx { } void PrintNet() { this->_impl->PrintNet(std::cout); } - void Savefile(const std::string &fname) { this->_impl->Savefile(fname); } + void Savefile(const std::filesystem::path &fname) { this->_impl->Savefile(fname); } }; ///@cond diff --git a/include/Symmetry.hpp b/include/Symmetry.hpp index 46b02e5a1..8f29a2a39 100644 --- a/include/Symmetry.hpp +++ b/include/Symmetry.hpp @@ -1,16 +1,20 @@ #ifndef CYTNX_SYMMETRY_H_ #define CYTNX_SYMMETRY_H_ +#include #include +#include #include +#include #include #include +#include "H5Cpp.h" #include "boost/smart_ptr/intrusive_ptr.hpp" +#include "Type.hpp" #include "cytnx_error.hpp" #include "intrusive_ptr_base.hpp" -#include "Type.hpp" #include "utils/dynamic_arg_resolver.hpp" namespace cytnx { @@ -27,7 +31,7 @@ namespace cytnx { * fPar | -2, fermionParity symmetry * fNum | -3, fermionNumber symmetry * - * @see Symmetry::stype(), Symmetry::stype_str() + * @see Symmetry::stype(), Symmetry::getname(), Symmetry::stype_str() */ enum SymmetryType : int { Void = -99, U = -1, Z = 0, fPar = -2, fNum = -3 }; @@ -63,7 +67,7 @@ namespace cytnx { ///@cond class Symmetry_base : public intrusive_ptr_base { public: - int stype_id; + SymmetryType stype_id; int n; Symmetry_base() : stype_id(SymmetryType::Void){}; Symmetry_base(const int &n) : stype_id(SymmetryType::Void) { this->Init(n); }; @@ -91,6 +95,7 @@ namespace cytnx { virtual bool is_fermionic() const { return false; }; virtual void print_info() const; + virtual std::string getname() const; virtual std::string stype_str() const; // virtual std::vector& combine_rule(const std::vector &inL, const // std::vector &inR); @@ -119,6 +124,7 @@ namespace cytnx { const bool &is_reverse); void reverse_rule_(cytnx_int64 &out, const cytnx_int64 &in); void print_info() const; + std::string getname() const override { return "U1"; } std::string stype_str() const override { return "U1"; }; }; ///@endcond @@ -145,6 +151,7 @@ namespace cytnx { const bool &is_reverse); void reverse_rule_(cytnx_int64 &out, const cytnx_int64 &in); void print_info() const; + std::string getname() const override { return "Z" + std::to_string(this->n); }; std::string stype_str() const override { return "Z" + std::to_string(this->n); }; }; ///@endcond @@ -170,6 +177,7 @@ namespace cytnx { fermionParity get_fermion_parity(const cytnx_int64 &in_qnum) const override; bool is_fermionic() const override { return true; }; void print_info() const; + std::string getname() const override { return "fermion parity"; } std::string stype_str() const override { return "fP"; } }; ///@endcond @@ -195,6 +203,7 @@ namespace cytnx { fermionParity get_fermion_parity(const cytnx_int64 &in_qnum) const override; bool is_fermionic() const override { return true; }; void print_info() const; + std::string getname() const override { return "fermion number"; } std::string stype_str() const override { return "f#"; } }; ///@endcond @@ -226,9 +235,35 @@ namespace cytnx { boost::intrusive_ptr tmp(new FermionNumberSymmetry()); this->_impl = tmp; } else { - cytnx_error_msg(1, "%s", "[ERROR] invalid symmetry type."); + cytnx_error_msg(true, "[ERROR] Invalid Symmetry type %d.\n", stype); } } + void Init(const std::string name) { + if (name == "U1") { + boost::intrusive_ptr tmp(new U1Symmetry(1)); + this->_impl = tmp; + } else if (name == "fermion parity") { + boost::intrusive_ptr tmp(new FermionParitySymmetry()); + this->_impl = tmp; + } else if (name == "fermion number") { + boost::intrusive_ptr tmp(new FermionNumberSymmetry()); + this->_impl = tmp; + } else { // check if Z(N) + std::regex pattern(R"(Z(\d+))"); + std::smatch matches; + if (std::regex_match(name, matches, pattern)) { + // matches[0] is the whole string "U(456)" + // matches[1] is the first capture group "456" + int n = std::stoi(matches[1].str()); + boost::intrusive_ptr tmp(new ZnSymmetry(n)); + this->_impl = tmp; + } else { + cytnx_error_msg(true, "[ERROR] No Symmetry type matches the string '%s'.\n", + name.c_str()); + } + } + } + Symmetry &operator=(const Symmetry &rhs) { this->_impl = rhs._impl; return *this; @@ -236,7 +271,7 @@ namespace cytnx { Symmetry(const Symmetry &rhs) { this->_impl = rhs._impl; } ///@endcond - //[genenrators] + //[generators] /** @brief create a U1 symmetry object @@ -380,11 +415,13 @@ namespace cytnx { int &n() const { return this->_impl->n; } /** - @brief return the symmetry type name of current Symmetry object in string form, see - cytnx::SymmetryType:: - @return [std::string] - the symmetry type name. - + @brief Type name of the Symmetry in long form + @return [std::string] the symmetry type long name. + */ + std::string getname() const { return this->_impl->getname(); } + /** + @brief Type name of the Symmetry in short form + @return [std::string] the symmetry type short name. */ std::string stype_str() const { return this->_impl->stype_str(); } @@ -492,34 +529,100 @@ namespace cytnx { bool is_fermionic() const { return this->_impl->is_fermionic(); } /** - * @brief Save the current Symmetry object to a file. - * @param[in] fname the file name. - * @post the file extension will be automatically added as ".cysym". + * @brief Save Symmetry to file + * @details Save the Symmetry to a file. The file ending should be one of ".h5", ".hdf5", ".H5", + * ".HDF5", ".hdf" to save in HDF5 file format. Otherwise, a binary file format is used. + * @param[in] fname file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/Symm' will + * write the Symmetry to the attribute 'Symm' the group '/foo/bar' in the file. + * @param[in] mode the write mode:\n + * `w` Creates a new file. If the given file exists, its contents are destroyed.\n + * `x` Creates a new file. Fails if the given file exists already.\n + * `a` Opens for writing without overwriting any existing content. Creates the file if it + * doesn't exist. Only available for HDF5 files.\n + * `u` Opens for writing. Existing content will be updated(overwritten). + * Creates the file if it doesn't exist. Only available for HDF5 files. + * @note The common file ending for saving a Symmetry in binary format is ".cysym". + * @warning HDF5 file format is strongly recommended for compatibility with other libraries, + * readability, and future-proofing. + * @see Load() + */ + void Save(const std::filesystem::path &fname, const std::string &path = "/Symmetry", + const char mode = 'w') const; + /** + * @see Save(const std::filesystem::path &fname, const std::string &path, const char mode) + * const; */ - void Save(const std::string &fname) const; + void Save(const char *fname, const std::string &path = "/Symmetry", + const char mode = 'w') const; /** - * @brief Same as Save(const std::string &fname) const; + * @brief Load Symmetry from file and create new instance + * @details This function creates a new Symmetry and keeps the original Symmetry unchanged. See + * Load_() for loading the Symmetry to the current Symmetry. + * @param fname[in] file name + * @param[in] path path inside the file. Only used for HDF5 files. A path /foo/bar/Symm will + * read the Symmetry from the attribute 'Symm' the group '/foo/bar' in the file. + * @pre The file must be a Symmetry object which is saved by Save(). + * @note For HDF5 file format, one of the file endings ".h5", ".hdf5", ".H5", ".HDF5", ".hdf" is + * expected. For binary format, the common file ending for a Symmetry is ".cysym". */ - void Save(const char *fname) const; + static cytnx::Symmetry Load(const std::filesystem::path &fname, + const std::string &path = "/Symmetry"); + /** + * @see Load(const std::filesystem::path &fname, const std::string &path) + */ + static cytnx::Symmetry Load(const char *fname, const std::string &path = "/Symmetry"); /** - * @brief Load a Symmetry object from a file. - * @param[in] fname the file name. - * @pre the file extension must be ".cysym". - * @return the loaded Symmetry object. + * @brief Load Symmetry from file and overwrite current instance + * @details This function overwrites the existing Symmetry. See Load() for creating a new + * Symmetry. + * @see Load() + */ + void Load_(const std::filesystem::path &fname, const std::string &path = "/Symmetry"); + /** + * @see Load_(const std::filesystem::path &fname, const std::string &path) */ - static Symmetry Load(const std::string &fname); + void Load_(const char *fname, const std::string &path = "/Symmetry"); /** - * @brief Same as static Symmetry Load(const std::string &fname); + * @brief Save Symmetry to HDF5 file + * @param[in] container the HDF5 group where the Symmetry will be saved. + * @param[in] name the name of the attribute in the HDF5 file. + * @param[in] overwrite overwrite previous Bond information in the container. + * @warning This function is only available in C++. Use Save() for saving to file in C++ or + * Python. + * @see from_hdf5() */ - static Symmetry Load(const char *fname); + void to_hdf5(H5::Group &container, const std::string &name = "Symmetry", + const bool overwrite = false) const; + /** + * @brief Load Symmetry from HDF5 file (inline) + * @param[in] container the HDF5 group where the Symmetry will be loaded from. + * @param[in] name the name of the attribute in the HDF5 file. + * @warning This function is only available in C++. Use Load() for loading from file in C++ or + * Python. + * @see to_hdf5() + */ + void from_hdf5(H5::Group &container, const std::string &name = "Symmetry"); - /// @cond - void _Save(std::fstream &f) const; - void _Load(std::fstream &f); - /// @endcond + /** + * @brief Save Symmetry to binary file + * @param[in] f the output stream where the Symmetry will be saved. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Save() for saving to file in C++ or Python. + * @see from_binary() + */ + void to_binary(std::ostream &f) const; + /** + * @brief Load Symmetry from binary file + * @param[in] f the input stream from which the Symmetry will be loaded. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Load() for loading from file in C++ or Python. + * @see to_binary() + */ + void from_binary(std::istream &f); /** * @brief Print the information of current Symmetry object. diff --git a/include/Tensor.hpp b/include/Tensor.hpp index 11fd45bf7..80c89e1f3 100644 --- a/include/Tensor.hpp +++ b/include/Tensor.hpp @@ -1,18 +1,22 @@ #ifndef CYTNX_TENSOR_H_ #define CYTNX_TENSOR_H_ -#include "Type.hpp" -#include "cytnx_error.hpp" -#include "Device.hpp" -#include "intrusive_ptr_base.hpp" -#include +#include #include -#include "utils/dynamic_arg_resolver.hpp" -#include "Accessor.hpp" -#include -#include #include +#include #include +#include +#include + +#include "H5Cpp.h" +#include "intrusive_ptr_base.hpp" + +#include "Accessor.hpp" +#include "cytnx_error.hpp" +#include "Device.hpp" +#include "Type.hpp" +#include "utils/dynamic_arg_resolver.hpp" #ifdef BACKEND_TORCH #else @@ -311,25 +315,113 @@ namespace cytnx { ///@endcond //------------------------------------------- - /// @cond - void _Save(std::fstream &f) const; - void _Load(std::fstream &f); + /** + * @brief Save Tensor to file + * @details Save the Tensor to a file. The file ending should be one of ".h5", ".hdf5", ".H5", + * ".HDF5", ".hdf" to save in HDF5 file format. Otherwise, a binary file format is used. + * @param[in] fname file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/Ten' will + * write the Tensor to the dataset 'Ten' the group '/foo/bar' in the file. + * @param[in] mode the write mode:\n + * `w` Creates a new file. If the given file exists, its contents are destroyed.\n + * `x` Creates a new file. Fails if the given file exists already.\n + * `a` Opens for writing without overwriting any existing content. Creates the file if it + * doesn't exist. Only available for HDF5 files.\n + * `u` Opens for writing. Existing content will be updated(overwritten). + * Creates the file if it doesn't exist. Only available for HDF5 files. + * @note The common file ending for saving a Tensor in binary format is ".cytn". + * @warning HDF5 file format is strongly recommended for compatibility with other libraries, + * readability, and future-proofing. + * @see Load() + */ + void Save(const std::filesystem::path &fname, const std::string &path = "/Tensor", + const char mode = 'w') const; + /** + * @see Save(const std::filesystem::path &fname, const std::string &path, const char mode) + * const; + */ + void Save(const char *fname, const std::string &path = "/Tensor", const char mode = 'w') const; + + /** + * @brief Load Tensor from file and create new instance + * @details This function creates a new Tensor and keeps the original Tensor unchanged. See + * Load_() for loading the Tensor to the current Tensor. + * @param fname[in] file name + * @param[in] path path inside the file. Only used for HDF5 files. A path /foo/bar/Ten will read + * the Tensor from the dataset 'Ten' the group '/foo/bar' in the file. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @pre The file must be a Tensor object which is saved by Save(). + * @note For HDF5 file format, one of the file endings ".h5", ".hdf5", ".H5", ".HDF5", ".hdf" is + * expected. For binary format, the common file ending for a Tensor is ".cytn". + */ + static cytnx::Tensor Load(const std::filesystem::path &fname, + const std::string &path = "/Tensor", bool restore_device = true); + /** + * @see Load(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + static cytnx::Tensor Load(const char *fname, const std::string &path = "/Tensor", + bool restore_device = true); - /// @endcond /** - @brief Save current Tensor to file - @param[in] fname file name (without file extension) + * @brief Load Tensor from file and overwrite current instance + * @details This function overwrites the existing Tensor. See Load() for creating a new Tensor. + * @see Load() + */ + void Load_(const std::filesystem::path &fname, const std::string &path = "/Tensor", + bool restore_device = true); + /** + * @see Load_(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + void Load_(const char *fname, const std::string &path = "/Tensor", bool restore_device = true); - @details - save the Tensor to file with file path specify with input param \p fname with postfix - ".cytn" - @see Load(const std::string &fname) - */ - void Save(const std::string &fname) const; /** - * @see Save(const std::string &fname) const + * @brief Save Tensor to HDF5 file + * @param[in] container the HDF5 parent group. + * @param[in] name the subgroup in which the Tensor will be saved. + * @param[in] overwrite overwrite previous Tensor information in the container. + * @warning This function is only available in C++. Use Save() for saving to file in C++ or + * Python. + * @see from_hdf5() */ - void Save(const char *fname) const; + void to_hdf5(H5::Group &container, const std::string &name = "Tensor", + const bool overwrite = false) const; + /** + * @brief Load Tensor from HDF5 file (inline) + * @param[in] container the HDF5 group where the Tensor will be loaded from. + * @param[in] name the name of the dataset in the HDF5 file. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @warning This function is only available in C++. Use Load() for loading from file in C++ or + * Python. + * @see to_hdf5() + */ + void from_hdf5(H5::Group &container, const std::string &name = "Tensor", + bool restore_device = true); + + /** + * @brief Save Tensor to binary file + * @param[in] f the output stream where the Tensor will be saved. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Save() for saving to file in C++ or Python. + * @see from_binary() + */ + void to_binary(std::ostream &f) const; + /** + * @brief Load Tensor from binary file + * @param[in] f the input stream from which the Tensor will be loaded. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Load() for loading from file in C++ or Python. + * @see to_binary() + */ + void from_binary(std::istream &f, bool restore_device = true); /** * @brief Save current Tensor to the binary file @@ -338,32 +430,21 @@ namespace cytnx { * @param fname[in] the file name of the binary file. * @pre The file name @p fname must be valid. * @see cytnx::Tensor::Fromfile + * @deprecated This function is deprecated. Please use \ref Save(const std::filesystem::path + * &fname) instead for saving raw data together with metadata. */ - void Tofile(const std::string &fname) const; - + [[deprecated("Please use Save(const std::filesystem::path &fname) instead.")]] void Tofile( + const std::filesystem::path &fname) const; /** - * @see Tofile(const std::string &fname) const + * @see Tofile(const std::filesystem::path &fname) const */ - void Tofile(const char *fname) const; - + [[deprecated("Please use Save(const std::filesystem::path &fname) instead.")]] void Tofile( + const char *fname) const; /** - * @see Tofile(const std::string &fname) const + * @see Tofile(const std::filesystem::path &fname) const */ - void Tofile(std::fstream &f) const; - - /** - @brief Load current Tensor from file - @param fname[in] file name - @details - load the Storage from file with file path specify with input param 'fname' - @pre the file must be a Tensor object which is saved by cytnx::Tensor::Save. - */ - - static Tensor Load(const std::string &fname); - /** - * @see Load(const std::string &fname) - */ - static Tensor Load(const char *fname); + [[deprecated("Please use to_binary(std::ostream &f) instead.")]] void Tofile( + std::fstream &f) const; /** * @brief Load current Tensor from the binary file @@ -371,11 +452,13 @@ namespace cytnx { * cytnx::Tensor::Tofile. Given the file name \p fname , data type \p dtype and * number of elements \p count, this function will load the first \p count elements * from the binary file \p fname with data type \p dtype. - * @param fname[in] the file name of the binary file. - * @param dtype[in] the data type of the binary file. This can be any of the type defined in + * @param[in] fname the file name of the binary file. + * @param[in] dtype the data type of the binary file. This can be any of the type defined in * cytnx::Type. - * @param count[in] the number of elements to be loaded from the binary file. If set to -1, + * @param[in] count the number of elements to be loaded from the binary file. If set to -1, * all elements in the binary file will be loaded. + * @param[in] device the device that tensor to be created. This can be cytnx::Device.cpu or + * cytnx::Device.cuda+, see cytnx::Device for more detail. * @return Tensor * @pre * 1. The @p dtype cannot be Type.Void. @@ -384,13 +467,16 @@ namespace cytnx { * 4. The @p Nelem cannot be larger than the number of elements in the binary file. * 5. The file name @p fname must be valid. * @see cytnx::Tensor::Tofile + * @deprecated This function is deprecated. Please use Save/Load functions instead for storing + * raw data together with metadata. */ - static Tensor Fromfile(const std::string &fname, const unsigned int &dtype, - const cytnx_int64 &count = -1); - static Tensor Fromfile(const char *fname, const unsigned int &dtype, - const cytnx_int64 &count = -1); - - // static Tensor Frombinary(const std::string &fname); + [[deprecated("Please use Save/Load functions instead.")]] static Tensor Fromfile( + const std::filesystem::path &fname, const unsigned int &dtype, const cytnx_int64 &count = -1, + const int device = Device.cpu); + [[deprecated("Please use Save/Load functions instead.")]] static Tensor Fromfile( + const char *fname, const unsigned int &dtype, const cytnx_int64 &count = -1, + const int device = Device.cpu); + // static Tensor Frombinary(const std::filesystem::path &fname); ///@cond boost::intrusive_ptr _impl; @@ -424,6 +510,7 @@ namespace cytnx { @param[in] shape the shape of tensor. @param[in] dtype the dtype of tensor. This can be any of type defined in cytnx::Type @param[in] device the device that tensor to be created. This can be cytnx::Device.cpu or + cytnx::Device.cuda+, see cytnx::Device for more detail. @param[in] init_zero if true, the content of Tensor will be initialized to zero. if false, the content of Tensor will be un-initialize. cytnx::Device.cuda+, see cytnx::Device for more detail. @@ -444,7 +531,7 @@ namespace cytnx { \verbinclude example/Tensor/Init.py.out */ void Init(const std::vector &shape, const unsigned int &dtype = Type.Double, - const int &device = -1, const bool &init_zero = true) { + const int &device = Device.cpu, const bool &init_zero = true) { boost::intrusive_ptr tmp(new Tensor_impl()); this->_impl = tmp; this->_impl->Init(shape, dtype, device, init_zero); @@ -455,7 +542,7 @@ namespace cytnx { // this->_impl->Init(storage); // } // void Init(const Storage& storage, const std::vector &shape, - // const unsigned int &dtype = Type.Double, const int &device = -1) { + // const unsigned int &dtype = Type.Double, const int &device = Device.cpu) { // boost::intrusive_ptr tmp(new Tensor_impl()); // this->_impl = tmp; // this->_impl->Init(storage, shape, dtype, device); @@ -474,7 +561,7 @@ namespace cytnx { * @see cytnx::Tensor::Init */ Tensor(const std::vector &shape, const unsigned int &dtype = Type.Double, - const int &device = -1, const bool &init_zero = 1) + const int &device = Device.cpu, const bool &init_zero = 1) : _impl(new Tensor_impl()) { this->Init(shape, dtype, device, init_zero); } @@ -483,31 +570,14 @@ namespace cytnx { // this->Init(storage); // } // Tensor(const Storage& storage, const std::vector &shape, - // const unsigned int &dtype = Type.Double, const int &device = -1) + // const unsigned int &dtype = Type.Double, const int &device = Device.cpu) // : _impl(new Tensor_impl()) { // this->Init(storage, shape, dtype, device); // } //@} - // This mechanism is to remove the 'void' type from Type_list. Taking advantage of it - // appearing first ... - - /// @cond - struct internal { - template - struct exclude_first; - - template - struct exclude_first> { - using type = std::variant; - }; - }; // internal - /// @endcond - // std::variant of pointers to Type_list, without void .... - using pointer_types = - make_variant_from_transform_t::type, - std::add_pointer>; + using pointer_types = make_variant_from_transform_t; // convert this->_impl->_storage._impl->Mem to a typed variant of pointers, excluding void* pointer_types ptr() const; @@ -527,9 +597,7 @@ namespace cytnx { #ifdef UNI_GPU // std::variant of pointers to Type_list_gpu, without void .... - using gpu_pointer_types = - make_variant_from_transform_t::type, - std::add_pointer>; + using gpu_pointer_types = make_variant_from_transform_t; // convert this->_impl->_storage->Mem to a typed variant of pointers, excluding void* gpu_pointer_types gpu_ptr() const; @@ -596,6 +664,12 @@ namespace cytnx { */ const std::vector &shape() const { return this->_impl->shape(); } + /** + @brief the strides of the Tensor + @return [std::vector] the strides of the Tensor + */ + const std::vector strides() const { return this->_impl->strides(); } + /** @brief the rank of the Tensor @return [cytnx_uint64] the rank of the Tensor @@ -1256,7 +1330,7 @@ namespace cytnx { * @param[in] rhs the added Tensor or scalar. */ template - Tensor Add(const T &rhs) { + Tensor Add(const T &rhs) const { return *this + rhs; } @@ -1276,7 +1350,7 @@ namespace cytnx { * @param[in] rhs the subtracted Tensor or scalar. */ template - Tensor Sub(const T &rhs) { + Tensor Sub(const T &rhs) const { return *this - rhs; } @@ -1296,7 +1370,7 @@ namespace cytnx { * @param[in] rhs the multiplied Tensor or scalar. */ template - Tensor Mul(const T &rhs) { + Tensor Mul(const T &rhs) const { return *this * rhs; } @@ -1317,7 +1391,7 @@ namespace cytnx { * @attension \p rhs cannot be zero. */ template - Tensor Div(const T &rhs) { + Tensor Div(const T &rhs) const { return *this / rhs; } @@ -1339,7 +1413,7 @@ namespace cytnx { * @param[in] rhs the compared object. */ template - Tensor Cpr(const T &rhs) { + Tensor Cpr(const T &rhs) const { return *this == rhs; } @@ -1382,7 +1456,7 @@ namespace cytnx { // } template - Tensor Mod(const T &rhs) { + Tensor Mod(const T &rhs) const { return *this % rhs; } @@ -1392,7 +1466,7 @@ namespace cytnx { * tensor is \f$A\f$, then the output tensor is \f$-A\f$. * @return The negation of the current tensor. */ - Tensor operator-() { return this->Mul(-1.); } + Tensor operator-() const { return this->Mul(-1.); } /** * @brief The flatten function. diff --git a/include/Type.hpp b/include/Type.hpp index c8880af4c..7826382f1 100644 --- a/include/Type.hpp +++ b/include/Type.hpp @@ -1,15 +1,17 @@ #ifndef CYTNX_TYPE_H_ #define CYTNX_TYPE_H_ +#include #include #include #include -#include #include -#include +#include #include -#include #include +#include + +#include "H5Cpp.h" #include "cytnx_error.hpp" // also brings in cuComplex.h @@ -73,7 +75,6 @@ namespace cytnx { return index_in_tuple_helper(); } } - } // namespace internal // helper metafunction to transform a variant into another variant via a @@ -154,6 +155,61 @@ namespace cytnx { cytnx_uint64, cytnx_int32, cytnx_uint32, cytnx_int16, cytnx_uint16, cytnx_bool>; #endif + namespace internal { + /// @cond + // This mechanism is to remove the 'void' type from Type_list. Taking advantage of it + // appearing first ... + struct truncate_variant { + template + struct exclude_first; + + template + struct exclude_first> { + using type = std::variant; + }; + }; // truncate_variant + /// @endcond + } // namespace internal + + // the list of supported Scalar types. Removes void because it cannot be used in visit + using Scalar_list = internal::truncate_variant::exclude_first::type; + +#ifdef UNI_GPU + using Scalar_list_gpu = internal::truncate_variant::exclude_first::type; +#endif + + namespace internal { + /// @cond + // This is create a variant containing vectors of all supported types + template + struct vector_variant; + + template + struct vector_variant> { + using type = std::variant...>; + }; + /// @endcond + } // namespace internal + + // the list of vectors of all supported Scalar types. + using Vector_list = typename internal::vector_variant::type; + + namespace internal { + /// @cond + // This is create a variant containing vectors of vectors of all supported types + template + struct matrix_variant; + + template + struct matrix_variant> { + using type = std::variant>...>; + }; + /// @endcond + } // namespace internal + + // the list of vectors of vectors of all supported Scalar types. + using Matrix_list = typename internal::matrix_variant::type; + // The number of supported types constexpr int N_Type = std::variant_size_v; constexpr int N_fType = 5; @@ -393,6 +449,105 @@ namespace cytnx { using type_promote_from_gpu_pointer_t = typename type_promote_from_gpu_pointer::type; #endif + H5::DataType dtype_to_hdf5_type(unsigned int type_id) const { + switch (type_id) { + case Type::Double: + return H5::PredType::NATIVE_DOUBLE; + + case Type::Float: + return H5::PredType::NATIVE_FLOAT; + + case Type::Int64: + return H5::PredType::NATIVE_INT64; + + case Type::Uint64: + return H5::PredType::NATIVE_UINT64; + + case Type::Int32: + return H5::PredType::NATIVE_INT32; + + case Type::Uint32: + return H5::PredType::NATIVE_UINT32; + + case Type::Int16: + return H5::PredType::NATIVE_INT16; + + case Type::Uint16: + return H5::PredType::NATIVE_UINT16; + + case Type::Bool: + return H5::PredType::NATIVE_HBOOL; + +#if H5_VERSION_GE(2, 0, 0) + case Type::ComplexDouble: + return H5::DataType(H5Tcopy(H5T_NATIVE_DOUBLE_COMPLEX)); + + case Type::ComplexFloat: + return H5::DataType(H5Tcopy(H5T_NATIVE_FLOAT_COMPLEX)); +#else + case Type::ComplexDouble: { + static const H5::CompType ct = [] { + H5::CompType tmp(sizeof(cytnx_complex128)); + tmp.insertMember("r", 0, H5::PredType::NATIVE_DOUBLE); + tmp.insertMember("i", sizeof(double), H5::PredType::NATIVE_DOUBLE); + return tmp; + }(); + return ct; + } + + case Type::ComplexFloat: { + static const H5::CompType ct = [] { + H5::CompType tmp(sizeof(cytnx_complex64)); + tmp.insertMember("r", 0, H5::PredType::NATIVE_FLOAT); + tmp.insertMember("i", sizeof(float), H5::PredType::NATIVE_FLOAT); + return tmp; + }(); + return ct; + } +#endif + + case Type::Void: + cytnx_error_msg(true, "[ERROR] Void dtype cannot be mapped to HDF5%s", "\n"); + + default: + cytnx_error_msg(true, "[ERROR] Unsupported Cytnx dtype: %s\n", getname(type_id).c_str()); + } + } + template + H5::DataType get_hdf5_type(const T& rc) const { + return dtype_to_hdf5_type(cy_typeid(rc)); + } + + unsigned int from_hdf5_type(const H5::DataType& h5_type) const { + if (h5_type == H5::PredType::NATIVE_DOUBLE) return Type::Double; + if (h5_type == H5::PredType::NATIVE_FLOAT) return Type::Float; + if (h5_type == H5::PredType::NATIVE_INT64) return Type::Int64; + if (h5_type == H5::PredType::NATIVE_UINT64) return Type::Uint64; + if (h5_type == H5::PredType::NATIVE_INT32) return Type::Int32; + if (h5_type == H5::PredType::NATIVE_UINT32) return Type::Uint32; + if (h5_type == H5::PredType::NATIVE_INT16) return Type::Int16; + if (h5_type == H5::PredType::NATIVE_UINT16) return Type::Uint16; + if (h5_type == H5::PredType::NATIVE_HBOOL) return Type::Bool; +#if H5_VERSION_GE(2, 0, 0) + if (h5_type.getClass() == H5T_COMPLEX) { + if (h5_type.getSize() == sizeof(cytnx_complex128)) return Type::ComplexDouble; + if (h5_type.getSize() == sizeof(cytnx_complex64)) return Type::ComplexFloat; + } +#endif + if (h5_type.getClass() == H5T_COMPOUND) { // supporting older versions of HDF5 + H5::CompType ct(h5_type.getId()); + if (ct.getNmembers() == 2) { + H5::DataType m0 = ct.getMemberDataType(0); + H5::DataType m1 = ct.getMemberDataType(1); + if (m0 == m1 && m0.getClass() == H5T_FLOAT) { + if (ct.getSize() == sizeof(cytnx_complex128)) return Type::ComplexDouble; + if (ct.getSize() == sizeof(cytnx_complex64)) return Type::ComplexFloat; + } + } + } + cytnx_error_msg(true, "[ERROR] HDF5 DataType cannot be mapped to Cytnx dtype.%s", "\n"); + } + }; // Type_class /// @endcond @@ -402,7 +557,7 @@ namespace cytnx { * @details This is the variable about the data type of the UniTensor, Tensor, ... .\n * You can use it as following: * \code - * int type = Type.Double; + * int type = Type::Double; * \endcode * * The supported enumerations are as following: diff --git a/include/UniTensor.hpp b/include/UniTensor.hpp index 9cf6795f6..0968b578f 100644 --- a/include/UniTensor.hpp +++ b/include/UniTensor.hpp @@ -1,23 +1,27 @@ #ifndef CYTNX_UNITENSOR_H_ #define CYTNX_UNITENSOR_H_ -#include "Type.hpp" -#include "cytnx_error.hpp" -#include "Device.hpp" -#include "Tensor.hpp" -#include "utils/utils.hpp" -#include "intrusive_ptr_base.hpp" +#include +#include +#include +#include #include -#include #include +#include #include -#include -#include -#include -#include "Symmetry.hpp" +#include + +#include "H5Cpp.h" + #include "Bond.hpp" +#include "Device.hpp" #include "Generator.hpp" -#include +#include "Symmetry.hpp" +#include "Tensor.hpp" +#include "Type.hpp" +#include "cytnx_error.hpp" +#include "intrusive_ptr_base.hpp" +#include "utils/utils.hpp" #ifdef BACKEND_TORCH #else @@ -443,8 +447,11 @@ namespace cytnx { virtual const vec2d &get_itoi() const; virtual vec2d &get_itoi(); - virtual void _save_dispatch(std::fstream &f) const; - virtual void _load_dispatch(std::fstream &f); + virtual void to_hdf5_dispatch(H5::Group &container, const bool overwrite) const; + virtual void from_hdf5_dispatch(H5::Group &container, bool restore_device = true); + + virtual void to_binary_dispatch(std::ostream &f) const; + virtual void from_binary_dispatch(std::istream &f, bool restore_device = true); virtual ~UniTensor_base(){}; }; @@ -1078,8 +1085,11 @@ namespace cytnx { "\n"); } - void _save_dispatch(std::fstream &f) const; - void _load_dispatch(std::fstream &f); + void to_hdf5_dispatch(H5::Group &container, const bool overwrite) const; + void from_hdf5_dispatch(H5::Group &container, bool restore_device = true); + + void to_binary_dispatch(std::ostream &f) const; + void from_binary_dispatch(std::istream &f, bool restore_device = true); const std::vector &get_qindices(const cytnx_uint64 &bidx) const { cytnx_error_msg(true, "[ERROR] get_qindices can only be unsed on UniTensor with Symmetry.%s", @@ -1282,7 +1292,7 @@ namespace cytnx { std::vector inds(indices.begin(), indices.end()); - // find if the indices specify exists! + // find if the specified indices exists! cytnx_int64 b = -1; for (cytnx_uint64 i = 0; i < this->_inner_to_outer_idx.size(); i++) { if (inds == this->_inner_to_outer_idx[i]) { @@ -1754,8 +1764,11 @@ namespace cytnx { cytnx_uint16 &at_for_sparse(const std::vector &locator, const cytnx_uint16 &aux); cytnx_int16 &at_for_sparse(const std::vector &locator, const cytnx_int16 &aux); - void _save_dispatch(std::fstream &f) const; - void _load_dispatch(std::fstream &f); + void to_hdf5_dispatch(H5::Group &container, const bool overwrite) const; + void from_hdf5_dispatch(H5::Group &container, bool restore_device = true); + + void to_binary_dispatch(std::ostream &f) const; + void from_binary_dispatch(std::istream &f, bool restore_device = true); // this will remove the [q_index]-th qnum at [bond_idx]-th Bond! void truncate_(const std::string &label, const cytnx_uint64 &q_index); @@ -2550,8 +2563,11 @@ namespace cytnx { cytnx_uint16 &at_for_sparse(const std::vector &locator, const cytnx_uint16 &aux); cytnx_int16 &at_for_sparse(const std::vector &locator, const cytnx_int16 &aux); - void _save_dispatch(std::fstream &f) const; - void _load_dispatch(std::fstream &f); + void to_hdf5_dispatch(H5::Group &container, const bool overwrite) const; + void from_hdf5_dispatch(H5::Group &container, bool restore_device = true); + + void to_binary_dispatch(std::ostream &f) const; + void from_binary_dispatch(std::istream &f, bool restore_device = true); // this will remove the [q_index]-th qnum at [bond_idx]-th Bond! void truncate_(const std::string &label, const cytnx_uint64 &q_index); @@ -2763,6 +2779,10 @@ namespace cytnx { } //@} + /// @cond + void Init(const std::string name); + /// @endcond + //@{ /** @brief Construct a UniTensor. @@ -5457,40 +5477,115 @@ namespace cytnx { } /** - @brief save a UniTensor to file - @details Save a UniTensor to file. The file extension will be extended as '.cytnx' - @param[in] fname the file name (exclude the file extension). - @see Load(const std::string &fname) - */ - void Save(const std::string &fname) const; + * @brief Save UniTensor to file + * @details Save the UniTensor to a file. The file ending should be one of ".h5", ".hdf5", + * ".H5", ".HDF5", ".hdf" to save in HDF5 file format. Otherwise, a binary file format is used. + * @param[in] fname file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/' will write + * the UniTensor to the group '/foo/bar' in the file. + * @param[in] mode the write mode:\n + * `w` Creates a new file. If the given file exists, its contents are destroyed.\n + * `x` Creates a new file. Fails if the given file exists already.\n + * `a` Opens for writing without overwriting any existing content. Creates the file if it + * doesn't exist. Only available for HDF5 files.\n + * `u` Opens for writing. Existing content will be updated(overwritten). + * Creates the file if it doesn't exist. Only available for HDF5 files. + * @note The common file ending for saving a UniTensor in binary format is ".cytnx". + * @warning HDF5 file format is strongly recommended for compatibility with other libraries, + * readability, and future-proofing. + * @see Load() + */ + void Save(const std::filesystem::path &fname, const std::string &path = "/UniTensor/", + const char mode = 'w') const; + /** + * @see Save(const std::filesystem::path &fname, const std::string &path, const char mode) + * const; + */ + void Save(const char *fname, const std::string &path = "/UniTensor/", + const char mode = 'w') const; + + /** + * @brief Load UniTensor from file and create new instance + * @details This function creates a new UniTensor and keeps the original UniTensor unchanged. + * See Load_() for loading the UniTensor to the current UniTensor. + * @param fname[in] file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/' will read + * the UniTensor from the group '/foo/bar' in the file. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @pre The file must be a UniTensor object which is saved by Save(). + * @note For HDF5 file format, one of the file endings ".h5", ".hdf5", ".H5", ".HDF5", ".hdf" is + * expected. For binary format, the common file ending for a UniTensor is ".cytnx". + */ + static UniTensor Load(const std::filesystem::path &fname, + const std::string &path = "/UniTensor/", bool restore_device = true); + /** + * @see Load(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + static UniTensor Load(const char *fname, const std::string &path = "/UniTensor/", + bool restore_device = true); /** - @brief save a UniTensor to file - @details Save a UniTensor to file. The file extension will be extended as '.cytnx' - @param[in] fname the file name (exclude the file extension). - @see Load(const char *fname) - */ - void Save(const char *fname) const; + * @brief Load UniTensor from file and overwrite current instance + * @details This function overwrites the existing UniTensor. See Load() for creating a new + * UniTensor. + * @see Load() + */ + void Load_(const std::filesystem::path &fname, const std::string &path = "/UniTensor/", + bool restore_device = true); + /** + * @see Load_(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + void Load_(const char *fname, const std::string &path = "/UniTensor/", + bool restore_device = true); /** - @brief load a UniTensor from file - @param[in] fname the file name - @return the loaded UniTensor - @pre The file must be a UniTensor object. That is, the file must be created by - UniTensor::Save(). - @see Save(const std::string &fname) const - */ - static UniTensor Load(const std::string &fname); + * @brief Save UniTensor to HDF5 file + * @param[in] container the HDF5 parent group. + * @param[in] name the subgroup in which the UniTensor will be saved. + * @param[in] overwrite overwrite previous UniTensor information in the container. + * @warning This function is only available in C++. Use Save() for saving to file in C++ or + * Python. + * @see from_hdf5() + */ + void to_hdf5(H5::Group &container, const std::string &name = "UniTensor", + const bool overwrite = false) const; + /** + * @brief Load UniTensor from HDF5 file (inline) + * @param[in] container the HDF5 parent group. + * @param[in] name the subgroup from which the UniTensor will be loaded. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @warning This function is only available in C++. Use Load() for loading from file in C++ or + * Python. + * @see to_hdf5() + */ + void from_hdf5(H5::Group &container, const std::string &name = "UniTensor", + bool restore_device = true); /** - @brief load a UniTensor from file - @param[in] fname: the file name - @return the loaded UniTensor - @pre The file must be a UniTensor object. That is, the file must be created by - UniTensor::Save(). - @see Save(const char* fname) const - */ - static UniTensor Load(const char *fname); + * @brief Save UniTensor to binary file + * @param[in] f the output stream where the UniTensor will be saved. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Save() for saving to file in C++ or Python. + * @see from_binary() + */ + void to_binary(std::ostream &f) const; + /** + * @brief Load UniTensor from binary file + * @param[in] f the input stream from which the UniTensor will be loaded. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Load() for loading from file in C++ or Python. + * @see to_binary() + */ + void from_binary(std::istream &f, bool restore_device = true); /** * @brief truncate bond dimension of the UniTensor by the given bond label and dimension. @@ -5578,11 +5673,6 @@ namespace cytnx { const vec2d &get_itoi() const { return this->_impl->get_itoi(); } vec2d &get_itoi() { return this->_impl->get_itoi(); } - /// @cond - void _Load(std::fstream &f); - void _Save(std::fstream &f) const; - /// @endcond - UniTensor &convert_from(const UniTensor &rhs, const bool &force = false, const cytnx_double &tol = 1e-14) { this->_impl->from_(rhs._impl, force, tol); @@ -5684,7 +5774,7 @@ namespace cytnx { @note The resulting UniTensor has two bonds. The data is one-dimensional (two-dimensional) if is_diag is true (false). @see cytnx::UniTensor::identity - @note This function is a alias of cytnx::UniTensor::identity(). + @note This function is an alias of cytnx::UniTensor::identity(). */ static UniTensor eye(const cytnx_uint64 &dim, const std::vector &in_labels = {}, const cytnx_bool &is_diag = false, const unsigned int &dtype = Type.Double, @@ -5945,8 +6035,7 @@ namespace cytnx { @param[in] cacheR if the inR should be contiguous align after calling @return [UniTensor] - - @see cytnx::UniTensor::contract + @see cytnx::Contract() */ UniTensor Contract(const UniTensor &inL, const UniTensor &inR, const bool &cacheL = false, @@ -5960,9 +6049,7 @@ namespace cytnx { @param[in] optimal wheather to find the optimal contraction order automatically. @return [UniTensor] - - See also \link cytnx::UniTensor::contract UniTensor.contract \endlink - + @see cytnx::Contract() */ UniTensor Contract(const std::vector &TNs, const std::string &order, const bool &optimal); @@ -5995,9 +6082,7 @@ namespace cytnx { @param args the Tensors. @return [UniTensor] - - See also \link cytnx::UniTensor::contract UniTensor.contract \endlink - + @see cytnx::Contract() */ template UniTensor Contract(const UniTensor &in, const T &...args, const std::string &order, diff --git a/include/backend/Storage.hpp b/include/backend/Storage.hpp index 4c693a280..719c09a48 100644 --- a/include/backend/Storage.hpp +++ b/include/backend/Storage.hpp @@ -4,6 +4,7 @@ #ifndef BACKEND_TORCH #include + #include #include #include #include @@ -11,13 +12,14 @@ #include #include + #include "H5Cpp.h" #include "boost/smart_ptr/intrusive_ptr.hpp" + #include "Device.hpp" + #include "Type.hpp" #include "backend/Scalar.hpp" #include "cytnx_error.hpp" - #include "Device.hpp" #include "intrusive_ptr_base.hpp" - #include "Type.hpp" #define STORAGE_DEFT_SZ 2 @@ -162,10 +164,11 @@ namespace cytnx { // these is the one that do the work, and customize with Storage_base // virtual void Init(const std::vector &init_shape); - virtual void Init(const unsigned long long &len_in, const int &device = -1, + virtual void Init(const unsigned long long &len_in, const int &device = Device.cpu, const bool &init_zero = true); - virtual void _Init_byptr(void *rawptr, const unsigned long long &len_in, const int &device = -1, - const bool &iscap = false, const unsigned long long &cap_in = 0); + virtual void _Init_byptr(void *rawptr, const unsigned long long &len_in, + const int &device = Device.cpu, const bool &iscap = false, + const unsigned long long &cap_in = 0); // this function will return a new storage with the same type as the one // that initiate this function. @@ -242,9 +245,9 @@ namespace cytnx { public: StorageImplementation() : capacity_(0), size_(0), start_(nullptr), dtype_(Type.cy_typeid(DType())), device_(-1){}; - void Init(const unsigned long long &len_in, const int &device = -1, + void Init(const unsigned long long &len_in, const int &device = Device.cpu, const bool &init_zero = true); - void _Init_byptr(void *rawptr, const unsigned long long &len_in, const int &device = -1, + void _Init_byptr(void *rawptr, const unsigned long long &len_in, const int &device = Device.cpu, const bool &iscap = false, const unsigned long long &cap_in = 0); boost::intrusive_ptr _create_new_sametype(); boost::intrusive_ptr clone(); @@ -435,7 +438,7 @@ namespace cytnx { extern Storage_init_interface __SII; ///@endcond; - ///@brief an memeory storage with multi-type/multi-device support + ///@brief a memory storage with multi-type/multi-device support class Storage { private: // Interface: @@ -465,13 +468,13 @@ namespace cytnx { \verbinclude example/Storage/Init.py.out */ void Init(const unsigned long long &size, const unsigned int &dtype = Type.Double, - int device = -1, const bool &init_zero = true) { + int device = Device.cpu, const bool &init_zero = true) { cytnx_error_msg(dtype >= N_Type, "%s", "[ERROR] invalid argument: dtype"); this->_impl = __SII.USIInit[dtype](); this->_impl->Init(size, device, init_zero); } // void _Init_byptr(void *rawptr, const unsigned long long &len_in, const unsigned int &dtype = - // Type.Double, const int &device = -1, + // Type.Double, const int &device = Device.cpu, // const bool &iscap = false, const unsigned long long &cap_in = // 0){ // cytnx_error_msg(dtype >= N_Type, "%s", "[ERROR] invalid argument: dtype"); @@ -488,12 +491,12 @@ namespace cytnx { * &init_zero) */ Storage(const unsigned long long &size, const unsigned int &dtype = Type.Double, - int device = -1, const bool &init_zero = true) + int device = Device.cpu, const bool &init_zero = true) : _impl(new Storage_base()) { Init(size, dtype, device, init_zero); } // Storage(void *rawptr, const unsigned long long &len_in, const unsigned int &dtype = - // Type.Double, const int &device = -1, + // Type.Double, const int &device = Device.cpu, // const bool &iscap = false, const unsigned long long &cap_in = 0) // : _impl(new Storage_base()){ // _Init_byptr(rawptr,len_in,dtype,device,iscap,cap_in); @@ -523,78 +526,217 @@ namespace cytnx { ///@endcond - /// @cond - void _Save(std::fstream &f) const; - void _Load(std::fstream &f); - void _Loadbinary(std::fstream &f, const unsigned int &dtype, const cytnx_uint64 &Nelem); - void _Savebinary(std::fstream &f) const; + /** + * @brief Save Storage to file + * @details Save the Storage to a file. The file ending should be one of ".h5", ".hdf5", ".H5", + * ".HDF5", ".hdf" to save in HDF5 file format. Otherwise, a binary file format is used. + * @param[in] fname file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/Ten' will + * write the Storage to the dataset 'Ten' the group '/foo/bar' in the file. + * @param[in] mode the write mode:\n + * `w` Creates a new file. If the given file exists, its contents are destroyed.\n + * `x` Creates a new file. Fails if the given file exists already.\n + * `a` Opens for writing without overwriting any existing content. Creates the file if it + * doesn't exist. Only available for HDF5 files.\n + * `u` Opens for writing. Existing content will be updated(overwritten). + * Creates the file if it doesn't exist. Only available for HDF5 files. + * @note The common file ending for saving a Storage in binary format is ".cyst". + * @warning HDF5 file format is strongly recommended for compatibility with other libraries, + * readability, and future-proofing. + * @see Load() + */ + void Save(const std::filesystem::path &fname, const std::string &path = "/Storage", + const char mode = 'w') const; + /** + * @see Save(const std::filesystem::path &fname, const std::string &path, const char mode) + * const; + */ + void Save(const char *fname, const std::string &path = "/Storage", const char mode = 'w') const; - /// @endcond + /** + * @brief Load Storage from file and create new instance + * @details This function creates a new Storage and keeps the original Storage unchanged. See + * Load_() for loading the Storage to the current Storage. + * @param fname[in] file name + * @param[in] path path inside the file. Only used for HDF5 files. A path /foo/bar/Ten will read + * the Storage from the dataset 'Ten' the group '/foo/bar' in the file. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @pre The file must be a Storage object which is saved by Save(). + * @note For HDF5 file format, one of the file endings ".h5", ".hdf5", ".H5", ".HDF5", ".hdf" is + * expected. For binary format, the common file ending for a Storage is ".cyst". + */ + static cytnx::Storage Load(const std::filesystem::path &fname, + const std::string &path = "/Storage", bool restore_device = true); + /** + * @see Load(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + static cytnx::Storage Load(const char *fname, const std::string &path = "/Storage", + bool restore_device = true); /** - @brief Save current Storage to file - @param[in] fname file name - @details - Save the Storage to file with file path specify with input param \p fname with postfix - ".cyst" - @post The file extension will be ".cyst". - */ - void Save(const std::string &fname) const; + * @brief Load Storage from file and overwrite current instance + * @details This function overwrites the existing Storage. See Load() for creating a new + * Storage. + * @see Load() + */ + void Load_(const std::filesystem::path &fname, const std::string &path = "/Storage", + bool restore_device = true); + /** + * @see Load_(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + void Load_(const char *fname, const std::string &path = "/Storage", bool restore_device = true); /** - * @brief Save current Storage to file, same as \ref Save(const std::string &fname) + * @brief Save Storage to HDF5 file + * @param[in] container the HDF5 group where the Storage will be saved. + * @param[in] name the name of the dataset in the HDF5 file. + * @param[in] overwrite overwrite previous Bond information in the container. + * @warning This function is only available in C++. Use Save() for saving to file in C++ or + * Python. + * @see from_hdf5() */ - void Save(const char *fname) const; + void to_hdf5(H5::Group &container, const std::string &name = "Storage", + const bool overwrite = false) const; /** - * @brief Save current Storage to a binary file, which only contains the raw data. - * @see Fromfile(const std::string &fname, const unsigned int &dtype, const cytnx_int64 &count) + * @brief Load Storage from HDF5 file (inline) + * @param[in] container the HDF5 group where the Storage will be loaded from. + * @param[in] name the name of the dataset in the HDF5 file. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @warning This function is only available in C++. Use Load() for loading from file in C++ or + * Python. + * @see to_hdf5() */ - void Tofile(const std::string &fname) const; - /// @see Tofile(const std::string &fname) const - void Tofile(const char *fname) const; - /// @see Tofile(const std::string &fname) const - void Tofile(std::fstream &f) const; + void from_hdf5(H5::Group &container, const std::string &name = "Storage", + bool restore_device = true); /** - @brief Load current Storage from file - @param[in] fname file name - @details - load the Storage from file with file path specify with input param 'fname'. - @pre The file must be a Storage object, which is saved by the function - Save(const std::string &fname) const. - */ - static Storage Load(const std::string &fname); + * @brief Save only the data of the Storage to HDF5 dataset. + * @param[in] dataset the HDF5 dataset where the Storage will be saved. + * @param[in] hdf5type the HDF5 data type for the dataset, must match the data type of the + * Storage. + * @warning This function is only available in C++. Use \link Save(const std::filesystem::path + * &fname) Save() \endlink for saving to file in C++ or Python. + * @see data_from_hdf5(H5::DataSet &dataset, const cytnx_uint64 &Nelem, const unsigned int + * &dtype, H5::DataType &hdf5type, const int &device) + */ + void data_to_hdf5(H5::DataSet &dataset, H5::DataType &hdf5type) const; + /** + * @brief Load only the data of the Storage from an HDF5 dataset, for given Storage parameters. + * @param[in] dataset the HDF5 dataset from which the Storage will be loaded. + * @param[in] Nelem the number of elements to load from the HDF5 dataset. This should match the + * size of the Storage. + * @param[in] dtype the data type of the Storage. See cytnx.Type. This should match the data + * type of the HDF5 dataset. + * @param[in] hdf5type the HDF5 data type for the dataset, must match the data type of the + * Storage. + * @param[in] device the device on which the Storage will be loaded. + * @note This function overwrites the current Storage with a new instance. + * @warning This function is only available in C++. Use \link Save(const std::filesystem::path + * &fname) Save() \endlink for saving to file in C++ or Python. + * @see data_to_hdf5(H5::DataSet &dataset, H5::DataType &hdf5type) + */ + void data_from_hdf5(H5::DataSet &dataset, const cytnx_uint64 &Nelem, const unsigned int &dtype, + H5::DataType &hdf5type, const int &device = Device.cpu); /** - * @brief Load current Storage from file, same as \ref Load(const std::string &fname) + * @brief Save Storage to binary file + * @param[in] f the output stream where the Storage will be saved. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Save() for saving to file in C++ or Python. + * @see from_binary() */ - static Storage Load(const char *fname); + void to_binary(std::ostream &f) const; /** - * @brief Load the binary file, which only contains the raw data, to current Storage. + * @brief Load Storage from binary file + * @param[in] f the input stream from which the Storage will be loaded. + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Use .to_() to move it to the target device after + * loading. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Load() for loading from file in C++ or Python. + * @see to_binary() + */ + void from_binary(std::istream &f, bool restore_device = true); + + /** + * @brief Save only the data of the Storage to binary filestream. + * @param[in] f the output stream where the Storage will be saved. + * @warning This function is only available in C++. Use \link Save(const std::filesystem::path + * &fname) Save() \endlink for saving to file in C++ or Python. + * @see data_from_binary(std::istream &f, const cytnx_uint64 &Nelem, const unsigned int &dtype, + * const int &device) + */ + void data_to_binary(std::ostream &f) const; + /** + * @brief Load Storage from binary file + * @param[in] f the input stream from which the Storage will be loaded. + * @param[in] Nelem the number of elements to load from the dataset. + * @param[in] dtype the data type of the Storage. See cytnx.Type. + * @param[in] device the device on which the Storage will be loaded. + * @note This function overwrites the current Storage with a new instance. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use \link Load(const std::filesystem::path &fname, bool restore_device) + * Load() \endlink for loading from file in C++ or Python. + * @see data_to_binary(std::ostream &f) const + */ + void data_from_binary(std::istream &f, const cytnx_uint64 &Nelem, const unsigned int &dtype, + const int &device = Device.cpu); + + /** + * @brief Save current Storage to a binary file, which only contains the raw data. + * @see Fromfile(const std::filesystem::path &fname, const unsigned int &dtype, const + * cytnx_int64 &count) + * @deprecated This function is deprecated. Please use \ref Save(const std::filesystem::path + * &fname) instead for saving raw data together with metadata. + */ + [[deprecated("Please use Save(const std::filesystem::path &fname) instead.")]] void Tofile( + const std::filesystem::path &fname) const; + /** + * @see Tofile(const std::filesystem::path &fname) const + */ + [[deprecated("Please use Save(const std::filesystem::path &fname) instead.")]] void Tofile( + const char *fname) const; + /** + * @see Tofile(const std::filesystem::path &fname) const + */ + [[deprecated("Please use to_binary(std::ostream &f) instead.")]] void Tofile( + std::fstream &f) const; + + /** + * @brief Load the binary file, which only contains the raw data. * @details This function will load the binary file, which only contains the raw data, * to current Storage with specified dtype and number of elements. * @param[in] fname file name - * @param[in] dtype the data type of the binary file. See cytnx::Type. - * @param[in] Nelem the number of elements you want to load from the binary file. If - * \p Nelem is -1, then it will load all the elements in the binary file. - * @pre + * @param[in] dtype the data type of the binary file. See cytnx.Type. + * @param[in] count the number of elements you want to load from the binary file. If + * \p count is -1, then it will load all the elements in the binary file. + * @param[in] device the device on which the data will be loaded. * 1. The @p dtype cannot be Type.Void. * 2. The @p dtype must be the same as the data type of the binary file. - * 3. The @p Nelem cannot be 0. - * 4. The @p Nelem cannot be larger than the number of elements in the binary file. + * 3. The @p count cannot be 0. + * 4. The @p count cannot be larger than the number of elements in the binary file. * 5. The file name @p fname must be valid. * - * @see Tofile(const std::string &fname) const + * @see Tofile(const std::filesystem::path &fname) const + * @deprecated This function is deprecated. Please use Save/Load functions instead for storing + * raw data together with metadata. */ - static Storage Fromfile(const std::string &fname, const unsigned int &dtype, - const cytnx_int64 &count = -1); - + [[deprecated("Please use Save/Load functions instead.")]] static Storage Fromfile( + const std::filesystem::path &fname, const unsigned int &dtype, const cytnx_int64 &count = -1, + const int device = Device.cpu); /** - * @see Fromfile(const std::string &fname, const unsigned int &dtype, const cytnx_int64 &count = - * -1) + * @see Fromfile(const std::filesystem::path &fname, const unsigned int &dtype, const + * cytnx_int64 &count) */ - static Storage Fromfile(const char *fname, const unsigned int &dtype, - const cytnx_int64 &count = -1); + [[deprecated("Please use Save/Load functions instead.")]] static Storage Fromfile( + const char *fname, const unsigned int &dtype, const cytnx_int64 &count = -1, + const int device = Device.cpu); /** @brief cast the type of current Storage @@ -824,7 +966,7 @@ namespace cytnx { @note This function is C++ only */ template - static Storage from_vector(const std::vector &vin, const int device = -1) { + static Storage from_vector(const std::vector &vin, const int device = Device.cpu) { Storage out; out._from_vector(vin, device); return out; @@ -867,7 +1009,7 @@ namespace cytnx { /// @cond template - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { // auto dispatch: // check: cytnx_error_msg(1, "[FATAL] ERROR unsupport type%s", "\n"); @@ -875,57 +1017,57 @@ namespace cytnx { // memcpy(this->_impl->data(),&vin[0],sizeof(T)*vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.ComplexDouble](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_complex128) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.ComplexFloat](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_complex64) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Double](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_double) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Float](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_float) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Uint64](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_uint64) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Int64](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_int64) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Uint32](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_uint32) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Int32](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_int32) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Uint16](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_uint16) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Int16](); this->_impl->Init(vin.size(), device); memcpy(this->_impl->data(), &vin[0], sizeof(cytnx_int16) * vin.size()); } - void _from_vector(const std::vector &vin, const int device = -1) { + void _from_vector(const std::vector &vin, const int device = Device.cpu) { this->_impl = __SII.USIInit[Type.Bool](); this->_impl->Init(vin.size(), device); this->_impl->_cpy_bool(this->_impl->data(), vin); diff --git a/include/backend/Tensor_impl.hpp b/include/backend/Tensor_impl.hpp index 524a62ffa..6cafd67d2 100644 --- a/include/backend/Tensor_impl.hpp +++ b/include/backend/Tensor_impl.hpp @@ -51,7 +51,7 @@ namespace cytnx { Tensor_impl() : _contiguous(true){}; void Init(const std::vector &shape, const unsigned int &dtype = Type.Double, - int device = -1, const bool &init_zero = true); + int device = Device.cpu, const bool &init_zero = true); void Init(const Storage &in); // void Init(const Storage &in, const std::vector &shape, // const unsigned int &dtype, int device); @@ -78,6 +78,16 @@ namespace cytnx { const std::vector &shape() const { return _shape; } + const std::vector strides() const { + std::vector strides(this->_shape.size()); + cytnx_uint64 accu = 1; + for (cytnx_int64 i = this->_shape.size() - 1; i >= 0; i--) { + strides[this->_mapper[i]] = accu; // calculate strides here + accu *= this->_shape[this->_invmapper[i]]; + } + return strides; + } + const bool &is_contiguous() const { return this->_contiguous; } const std::vector &mapper() const { return this->_mapper; } diff --git a/include/cytnx.hpp b/include/cytnx.hpp index 619adb704..900d51ee2 100644 --- a/include/cytnx.hpp +++ b/include/cytnx.hpp @@ -38,6 +38,7 @@ #include "LinOp.hpp" #include "utils/is.hpp" #include "utils/print.hpp" +#include "io.hpp" #include "tn_algo/MPS.hpp" #include "tn_algo/MPO.hpp" diff --git a/include/io.hpp b/include/io.hpp new file mode 100644 index 000000000..6e73116e1 --- /dev/null +++ b/include/io.hpp @@ -0,0 +1,334 @@ +#ifndef CYTNX_IO_H_ +#define CYTNX_IO_H_ + +#include +#include + +#include "H5Cpp.h" + +#include "backend/Storage.hpp" +#include "Bond.hpp" +#include "tn_algo/MPS.hpp" +#include "Symmetry.hpp" +#include "Tensor.hpp" +#include "Type.hpp" +#include "UniTensor.hpp" +#include "cytnx_error.hpp" + +namespace cytnx { + /** + @namespace cytnx::io + @brief IO functions for saving an loading to HDF5 file format. + */ + namespace io { + + namespace internal { + ///@cond + /** + * @brief Check if a dataset of correct type and size exists, otherwise create a new one + * @param[in] datatype datatype to be written + * @param[in] dimensions size of the tensor to be written + * @param[in] container group to create dataset in + * @param[in] name the name of the dataset + * @param[in] overwrite if true, previous data will be overwritten or deleted. + * @returns an opened dataset, compatible for writing + */ + H5::DataSet create_dataset(H5::DataType &datatype, const std::vector &dimensions, + H5::Group &container, const std::string &name, bool overwrite); + ///@endcond + } // namespace internal + + /** + * @brief Create a group, given a path that can contain subpathes + * @details Opens the group or creates it newly + * @param[in] container root group + * @param[in] path a path that can contain subpathes + * @param[in] recursive if the path can contain subpathes, such that groups in group are opened + * or created + * @returns the opened group or a newly created group + */ + H5::Group create_group(H5::Group &container, const std::string &path, bool recursive = true); + + /** + * @brief Remove a group if dataset if it exists + * @param[in] container group or file from which the attribute will be removed + * @param[in] name the name of the dataset or group + * @param[in] overwrite if true, the data will be removed. If false, an error is thrown if the + * data exists + * @returns true if the name existed, false otherwise + */ + bool unlink(H5::Group &container, const std::string &name, bool overwrite = true); + + /** + * @brief Save scalar to an attribute + * @param[in] object a scalar of any type that is supported by cytnx + * @param[in] container file, group or dataset; should be opened for writing + * @param[in] name the name of the attribute + * @param[in] overwrite if true, overwrite previous data in the container + */ + void save_attribute(const Scalar_list &object, H5::H5Object &container, const std::string &name, + bool overwrite = false); + /** + * @brief Save string to an attribute + * @see save_attribute(const Scalar_list &object, H5::Group &container, const std::string &name) + */ + void save_attribute(const std::string &object, H5::H5Object &container, const std::string &name, + bool overwrite = false); + /** + * @brief Save vector of strings to an attribute + * @see save_attribute(const Scalar_list &object, H5::Group &container, const std::string &name) + */ + void save_attribute(const std::vector &object, H5::H5Object &container, + const std::string &name, bool overwrite = false); + + /** + * @brief Load scalar from an attribute + * @param[out] object a scalar of any type that is supported by cytnx; needs to be compatible + * with the data format in the file. + * @param[in] container file, group or dataset + * @param[in] name the name of the attribute + */ + template + void load_attribute(T &object, H5::H5Object &container, const std::string &name) { + H5::Attribute attr = container.openAttribute(name); + H5::DataType datatype = attr.getDataType(); + H5::DataType scalartype = Type.get_hdf5_type(object); + cytnx_error_msg(datatype.getClass() != scalartype.getClass(), + "[ERROR] Attribute '%s' data type class mismatch.\n", name.c_str()); + attr.read(scalartype, &object); + } + /** + * @brief Load string from an attribute + * @see load_attribute(T &object, H5::H5Object &container, const std::string &name) + */ + void load_attribute(std::string &object, H5::H5Object &container, const std::string &name); + + /** + * @brief Remove an attribute if it exists + * @param[in] container file, group or dataset from which the attribute will be removed + * @param[in] name the name of the attribute + * @param[in] overwrite if true, the attribute will be removed. If false, an error is thrown if + * the attribute exists + * @returns true if the attribute existed, false otherwise + */ + bool remove_attribute(H5::H5Object &container, const std::string &name, bool overwrite = true); + + /** + * @brief Save data to a dataset + * @param[in] object a vector of supported types + * @param[in] container file, group or dataset; should be opened for writing + * @param[in] name the name of the dataset + * @param[in] overwrite if true, overwrite previous data in the container + * @returns the dataset that was written to + */ + H5::DataSet save_dataset(const Vector_list &object, H5::Group &container, + const std::string &name, bool overwrite = false); + /** + * @brief Save string to a dataset + * @see save_dataset(const Vector_list &object, H5::Group &container, const std::string &name, + * bool overwrite) + */ + H5::DataSet save_dataset(const std::vector &object, H5::Group &container, + const std::string &name, bool overwrite = false); + /** + * @brief Save a matrix of a supported type + * @param[in] object a vector of a vector based on a supported type; if the outer vector has \em + * rownum elements, and all inner vectors have \em colnum elements, then the result will be + * written as a \em rownum \em x \em colnum matrix. + * @see save_dataset(const Vector_list &object, H5::Group &container, const std::string &name, + * bool overwrite) + * @note All inner vectors need to have the same length, such that the data structure + * corresponds to a rectangular matrix. + */ + H5::DataSet save_dataset(const Matrix_list &object, H5::Group &container, + const std::string &name, bool overwrite = false); + + /** + * @brief Load data from a dataset + * @param[out] object vector of supported types + * @param[in] container file, group or dataset + * @param[in] name the name of the attribute + */ + template + void load_dataset(std::vector &object, H5::H5Object &container, const std::string &name) { + H5::DataSet dataset = container.openDataSet(name); + H5::DataSpace dataspace = dataset.getSpace(); + object.resize(dataspace.getSimpleExtentNpoints()); + H5::DataType datatype = dataset.getDataType(); + T scalar; + H5::DataType scalartype = Type.get_hdf5_type(scalar); + cytnx_error_msg(datatype.getClass() != scalartype.getClass(), + "[ERROR] Dataset '%s' type class mismatch.\n", name.c_str()); + dataset.read(object.data(), scalartype); + } + + /** + * @brief Load vector of strings from a dataset + * @param[out] object vector of strings + * @see void load_dataset(std::vector &object, H5::H5Object &container, const std::string + * &name) + */ + void load_dataset(std::vector &object, H5::H5Object &container, + const std::string &name); + /** + * @brief Load a matrix of a supported type + * @param[out] object a vector of a vector based on a supported type + * @see save_dataset(const Matrix_list &object, H5::Group &container, const std::string &name, + * bool overwrite) + * @see void load_dataset(std::vector &object, H5::H5Object &container, const std::string + * &name) + */ + template + void load_dataset(std::vector> &object, H5::H5Object &container, + const std::string &name) { + H5::DataSet dataset = container.openDataSet(name); + H5::DataSpace dataspace = dataset.getSpace(); + cytnx_error_msg( + dataspace.getSimpleExtentNdims() != 2, + "[ERROR] Dataset '%s' should be a two-dimensional array. The HDF5 data seems corrupt!\n", + name.c_str()); + hsize_t dims[2]; + dataspace.getSimpleExtentDims(dims); + hsize_t rownum = dims[0]; + hsize_t colnum = dims[1]; + H5::DataType datatype = dataset.getDataType(); + T scalar; + H5::DataType scalartype = Type.get_hdf5_type(scalar); + cytnx_error_msg(datatype.getClass() != scalartype.getClass(), + "[ERROR] Dataset '%s' type class mismatch.\n", name.c_str()); + // Read HDF5 data into a flattened temporary vector + std::vector flat(rownum * colnum); + dataset.read(flat.data(), scalartype); + // Reconstruct the vector of vectors + object.assign(rownum, std::vector(colnum)); + for (hsize_t i = 0; i < rownum; ++i) { + std::copy(flat.begin() + i * colnum, flat.begin() + (i + 1) * colnum, object[i].begin()); + } + } + + /** + * @brief Input/output file access mode for HDF5 files. + */ + enum IoMode : int { + ACC_TRUNC, /**< Open file for reading and writing; overwrites if file exists; creates new file + otherwise */ + ACC_NOREPLACE, /**< Open file for reading and writing; fails if file exists; creates new file + otherwise */ + ACC_IN, /**< Open file for reading only; fails if file does not exist */ + ACC_INOUT /**< Open file for reading and writing; opens an existing file; creates new file + otherwise */ + }; + + /** + * @brief Open HDF5 file + * @details Use file.close() to close the file after use. + * @param[in] fname file name + * @param[in] mode the write mode:\n + * ACC_TRUNC Open file for reading and writing; overwrites if file exists; creates new file + * otherwise\n ACC_NOREPLACE Open file for reading and writing; fails if file exists; creates + * new file otherwise\n ACC_IN Open file for reading only; fails if file does not exist\n + * ACC_INOUT Open file for reading and writing; opens an existing file; creates new file + * otherwise + * @returns the file handle + * @note The file ending should be one of ".h5", ".hdf5", ".H5", ".HDF5", ".hdf". + */ + H5::H5File open(const std::filesystem::path &fname, IoMode mode = ACC_TRUNC); + + /** + * @brief Classes that can be saved and loaded to/from HDF5 + */ + using savable_class = std::variant; + + /** + * @brief Save object to HDF5 + * @details Can be used with most cytnx classes as objects. + * @param[in] object the cytnx object to be saved + * @param[in] container HDF5 object that should be opened for writing; can be a file or a group + * @param[in] path path inside the file; a path '/foo/bar/Obj' will write the object to the + * dataset 'Obj' the group '/foo/bar' in the file. + * @param[in] name the name of the object to save to + * @param[in] overwrite if true, overwrite previous data in the container + * @note The file ending should be one of ".h5", ".hdf5", ".H5", ".HDF5", ".hdf". + * @see Load(), open() + */ + void Save(const savable_class &object, H5::Group &container, const std::string &name, + const std::string &path = "", bool overwrite = false); + + /** + * @brief Open HDF5 file, save object, and close file. + * @param[in] object the cytnx object to be saved + * @param[in] fname file name + * @param[in] path path inside the file; a path '/foo/bar/Obj' will write the object to the + * dataset 'Obj' the group '/foo/bar' in the file. + * @param[in] name the name of the attribute, dataset, or group + * @param[in] mode the write mode:\n + * `w` write; creates a new file. If the given file exists, its contents are destroyed.\n + * `x` exclusive; creates a new file. Fails if the given file exists already.\n + * `a` append. opens for writing without overwriting any existing content. Creates the file if + * it doesn't exist.\n `u` update; opens for writing. Existing content will be + * updated(overwritten). Creates the file if it doesn't exist. + * @see Save(const savable_class &object, H5::Group &file, const std::string &name, const + * std::string &path, bool overwrite), open() + */ + + /// @cond + // void Save(const savable_class object, const std::filesystem::path &fname, const std::string + // &name, const std::string &path = "", const char mode = 'w') { + // cytnx_error_msg(mode == 'r', "Mode 'r' is read-only and not available for writing + // files.%s", "\n"); bool overwrite = (mode == 'u'); if (mode == 'a') + // mode = 'u'; + // hid_t file = open(fname, mode); + // Save(object, file, name, path, overwrite); + // Close(file); + // } + /// @endcond + + /** + * @brief Load object from HDF5 file + * @details Can be used with most cytnx classes as objects. + * @param[out] object an object of the correct type that will be modified (inline) + * @param[in] container HDF5 object that should be opened for writing; can be a file or a group + * @param[in] name the name of the attribute, dataset, or group + * @param[in] path path inside the file; a path /foo/bar/Obj will read the object from the + * dataset 'Obj' the group '/foo/bar' in the file. + * @param[in] name the name of the attribute, dataset, or group + * @pre The file must be an object which was saved by Save(). + * @see Save(), open() + */ + void Load(savable_class &object, H5::Group &container, const std::string &name, + const std::string &path = ""); + + /** + * @brief savable_class objects that can be loaded to different devices + */ + using loadable_to_device = std::variant; + + /** + * @brief Load object from HDF5 file and restore the device the data was saved to + * @param[in] restore_device whether to try restoring the device on which the data is stored; if + * false, the data will be kept on the CPU. Only effects objects that contain Tensor or Storage. + * @see Load(savable_class &object, H5::Group &container, const std::string &name, const + * std::string &path) + */ + void Load(loadable_to_device &object, H5::Group &container, const std::string &name, + const std::string &path, bool restore_device); + + /// @cond + /** + * @brief Open HDF5 file, load object, and close file. + * @param[in] fname file name + * @see Load(const savable_class object, const std::string &name, const std::filesystem::path + * &fname, const std::string &path, bool restore_device), open() + */ + // void Load(const savable_class object, const std::filesystem::path &fname, const std::string + // &name, const std::string &path = "", bool restore_device = true) { + // hid_t file = open(fname, 'r'); + // Load(object, file, name, path, restore_device); + // Close(file); + // } + /// @endcond + + } // namespace io +} // namespace cytnx + +#endif // CYTNX_IO_H_ diff --git a/include/tn_algo/MPS.hpp b/include/tn_algo/MPS.hpp index c289e0351..e7a7973a8 100644 --- a/include/tn_algo/MPS.hpp +++ b/include/tn_algo/MPS.hpp @@ -1,18 +1,21 @@ #ifndef CYTNX_TN_ALGO_MPS_H_ #define CYTNX_TN_ALGO_MPS_H_ -#include "cytnx_error.hpp" -#include "Device.hpp" -#include "intrusive_ptr_base.hpp" -#include "UniTensor.hpp" -#include +#include #include - -#include "utils/vec_clone.hpp" -#include "Accessor.hpp" -#include #include +#include #include +#include + +#include "H5Cpp.h" + +#include "Accessor.hpp" +#include "Device.hpp" +#include "UniTensor.hpp" +#include "cytnx_error.hpp" +#include "intrusive_ptr_base.hpp" +#include "utils/vec_clone.hpp" #ifdef BACKEND_TORCH #else @@ -74,8 +77,8 @@ namespace cytnx { virtual void S_mvleft(); virtual void S_mvright(); - virtual void _save_dispatch(std::fstream &f); - virtual void _load_dispatch(std::fstream &f); + virtual void to_binary_dispatch(std::ostream &f); + virtual void from_binary_dispatch(std::istream &f, bool restore_device = true); }; // finite size: @@ -116,8 +119,8 @@ namespace cytnx { return out; } - void _save_dispatch(std::fstream &f); - void _load_dispatch(std::fstream &f); + void to_binary_dispatch(std::ostream &f); + void from_binary_dispatch(std::istream &f, bool restore_device = true); }; // infinite size: @@ -160,8 +163,8 @@ namespace cytnx { return out; } Scalar norm() const; - void _save_dispatch(std::fstream &f); - void _load_dispatch(std::fstream &f); + void to_binary_dispatch(std::ostream &f); + void from_binary_dispatch(std::istream &f, bool restore_device = true); }; ///@endcond @@ -286,16 +289,115 @@ namespace cytnx { cytnx_int64 &S_loc() { return this->_impl->S_loc; } - ///@cond - void _Save(std::fstream &f) const; - void _Load(std::fstream &f); - ///@endcond - - void Save(const std::string &fname) const; - void Save(const char *fname) const; - - static MPS Load(const std::string &fname); - static MPS Load(const char *fname); + /** + * @brief Save MPS to file + * @details Save the MPS to a file. The file ending should be one of ".h5", ".hdf5", ".H5", + * ".HDF5", ".hdf" to save in HDF5 file format. Otherwise, a binary file format is used. + * @param[in] fname file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/' will + * write the MPS to the group '/foo/bar' in the file. + * @param[in] mode the write mode:\n + * `w` Creates a new file. If the given file exists, its contents are destroyed.\n + * `x` Creates a new file. Fails if the given file exists already.\n + * `a` Opens for writing without overwriting any existing content. Creates the file if it + * doesn't exist. Only available for HDF5 files.\n + * `u` Opens for writing. Existing content will be updated(overwritten). + * Creates the file if it doesn't exist. Only available for HDF5 files. + * @note The common file ending for saving a MPS in binary format is ".cymps". + * @warning HDF5 file format is strongly recommended for compatibility with other libraries, + * readability, and future-proofing. + * @see Load() + * @warning HDF5 file format is still under development for MPS. + */ + void Save(const std::filesystem::path &fname, const std::string &path = "/MPS/", + const char mode = 'w') const; + /** + * @see Save(const std::filesystem::path &fname, const std::string &path, const char mode) + * const; + */ + void Save(const char *fname, const std::string &path = "/MPS/", const char mode = 'w') const; + + /** + * @brief Load MPS from file and create new instance + * @details This function creates a new MPS and keeps the original MPS unchanged. See Load_() + * for loading the MPS to the current MPS. + * @param fname[in] file name + * @param[in] path path inside the file. Only used for HDF5 files. A path '/foo/bar/' will + * read the MPS from the group '/foo/bar' in the file. + * @param[in] restore_device whether to try restoring the device on which the data is stored; + * if false, the data will be kept on the CPU. Use .to_() to move it to the target device + * after loading. + * @pre The file must be a MPS object which is saved by Save(). + * @note For HDF5 file format, one of the file endings ".h5", ".hdf5", ".H5", ".HDF5", ".hdf" + * is expected. For binary format, the common file ending for a MPS is ".cymps". + * @warning HDF5 file format is still under development for MPS. + */ + static MPS Load(const std::filesystem::path &fname, const std::string &path = "/MPS/", + bool restore_device = true); + /** + * @see Load(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + static MPS Load(const char *fname, const std::string &path = "/MPS/", + bool restore_device = true); + + /** + * @brief Load MPS from file and overwrite current instance + * @details This function overwrites the existing MPS. See Load() for creating a new MPS. + * @see Load() + */ + void Load_(const std::filesystem::path &fname, const std::string &path = "/MPS/", + bool restore_device = true); + /** + * @see Load_(const std::filesystem::path &fname, const std::string &path, const bool + * restore_device) + */ + void Load_(const char *fname, const std::string &path = "/MPS/", bool restore_device = true); + + /** + * @brief Save MPS to HDF5 file + * @param[in] container the HDF5 parent group. + * @param[in] name the subgroup in which the MPS will be saved. + * @param[in] overwrite overwrite previous MPS information in the container. + * @warning This function is only available in C++. Use Save() for saving to file in C++ or + * Python. + * @see from_hdf5() + */ + void to_hdf5(H5::Group &container, const std::string &name = "MPS", + const bool overwrite = false) const; + /** + * @brief Load MPS from HDF5 file (inline) + * @param[in] container the HDF5 parent group. + * @param[in] name the subgroup from which the MPS will be loaded. + * @param[in] restore_device whether to try restoring the device on which the data is stored; + * if false, the data will be kept on the CPU. Use .to_() to move it to the target device + * after loading. + * @warning This function is only available in C++. Use Load() for loading from file in C++ or + * Python. + * @see to_hdf5() + */ + void from_hdf5(H5::Group &container, const std::string &name = "MPS", + bool restore_device = true); + + /** + * @brief Save MPS to binary file + * @param[in] f the output stream where the MPS will be saved. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Save() for saving to file in C++ or Python. + * @see from_binary() + */ + void to_binary(std::ostream &f) const; + /** + * @brief Load MPS from binary file + * @param[in] f the input stream from which the MPS will be loaded. + * @param[in] restore_device whether to try restoring the device on which the data is stored; + * if false, the data will be kept on the CPU. Use .to_() to move it to the target device + * after loading. + * @warning This function is only available in C++. In Python, use pickle for the same binary + * file format. Use Load() for loading from file in C++ or Python. + * @see to_binary() + */ + void from_binary(std::istream &f, bool restore_device = true); }; std::ostream &operator<<(std::ostream &os, const MPS &in); diff --git a/pybind/algo_py.cpp b/pybind/algo_py.cpp index 53ceb4eaf..9cefd2127 100644 --- a/pybind/algo_py.cpp +++ b/pybind/algo_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/pybind/bond_py.cpp b/pybind/bond_py.cpp index 1cffbd107..5c27b2322 100644 --- a/pybind/bond_py.cpp +++ b/pybind/bond_py.cpp @@ -1,18 +1,21 @@ -#include +#include "cytnx.hpp" + +#include #include #include +#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include -#include "cytnx.hpp" -// #include "../include/cytnx_error.hpp" #include "complex.h" +#include "H5Cpp.h" namespace py = pybind11; using namespace pybind11::literals; @@ -179,9 +182,37 @@ void bond_binding(py::module &m) { .def("calc_reverse_qnums", &Bond::calc_reverse_qnums) .def( - "Save", [](Bond &self, const std::string &fname) { self.Save(fname); }, py::arg("fname")) + "Save", + [](Bond &self, const std::filesystem::path &fname, const std::string &path, const char mode) { + self.Save(fname, path, mode); + }, + py::arg("fname"), py::arg("path") = "/Bond/", py::arg("mode") = 'w') .def_static( - "Load", [](const std::string &fname) { return Bond::Load(fname); }, py::arg("fname")) + "Load", + [](const std::filesystem::path &fname, const std::string &path) { + return Bond::Load(fname, path); + }, + py::arg("fname"), py::arg("path") = "/Bond/") + .def( + "Load_", + [](cytnx::Bond &self, const std::filesystem::path &fname, const std::string &path) { + return self.Load_(fname, path); + }, + py::arg("fname"), py::arg("path") = "/Bond/") + + .def(py::pickle( + [](const Bond &self) { // __getstate__ + std::ostringstream oss(std::ios::binary); + self.to_binary(oss); + return py::bytes(oss.str()); + }, + [](py::bytes state) { // __setstate__ + std::string data = state; + std::istringstream iss(data, std::ios::binary); + Bond out; + out.from_binary(iss); + return out; + })) ; // end of object line } diff --git a/pybind/cytnx.cpp b/pybind/cytnx.cpp index 18b4e945c..5b14e7a33 100644 --- a/pybind/cytnx.cpp +++ b/pybind/cytnx.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,7 @@ void symmetry_binding(py::module &m); #else void generator_binding(py::module &m); +void io_binding(py::module &m); void storage_binding(py::module &m); void tensor_binding(py::module &m); @@ -117,6 +119,7 @@ PYBIND11_MODULE(cytnx, m) { // py::arg("out_labels") = std::vector()); generator_binding(m); + io_binding(m); scalar_binding(m); storage_binding(m); tensor_binding(m); diff --git a/pybind/generator_py.cpp b/pybind/generator_py.cpp index 06b26d9e9..a716eda6c 100644 --- a/pybind/generator_py.cpp +++ b/pybind/generator_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/pybind/io_py.cpp b/pybind/io_py.cpp new file mode 100644 index 000000000..ed4d55180 --- /dev/null +++ b/pybind/io_py.cpp @@ -0,0 +1,207 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "cytnx.hpp" +#include "complex.h" +#include "H5Cpp.h" + +namespace py = pybind11; +using namespace pybind11::literals; +using namespace cytnx; + +#ifdef BACKEND_TORCH +#else + +// Helper for the three polymorphic wrapper structures +template +void InPlaceLoad(T &object, H5::Group &container, const std::string &name, + const std::string &path) { + cytnx::io::savable_class temp_savable = T(); + cytnx::io::Load(temp_savable, container, name, path); + T &loaded_tensor = std::get(temp_savable); + object = std::move(loaded_tensor); +} + +template +void InPlaceLoadDevice(T &object, H5::Group &container, const std::string &name, + const std::string &path, bool restore_device) { + cytnx::io::loadable_to_device temp_savable = T(); + cytnx::io::Load(temp_savable, container, name, path, restore_device); + T &loaded_tensor = std::get(temp_savable); + object = std::move(loaded_tensor); +} + +void io_binding(py::module &m) { + // [Submodule io] + pybind11::module m_io = m.def_submodule("io", "Input/Output related."); + + py::enum_(m_io, "IoMode") + .value("ACC_TRUNC", cytnx::io::IoMode::ACC_TRUNC) + .value("ACC_NOREPLACE", cytnx::io::IoMode::ACC_NOREPLACE) + .value("ACC_IN", cytnx::io::IoMode::ACC_IN) + .value("ACC_INOUT", cytnx::io::IoMode::ACC_INOUT) + .export_values(); + + // H5Object is an abstract class for all H5 objects that can contain attributes + py::class_>(m_io, "H5Object") + // Attribute management + .def( + "attrExists", + [](const H5::H5Object &self, const std::string &name) { return self.attrExists(name); }, + py::arg("name")) + .def( + "removeAttr", + [](const H5::H5Object &self, const std::string &name) { self.removeAttr(name); }, + py::arg("name")) + .def( + "renameAttr", + [](const H5::H5Object &self, const std::string &old_n, const std::string &new_n) { + self.renameAttr(old_n, new_n); + }, + py::arg("old_name"), py::arg("new_name")); // end of object line + + py::class_(m_io, "Group") + // construction + .def(py::init<>()) + .def(py::init(), py::arg("original")) + // Group specific methods + .def("assign", &H5::Group::operator=) + .def("close", &H5::Group::close) + .def("fromClass", &H5::Group::fromClass) + .def("getId", &H5::Group::getId) + + // inherited + // Group management + .def( + "createGroup", + [](const H5::Group &self, const std::string &name, size_t size_hint) { + return self.createGroup(name, size_hint); + }, + py::arg("name"), py::arg("size_hint") = 0) + .def( + "openGroup", + [](const H5::Group &self, const std::string &name) { return self.openGroup(name); }, + py::arg("name")) + + // link/object management + .def( + "moveLink", + [](const H5::Group &self, const std::string &src, const std::string &dst) { + self.moveLink(src, dst); + }, + py::arg("src_name"), py::arg("dst_name")) + .def( + "nameExists", + [](const H5::Group &self, const std::string &name) { return self.nameExists(name); }, + py::arg("name")) + .def( + "link", + [](const H5::Group &self, const std::string &curr, const std::string &next) { + self.link(curr, next); + }, + py::arg("curr_name"), py::arg("new_name")) + .def( + "unlink", [](const H5::Group &self, const std::string &name) { self.unlink(name); }, + py::arg("name")) + + // comments + .def( + "setComment", + [](const H5::Group &self, const std::string &name, const std::string &comment) { + self.setComment(name, comment); + }, + py::arg("name"), py::arg("comment")) + .def( + "getComment", + [](const H5::Group &self, const std::string &name) { return self.getComment(name); }, + py::arg("name")) + .def( + "removeComment", + [](const H5::Group &self, const std::string &name) { self.removeComment(name); }, + py::arg("name")) + + // helper + .def("getFileName", + [](const H5::Group &self) { return self.getFileName(); }); // end of object line + + py::class_(m_io, "H5File") + // construction + .def(py::init<>()) + .def(py::init(), py::arg("original")) + .def(py::init(), py::arg("name"), py::arg("flags")) + // methods + .def("close", &H5::H5File::close) + .def("fromClass", &H5::H5File::fromClass) + .def("getId", &H5::H5File::getId) + .def("getFileSize", &H5::H5File::getFileSize) + .def("getFreeSpace", &H5::H5File::getFreeSpace) + .def( + "isAccessible", + [](H5::H5File &self, const std::string &name) { return self.isAccessible(name); }, + py::arg("name")) + .def( + "isHdf5", [](H5::H5File &self, const std::string &name) { return self.isHdf5(name); }, + py::arg("name")) + .def("getObjCount", [](H5::H5File &self) { return self.getObjCount(); }); // end of object line + + // implementations from cytnx::io + m_io.def( + "create_group", + [](H5::Group &container, const std::string &path, bool recursive) { + return cytnx::io::create_group(container, path, recursive); + }, + py::arg("container"), py::arg("path"), py::arg("recursive") = true); + + m_io.def( + "open", + [](const std::filesystem::path &fname, cytnx::io::IoMode mode) { + return cytnx::io::open(fname, mode); + }, + py::arg("fname"), py::arg("mode") = cytnx::io::ACC_TRUNC); + // m_io.def("Close", [](H5::H5File file) { cytnx::io::Close(file); }, py::arg("file")); + + m_io.def("Save", &cytnx::io::Save, py::arg("object"), py::arg("container"), py::arg("name"), + py::arg("path") = "", py::arg("overwrite") = false); + + // ---- Load function ---- + // Wrapper types that break references without an in-place move + m_io.def("Load", &InPlaceLoad, py::arg("object"), py::arg("container"), + py::arg("name"), py::arg("path") = ""); + m_io.def("Load", &InPlaceLoad, py::arg("object"), py::arg("container"), + py::arg("name"), py::arg("path") = ""); + m_io.def("Load", &InPlaceLoad, py::arg("object"), py::arg("container"), + py::arg("name"), py::arg("path") = ""); + // All other types (must come after the previous more specific ones) + m_io.def( + "Load", + [](cytnx::io::savable_class &object, H5::Group &container, const std::string &name, + const std::string &path) { cytnx::io::Load(object, container, name, path); }, + py::arg("object"), py::arg("container"), py::arg("name"), py::arg("path") = ""); + + // ---- Load function for objects that can be restored on a given device ---- + // Wrapper types that break references without an in-place move + m_io.def("Load", &InPlaceLoadDevice, py::arg("object"), py::arg("container"), + py::arg("name"), py::arg("path") = "", py::arg("restore_device")); + m_io.def("Load", &InPlaceLoadDevice, py::arg("object"), py::arg("container"), + py::arg("name"), py::arg("path") = "", py::arg("restore_device")); + m_io.def("Load", &InPlaceLoadDevice, py::arg("object"), py::arg("container"), + py::arg("name"), py::arg("path") = "", py::arg("restore_device")); + // All other types (must come after the previous more specific ones) + m_io.def( + "Load", + [](cytnx::io::loadable_to_device &object, H5::Group &container, const std::string &name, + const std::string &path, + bool restore_device) { cytnx::io::Load(object, container, name, path, restore_device); }, + py::arg("object"), py::arg("container"), py::arg("name"), py::arg("path"), + py::arg("restore_device")); + +} // io_binding + +#endif diff --git a/pybind/linalg_py.cpp b/pybind/linalg_py.cpp index 73fa8ac17..edaac369e 100644 --- a/pybind/linalg_py.cpp +++ b/pybind/linalg_py.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include diff --git a/pybind/linop_py.cpp b/pybind/linop_py.cpp index 822a555cf..3c320d1a5 100644 --- a/pybind/linop_py.cpp +++ b/pybind/linop_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/pybind/ncon_py.cpp b/pybind/ncon_py.cpp index 77a96d183..df48516fb 100644 --- a/pybind/ncon_py.cpp +++ b/pybind/ncon_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/pybind/network_py.cpp b/pybind/network_py.cpp index 5f442d19f..aa7796827 100644 --- a/pybind/network_py.cpp +++ b/pybind/network_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/pybind/physics_related_py.cpp b/pybind/physics_related_py.cpp index 8bf5e6717..0255c47a2 100644 --- a/pybind/physics_related_py.cpp +++ b/pybind/physics_related_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/pybind/random_py.cpp b/pybind/random_py.cpp index db484b75e..6bc27c31b 100644 --- a/pybind/random_py.cpp +++ b/pybind/random_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -104,8 +105,8 @@ void random_binding(py::module &m) { } return cytnx::random::normal(Nelem, mean, std, device, seed, dtype); }, - py::arg("Nelem"), py::arg("mean"), py::arg("std"), py::arg("device") = -1, py::arg("seed") = -1, - py::arg("dtype") = (unsigned int)(Type.Double)); + py::arg("Nelem"), py::arg("mean"), py::arg("std"), py::arg("device") = (int)cytnx::Device.cpu, + py::arg("seed") = -1, py::arg("dtype") = (unsigned int)(Type.Double)); m_random.def( "normal", [](const std::vector &Nelem, const double &mean, const double &std, @@ -116,8 +117,8 @@ void random_binding(py::module &m) { } return cytnx::random::normal(Nelem, mean, std, device, seed, dtype); }, - py::arg("Nelem"), py::arg("mean"), py::arg("std"), py::arg("device") = -1, py::arg("seed") = -1, - py::arg("dtype") = (unsigned int)(Type.Double)); + py::arg("Nelem"), py::arg("mean"), py::arg("std"), py::arg("device") = (int)cytnx::Device.cpu, + py::arg("seed") = -1, py::arg("dtype") = (unsigned int)(Type.Double)); m_random.def( "uniform", [](const cytnx_uint64 &Nelem, const double &low, const double &high, const int &device, @@ -128,8 +129,8 @@ void random_binding(py::module &m) { } return cytnx::random::uniform(Nelem, low, high, device, seed, dtype); }, - py::arg("Nelem"), py::arg("low"), py::arg("high"), py::arg("device") = -1, py::arg("seed") = -1, - py::arg("dtype") = (unsigned int)(Type.Double)); + py::arg("Nelem"), py::arg("low"), py::arg("high"), py::arg("device") = (int)cytnx::Device.cpu, + py::arg("seed") = -1, py::arg("dtype") = (unsigned int)(Type.Double)); m_random.def( "uniform", [](const std::vector &Nelem, const double &low, const double &high, @@ -140,7 +141,7 @@ void random_binding(py::module &m) { } return cytnx::random::uniform(Nelem, low, high, device, seed, dtype); }, - py::arg("Nelem"), py::arg("low"), py::arg("high"), py::arg("device") = -1, py::arg("seed") = -1, - py::arg("dtype") = (unsigned int)(Type.Double)); + py::arg("Nelem"), py::arg("low"), py::arg("high"), py::arg("device") = (int)cytnx::Device.cpu, + py::arg("seed") = -1, py::arg("dtype") = (unsigned int)(Type.Double)); } #endif diff --git a/pybind/scalar_py.cpp b/pybind/scalar_py.cpp index 6cd648864..5f0daa258 100644 --- a/pybind/scalar_py.cpp +++ b/pybind/scalar_py.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/pybind/storage_py.cpp b/pybind/storage_py.cpp index 2808b6672..dd5ec8a14 100644 --- a/pybind/storage_py.cpp +++ b/pybind/storage_py.cpp @@ -1,19 +1,22 @@ +#include "cytnx.hpp" + +#include #include #include #include #include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include -#include "cytnx.hpp" -// #include "../include/cytnx_error.hpp" #include "complex.h" +#include "H5Cpp.h" namespace py = pybind11; using namespace pybind11::literals; @@ -60,7 +63,7 @@ void storage_binding(py::module &m) { } else if (tmpIN.dtype() == Type.Bool) { chr_dtype = py::format_descriptor::format(); } else { - cytnx_error_msg(true, "[ERROR] Void Type Tensor cannot convert to numpy ndarray%s", + cytnx_error_msg(true, "[ERROR] Void Type Storage cannot convert to numpy ndarray%s", "\n"); } @@ -76,10 +79,10 @@ void storage_binding(py::module &m) { .def(py::init()) .def(py::init>()) .def(py::init(), - py::arg("size"), py::arg("dtype") = (cytnx_uint64)Type.Double, py::arg("device") = -1, - py::arg("init_zero") = true) + py::arg("size"), py::arg("dtype") = (cytnx_uint64)Type.Double, + py::arg("device") = (int)cytnx::Device.cpu, py::arg("init_zero") = true) .def("Init", &cytnx::Storage::Init, py::arg("size"), - py::arg("dtype") = (cytnx_uint64)Type.Double, py::arg("device") = -1, + py::arg("dtype") = (cytnx_uint64)Type.Double, py::arg("device") = (int)cytnx::Device.cpu, py::arg("init_zero") = true) .def("dtype", &cytnx::Storage::dtype) @@ -260,20 +263,48 @@ void storage_binding(py::module &m) { .def("c_pylist_bool", &cytnx::Storage::vector) .def( - "Save", [](cytnx::Storage &self, const std::string &fname) { self.Save(fname); }, - py::arg("fname")) - .def( - "Tofile", [](cytnx::Storage &self, const std::string &fname) { self.Tofile(fname); }, - py::arg("fname")) + "Save", + [](cytnx::Storage &self, const std::filesystem::path &fname, const std::string &path, + const char mode) { self.Save(fname, path, mode); }, + py::arg("fname"), py::arg("path") = "/Storage", py::arg("mode") = 'w') .def_static( - "Load", [](const std::string &fname) { return cytnx::Storage::Load(fname); }, + "Load", + [](const std::filesystem::path &fname, const std::string &path, const bool restore_device) { + return cytnx::Storage::Load(fname, path, restore_device); + }, + py::arg("fname"), py::arg("path") = "/Storage", py::arg("restore_device") = true) + .def( + "Load_", + [](cytnx::Storage &self, const std::filesystem::path &fname, const std::string &path, + const bool restore_device) { return self.Load_(fname, path, restore_device); }, + py::arg("fname"), py::arg("path") = "/Storage", py::arg("restore_device") = true) + + .def( + "Tofile", + [](cytnx::Storage &self, const std::filesystem::path &fname) { self.Tofile(fname); }, py::arg("fname")) .def_static( "Fromfile", - [](const std::string &fname, const unsigned int &dtype, const cytnx_int64 &count) { - return cytnx::Storage::Fromfile(fname, dtype, count); - }, - py::arg("fname"), py::arg("dtype"), py::arg("count") = (cytnx_int64)(-1)) + [](const std::filesystem::path &fname, const unsigned int &dtype, const cytnx_int64 &count, + const int device) { return cytnx::Storage::Fromfile(fname, dtype, count, device); }, + py::arg("fname"), py::arg("dtype"), py::arg("count") = (cytnx_int64)(-1), + py::arg("device") = (int)cytnx::Device.cpu) + + .def( + py::pickle( + [](const cytnx::Storage &self) { // __getstate__ + std::ostringstream oss(std::ios::binary); + self.to_binary(oss); + return py::bytes(oss.str()); + }, + [](py::bytes state) { // __setstate__ + std::string data = state; + std::istringstream iss(data, std::ios::binary); + cytnx::Storage out; + out.from_binary(iss); + return out; + })) + .def("real", &cytnx::Storage::real) .def("imag", &cytnx::Storage::imag) diff --git a/pybind/symmetry_py.cpp b/pybind/symmetry_py.cpp index 0c2d655a1..65accc2ae 100644 --- a/pybind/symmetry_py.cpp +++ b/pybind/symmetry_py.cpp @@ -1,18 +1,22 @@ +#include "cytnx.hpp" + +#include #include -#include #include #include +#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include -#include "cytnx.hpp" #include "complex.h" +#include "H5Cpp.h" namespace py = pybind11; using namespace pybind11::literals; @@ -64,6 +68,7 @@ void symmetry_binding(py::module &m) { .def_static("FermionNumber", &Symmetry::FermionNumber) .def("clone", &Symmetry::clone) .def("stype", &Symmetry::stype) + .def("getname", &Symmetry::getname) .def("stype_str", &Symmetry::stype_str) .def("n", &Symmetry::n) .def("clone", &Symmetry::clone) @@ -92,9 +97,37 @@ void symmetry_binding(py::module &m) { .def("is_fermionic", &Symmetry::is_fermionic) .def( - "Save", [](Symmetry &self, const std::string &fname) { self.Save(fname); }, py::arg("fname")) + "Save", + [](Symmetry &self, const std::filesystem::path &fname, const std::string &path, + const char mode) { self.Save(fname, path, mode); }, + py::arg("fname"), py::arg("path") = "/Symmetry", py::arg("mode") = 'w') .def_static( - "Load", [](const std::string &fname) { return Symmetry::Load(fname); }, py::arg("fname")) + "Load", + [](const std::filesystem::path &fname, const std::string &path) { + return Symmetry::Load(fname, path); + }, + py::arg("fname"), py::arg("path") = "/Symmetry") + .def( + "Load_", + [](cytnx::Symmetry &self, const std::filesystem::path &fname, const std::string &path) { + return self.Load_(fname, path); + }, + py::arg("fname"), py::arg("path") = "/Symmetry") + + .def(py::pickle( + [](const Symmetry &self) { // __getstate__ + std::ostringstream oss(std::ios::binary); + self.to_binary(oss); + return py::bytes(oss.str()); + }, + [](py::bytes state) { // __setstate__ + std::string data = state; + std::istringstream iss(data, std::ios::binary); + Symmetry out; + out.from_binary(iss); + return out; + })) + .def( "__repr__", [](Symmetry &self) { diff --git a/pybind/tensor_py.cpp b/pybind/tensor_py.cpp index 6977a3c45..72dd51cab 100644 --- a/pybind/tensor_py.cpp +++ b/pybind/tensor_py.cpp @@ -1,18 +1,21 @@ -#include +#include "cytnx.hpp" + +#include #include #include +#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include -#include "cytnx.hpp" -// #include "../include/cytnx_error.hpp" #include "complex.h" +#include "H5Cpp.h" namespace py = pybind11; using namespace pybind11::literals; @@ -159,6 +162,7 @@ void tensor_binding(py::module &m) { .def("device", &cytnx::Tensor::device) .def("device_str", &cytnx::Tensor::device_str) .def("shape", &cytnx::Tensor::shape) + .def("strides", &cytnx::Tensor::strides) .def("rank", &cytnx::Tensor::rank) .def("clone", &cytnx::Tensor::clone) .def("__copy__", &cytnx::Tensor::clone) @@ -287,20 +291,46 @@ void tensor_binding(py::module &m) { py::arg("val")) .def( - "Save", [](cytnx::Tensor &self, const std::string &fname) { self.Save(fname); }, - py::arg("fname")) + "Save", + [](cytnx::Tensor &self, const std::filesystem::path &fname, const std::string &path, + const char mode) { self.Save(fname, path, mode); }, + py::arg("fname"), py::arg("path") = "/Tensor", py::arg("mode") = 'w') .def_static( - "Load", [](const std::string &fname) { return cytnx::Tensor::Load(fname); }, py::arg("fname")) + "Load", + [](const std::filesystem::path &fname, const std::string &path, const bool restore_device) { + return cytnx::Tensor::Load(fname, path, restore_device); + }, + py::arg("fname"), py::arg("path") = "/Tensor", py::arg("restore_device") = true) + .def( + "Load_", + [](cytnx::Tensor &self, const std::filesystem::path &fname, const std::string &path, + const bool restore_device) { return self.Load_(fname, path, restore_device); }, + py::arg("fname"), py::arg("path") = "/Tensor", py::arg("restore_device") = true) + + .def(py::pickle( + [](const cytnx::Tensor &self) { // __getstate__ + std::ostringstream oss(std::ios::binary); + self.to_binary(oss); + return py::bytes(oss.str()); + }, + [](py::bytes state) { // __setstate__ + std::string data = state; + std::istringstream iss(data, std::ios::binary); + cytnx::Tensor out; + out.from_binary(iss); + return out; + })) .def( - "Tofile", [](cytnx::Tensor &self, const std::string &fname) { self.Tofile(fname); }, + "Tofile", [](cytnx::Tensor &self, const std::filesystem::path &fname) { self.Tofile(fname); }, py::arg("fname")) .def_static( "Fromfile", - [](const std::string &fname, const unsigned int &dtype, const cytnx::cytnx_int64 &count) { - return cytnx::Tensor::Load(fname); - }, - py::arg("fname"), py::arg("dtype"), py::arg("count") = cytnx::cytnx_int64(-1)) + [](const std::filesystem::path &fname, const unsigned int &dtype, + const cytnx::cytnx_int64 &count, + const int device) { return cytnx::Tensor::Fromfile(fname, dtype, count, device); }, + py::arg("fname"), py::arg("dtype"), py::arg("count") = cytnx::cytnx_int64(-1), + py::arg("device") = (int)cytnx::Device.cpu) .def_static( "from_storage", diff --git a/pybind/tnalgo_py.cpp b/pybind/tnalgo_py.cpp index 494597b6f..ccfc765b6 100644 --- a/pybind/tnalgo_py.cpp +++ b/pybind/tnalgo_py.cpp @@ -1,18 +1,21 @@ -#include +#include "cytnx.hpp" + +#include #include #include +#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include -#include "cytnx.hpp" -// #include "../include/cytnx_error.hpp" #include "complex.h" +#include "H5Cpp.h" namespace py = pybind11; using namespace pybind11::literals; @@ -51,12 +54,38 @@ void tnalgo_binding(py::module &m) { .def("c_Into_Lortho", &tn_algo::MPS::Into_Lortho) .def("c_S_mvleft", &tn_algo::MPS::S_mvleft) .def("c_S_mvright", &tn_algo::MPS::S_mvright) + .def( - "Save", [](cytnx::Storage &self, const std::string &fname) { self.Save(fname); }, - py::arg("fname")) + "Save", + [](tn_algo::MPS &self, const std::filesystem::path &fname, const std::string &path, + const char mode) { self.Save(fname, path, mode); }, + py::arg("fname"), py::arg("path") = "/MPS/", py::arg("mode") = 'w') .def_static( - "Load", [](const std::string &fname) { return cytnx::tn_algo::MPS::Load(fname); }, - py::arg("fname")) + "Load", + [](const std::filesystem::path &fname, const std::string &path, const bool restore_device) { + return tn_algo::MPS::Load(fname, path, restore_device); + }, + py::arg("fname"), py::arg("path") = "/MPS/", py::arg("restore_device") = true) + .def( + "Load_", + [](tn_algo::MPS &self, const std::filesystem::path &fname, const std::string &path, + const bool restore_device) { return self.Load_(fname, path, restore_device); }, + py::arg("fname"), py::arg("path") = "/MPS/", py::arg("restore_device") = true) + + .def(py::pickle( + [](const tn_algo::MPS &self) { // __getstate__ + std::ostringstream oss(std::ios::binary); + self.to_binary(oss); + return py::bytes(oss.str()); + }, + [](py::bytes state) { // __setstate__ + std::string data = state; + std::istringstream iss(data, std::ios::binary); + tn_algo::MPS out; + out.from_binary(iss); + return out; + })) + .def( "__repr__", [](cytnx::tn_algo::MPS &self) -> std::string { diff --git a/pybind/unitensor_py.cpp b/pybind/unitensor_py.cpp index 50bd02280..f35dc54ea 100644 --- a/pybind/unitensor_py.cpp +++ b/pybind/unitensor_py.cpp @@ -1,19 +1,23 @@ +#include "cytnx.hpp" + +#include #include -#include #include #include #include +#include -#include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include +#include #include -#include "cytnx.hpp" +#include "H5Cpp.h" namespace py = pybind11; using namespace pybind11::literals; @@ -515,10 +519,40 @@ void unitensor_binding(py::module &m) { .def("clone", &UniTensor::clone) .def("__copy__", &UniTensor::clone) .def("__deepcopy__", &UniTensor::clone) + .def( - "Save", [](UniTensor &self, const std::string &fname) { self.Save(fname); }, py::arg("fname")) + "Save", + [](UniTensor &self, const std::filesystem::path &fname, const std::string &path, + const char mode) { self.Save(fname, path, mode); }, + py::arg("fname"), py::arg("path") = "/UniTensor/", py::arg("mode") = 'w') .def_static( - "Load", [](const std::string &fname) { return UniTensor::Load(fname); }, py::arg("fname")) + "Load", + [](const std::filesystem::path &fname, const std::string &path, const bool restore_device) { + return UniTensor::Load(fname, path, restore_device); + }, + py::arg("fname"), py::arg("path") = "/UniTensor/", py::arg("restore_device") = true) + .def( + "Load_", + [](UniTensor &self, const std::filesystem::path &fname, const std::string &path, const bool restore_device) { + return self.Load_(fname, path, restore_device); + }, + py::arg("fname"), py::arg("path") = "/UniTensor/", py::arg("restore_device") = true) + + .def(py::pickle( + [](const UniTensor &self) { // __getstate__ + std::ostringstream oss(std::ios::binary); + self.to_binary(oss); + return py::bytes(oss.str()); + }, + [](py::bytes state) { // __setstate__ + std::string data = state; + std::istringstream iss(data, std::ios::binary); + UniTensor out; + out.from_binary(iss); + return out; + } + )) + //.def("permute",&UniTensor::permute,py::arg("mapper"),py::arg("rowrank")=(cytnx_int64)-1,py::arg("by_label")=false) //.def("permute_",&UniTensor::permute_,py::arg("mapper"),py::arg("rowrank")=(cytnx_int64)-1,py::arg("by_label")=false) .def( diff --git a/src/BlockFermionicUniTensor.cpp b/src/BlockFermionicUniTensor.cpp index 7bb7fd27d..cdfd9d9c4 100644 --- a/src/BlockFermionicUniTensor.cpp +++ b/src/BlockFermionicUniTensor.cpp @@ -10,6 +10,8 @@ #include #include #include +#include "io.hpp" + using namespace std; #ifdef BACKEND_TORCH @@ -145,7 +147,7 @@ namespace cytnx { } } else { - // checking how many blocks are there, and the size: + // find the number of blocks and their sizes: std::vector Loc(this->_bonds.size(), 0); std::vector tot_qns( this->_bonds[0].Nsym()); // use first bond to determine symmetry size @@ -155,21 +157,22 @@ namespace cytnx { // get elem this->_fx_get_total_fluxs(Loc, this->_bonds[0].syms(), tot_qns); - // if exists: + // if total flux is zero -> block exists: if (std::all_of(tot_qns.begin(), tot_qns.end(), [](const int &i) { return i == 0; })) { - // init block! + // get size & init block! for (cytnx_int32 i = 0; i < Loc.size(); i++) { size[i] = this->_bonds[i]._impl->_degs[Loc[i]]; } - if (!no_alloc) { - this->_blocks.push_back(zeros(size, dtype, device)); - } else { + if (no_alloc) { this->_blocks.push_back(Tensor(size, dtype, device, false)); + } else { + this->_blocks.push_back(zeros(size, dtype, device)); } // push its loc this->_inner_to_outer_idx.push_back(Loc); } + // increment Loc by one or emtpy Loc if last element is reached while (Loc.size() != 0) { if (Loc.back() == this->_bonds[Loc.size() - 1]._impl->_qnums.size() - 1) { Loc.pop_back(); @@ -1997,11 +2000,14 @@ namespace cytnx { } // helper function: + // @param[in] locator: indices to check + // @param[out] bidx: block index corresponding to the locator + // @param[out] loc_in_T: indices in block[bidx] that locator corresponds to void BlockFermionicUniTensor::_fx_locate_elem(cytnx_int64 &bidx, std::vector &loc_in_T, const std::vector &locator) const { //[21 Aug 2024] This is a copy from BlockUniTensor; error message differs - // 1. check if out of range: + // 1. check if out of range cytnx_error_msg(locator.size() != this->_bonds.size(), "[ERROR] len(locator) does not match the rank of tensor.%s", "\n"); @@ -2011,7 +2017,8 @@ namespace cytnx { "[ERROR][BlockFermionicUniTensor][elem_exists] locator @index: %d out of range.\n", i); } - // 2. calculate the location is in which qindices: + // 2. calculate qindices corresponding to the locator + // subtracts the degeneracies from the locator until loc_in_T < degeneracy on that index if (this->is_diag()) { if (locator[0] != locator[1]) bidx = -1; @@ -2257,7 +2264,81 @@ namespace cytnx { return this->_blocks[bidx].at(loc_in_T); } - void BlockFermionicUniTensor::_save_dispatch(std::fstream &f) const { + void BlockFermionicUniTensor::to_hdf5_dispatch(H5::Group &container, const bool overwrite) const { + // delete all entries that could be written by one of the UniTensor implementations; + io::remove_attribute(container, "directed", overwrite); + io::unlink(container, "Tensor", overwrite); + + // blocks; write to group + if (this->_blocks.empty()) { + io::unlink(container, "blocks", overwrite); + } else { + H5::Group dir = io::create_group(container, "blocks", false); + hsize_t blknum = 0; + for (; blknum < this->_blocks.size(); blknum++) { + if (this->_signflip[blknum]) { + Tensor block = -this->_blocks[blknum]; + block.to_hdf5(dir, "Tensor" + std::to_string(blknum), overwrite); + } else { + this->_blocks[blknum].to_hdf5(dir, "Tensor" + std::to_string(blknum), overwrite); + } + } + // delete further blocks if they exist + while (io::remove_attribute(dir, "Tensor" + std::to_string(blknum), overwrite)) blknum++; + } + // inner_to_outer_idx; write matrix (blocknum x rank) + if (this->_inner_to_outer_idx.empty()) { + io::unlink(container, "block_to_sectors", overwrite); + } else { + H5::DataSet dataset = + io::save_dataset(this->_inner_to_outer_idx, container, "block_to_sectors", overwrite); + // label axes + io::save_attribute(std::vector{"block", "bond"}, dataset, "axis_labels", + overwrite); + } + } + + void BlockFermionicUniTensor::from_hdf5_dispatch(H5::Group &container, bool restore_device) { + this->_is_tag = true; + // blocks; read from group + this->_blocks.clear(); + if (container.nameExists("blocks")) { + H5::Group dir = container.openGroup("blocks"); + hsize_t idx = 0; + while (true) { + std::string name = "Tensor" + std::to_string(idx); + if (!dir.nameExists(name)) { + break; + } + Tensor block; + block.from_hdf5(dir, name, restore_device); + this->_blocks.push_back(block); + idx++; + } + } + // inner_to_outer_idx; read matrix (blocknum x rank) + if (container.nameExists("block_to_sectors")) { + io::load_dataset(this->_inner_to_outer_idx, container, "block_to_sectors"); + cytnx_error_msg(this->_inner_to_outer_idx.size() != this->_blocks.size(), + "[ERROR] %zu blocks found, but first dimension of 'block_to_sectors' is %zu. " + "The HDF5 data seems corrupt!\n", + this->_blocks.size(), this->_inner_to_outer_idx.size()); + cytnx_error_msg(!(this->_inner_to_outer_idx.empty()) && + this->_inner_to_outer_idx[0].size() != this->_bonds.size(), + "[ERROR] %zu bonds found, but second dimension of 'block_to_sectors' is %zu. " + "The HDF5 data seems corrupt!\n", + this->_bonds.size(), this->_inner_to_outer_idx[0].size()); + } else { + cytnx_error_msg(!(this->_blocks.empty()), + "[ERROR] 'block_to_sectors' not found, but %zu blocks exist. The HDF5 data " + "seems corrupt!\n", + this->_blocks.size()); + this->_inner_to_outer_idx.clear(); + } + this->_signflip = std::vector(this->_blocks.size(), false); + } + + void BlockFermionicUniTensor::to_binary_dispatch(std::ostream &f) const { //[21 Aug 2024] This is a copy from BlockUniTensor; saves signs as well cytnx_uint64 Nblocks = this->_blocks.size(); f.write((char *)&Nblocks, sizeof(cytnx_uint64)); @@ -2268,7 +2349,7 @@ namespace cytnx { } for (unsigned int b = 0; b < Nblocks; b++) { - this->_blocks[b]._Save(f); + this->_blocks[b].to_binary(f); } // Saving signs; each sign is saved as a char @@ -2279,7 +2360,7 @@ namespace cytnx { } } - void BlockFermionicUniTensor::_load_dispatch(std::fstream &f) { + void BlockFermionicUniTensor::from_binary_dispatch(std::istream &f, bool restore_device) { //[21 Aug 2024] This is a copy from BlockUniTensor; reads signs as well cytnx_uint64 Nblocks; f.read((char *)&Nblocks, sizeof(cytnx_uint64)); @@ -2293,7 +2374,7 @@ namespace cytnx { this->_blocks.resize(Nblocks); for (unsigned int i = 0; i < this->_blocks.size(); i++) { - this->_blocks[i]._Load(f); + this->_blocks[i].from_binary(f, restore_device); } // Loading signs; each sign is assumed to be saved as a char, with 0 being false diff --git a/src/BlockUniTensor.cpp b/src/BlockUniTensor.cpp index d8b48ebe0..7ba068028 100644 --- a/src/BlockUniTensor.cpp +++ b/src/BlockUniTensor.cpp @@ -10,6 +10,8 @@ #include #include #include +#include "io.hpp" + using namespace std; #ifdef BACKEND_TORCH @@ -134,7 +136,7 @@ namespace cytnx { } } else { - // checking how many blocks are there, and the size: + // find the number of blocks and their sizes: std::vector Loc(this->_bonds.size(), 0); std::vector tot_qns( this->_bonds[0].Nsym()); // use first bond to determine symmetry size @@ -144,21 +146,22 @@ namespace cytnx { // get elem this->_fx_get_total_fluxs(Loc, this->_bonds[0].syms(), tot_qns); - // if exists: + // if total flux is zero -> block exists: if (std::all_of(tot_qns.begin(), tot_qns.end(), [](const int &i) { return i == 0; })) { - // init block! + // get size & init block! for (cytnx_int32 i = 0; i < Loc.size(); i++) { size[i] = this->_bonds[i]._impl->_degs[Loc[i]]; } - if (!no_alloc) { - this->_blocks.push_back(zeros(size, dtype, device)); - } else { + if (no_alloc) { this->_blocks.push_back(Tensor(size, dtype, device, false)); + } else { + this->_blocks.push_back(zeros(size, dtype, device)); } // push its loc this->_inner_to_outer_idx.push_back(Loc); } + // increment Loc by one or emtpy Loc if last element is reached while (Loc.size() != 0) { if (Loc.back() == this->_bonds[Loc.size() - 1]._impl->_qnums.size() - 1) { Loc.pop_back(); @@ -1361,9 +1364,12 @@ namespace cytnx { } // helper function: + // @param[in] locator: indices to check + // @param[out] bidx: block index corresponding to the locator + // @param[out] loc_in_T: indices in block[bidx] that locator corresponds to void BlockUniTensor::_fx_locate_elem(cytnx_int64 &bidx, std::vector &loc_in_T, const std::vector &locator) const { - // 1. check if out of range: + // 1. check if out of range cytnx_error_msg(locator.size() != this->_bonds.size(), "[ERROR] len(locator) does not match the rank of tensor.%s", "\n"); @@ -1372,7 +1378,8 @@ namespace cytnx { "[ERROR][BlockUniTensor][elem_exists] locator @index: %d out of range.\n", i); } - // 2. calculate the location is in which qindices: + // 2. calculate qindices corresponding to the locator + // subtracts the degeneracies from the locator until loc_in_T < degeneracy on that index if (this->is_diag()) { if (locator[0] != locator[1]) bidx = -1; @@ -1592,9 +1599,75 @@ namespace cytnx { return this->_blocks[bidx].at(loc_in_T); } - void BlockUniTensor::_save_dispatch(std::fstream &f) const { - // cytnx_error_msg(true,"[ERROR] Save for SparseUniTensor is under developing!!%s","\n"); + void BlockUniTensor::to_hdf5_dispatch(H5::Group &container, const bool overwrite) const { + // delete all entries that could be written by one of the UniTensor implementations; + io::remove_attribute(container, "directed", overwrite); + io::unlink(container, "Tensor", overwrite); + // blocks; write to group + if (this->_blocks.empty()) { + io::unlink(container, "blocks", overwrite); + } else { + H5::Group dir = io::create_group(container, "blocks", false); + hsize_t blknum = 0; + for (; blknum < this->_blocks.size(); blknum++) { + this->_blocks[blknum].to_hdf5(dir, "Tensor" + std::to_string(blknum), overwrite); + } + // delete further blocks if they exist + while (io::remove_attribute(dir, "Tensor" + std::to_string(blknum), overwrite)) blknum++; + } + // inner_to_outer_idx; write matrix (blocknum x rank) + if (this->_inner_to_outer_idx.empty()) { + io::unlink(container, "block_to_sectors", overwrite); + } else { + H5::DataSet dataset = + io::save_dataset(this->_inner_to_outer_idx, container, "block_to_sectors", overwrite); + // label axes + io::save_attribute(std::vector{"block", "bond"}, dataset, "axis_labels", + overwrite); + } + } + + void BlockUniTensor::from_hdf5_dispatch(H5::Group &container, bool restore_device) { + this->_is_tag = true; + // blocks; read from group + this->_blocks.clear(); + if (container.nameExists("blocks")) { + H5::Group dir = container.openGroup("blocks"); + hsize_t idx = 0; + while (true) { + std::string name = "Tensor" + std::to_string(idx); + if (!dir.nameExists(name)) { + break; + } + Tensor block; + block.from_hdf5(dir, name, restore_device); + this->_blocks.push_back(block); + idx++; + } + } + // inner_to_outer_idx; read matrix (blocknum x rank) + if (container.nameExists("block_to_sectors")) { + io::load_dataset(this->_inner_to_outer_idx, container, "block_to_sectors"); + cytnx_error_msg(this->_inner_to_outer_idx.size() != this->_blocks.size(), + "[ERROR] %zu blocks found, but first dimension of 'block_to_sectors' is %zu. " + "The HDF5 data seems corrupt!\n", + this->_blocks.size(), this->_inner_to_outer_idx.size()); + cytnx_error_msg(!(this->_inner_to_outer_idx.empty()) && + this->_inner_to_outer_idx[0].size() != this->_bonds.size(), + "[ERROR] %zu bonds found, but second dimension of 'block_to_sectors' is %zu. " + "The HDF5 data seems corrupt!\n", + this->_bonds.size(), this->_inner_to_outer_idx[0].size()); + } else { + cytnx_error_msg(!(this->_blocks.empty()), + "[ERROR] 'block_to_sectors' not found, but %zu blocks exist. The HDF5 data " + "seems corrupt!\n", + this->_blocks.size()); + this->_inner_to_outer_idx.clear(); + } + } + + void BlockUniTensor::to_binary_dispatch(std::ostream &f) const { cytnx_uint64 Nblocks = this->_blocks.size(); f.write((char *)&Nblocks, sizeof(cytnx_uint64)); @@ -1603,13 +1676,11 @@ namespace cytnx { f.write((char *)&this->_inner_to_outer_idx[b][0], sizeof(cytnx_uint64) * this->_bonds.size()); } for (unsigned int i = 0; i < this->_blocks.size(); i++) { - this->_blocks[i]._Save(f); + this->_blocks[i].to_binary(f); } } - void BlockUniTensor::_load_dispatch(std::fstream &f) { - // cytnx_error_msg(true,"[ERROR] Save for SparseUniTensor is under developing!!%s","\n"); - + void BlockUniTensor::from_binary_dispatch(std::istream &f, bool restore_device) { cytnx_uint64 Nblocks; f.read((char *)&Nblocks, sizeof(cytnx_uint64)); @@ -1622,7 +1693,7 @@ namespace cytnx { this->_blocks.resize(Nblocks); for (unsigned int i = 0; i < this->_blocks.size(); i++) { - this->_blocks[i]._Load(f); + this->_blocks[i].from_binary(f, restore_device); } } diff --git a/src/Bond.cpp b/src/Bond.cpp index c52b4fb83..2502fdb80 100644 --- a/src/Bond.cpp +++ b/src/Bond.cpp @@ -4,6 +4,8 @@ #include #include +#include "io.hpp" +#include "H5Cpp.h" #include "utils/utils.hpp" using namespace std; @@ -470,38 +472,272 @@ namespace cytnx { bool Bond::operator!=(const Bond &rhs) const { return !(*this == rhs); } - void Bond::Save(const std::string &fname) const { - fstream f; - if (std::filesystem::path(fname).has_extension()) { + void Bond::Save(const std::filesystem::path &fname, const std::string &path, + const char mode) const { + fstream f; // only for binary saving, not used for HDF5 + if (fname.has_extension()) { // filename extension is given - f.open(fname, ios::out | ios::trunc | ios::binary); - } else { - // add filename extension - f.open((fname + ".cybd"), ios::out | ios::trunc | ios::binary); + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { + // save as HDF5 + H5::H5File h5file; + // Enable reuse of space after data is deleted; + // Set the strategy: FSM_AGGR is standard for free-space management + // Parameters: strategy, persist (true), threshold (default 1: track all free-space + // sections) + H5::FileCreatPropList fcpl; + fcpl.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_FSM_AGGR, true, 1); + // Persistent free space requires HDF5 1.10.x format or later + H5::FileAccPropList fapl; + fapl.setLibverBounds(H5F_LIBVER_V110, H5F_LIBVER_LATEST); + // open file + bool overwrite = false; + if (mode == 'w') { // Write new file + h5file = H5::H5File(fname, H5F_ACC_TRUNC, fcpl, fapl); + } else if (mode == 'x') { // eXclusive create + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'a') { // Append data + if (std::filesystem::exists(fname)) + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + else + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'u') { // Update data + if (std::filesystem::exists(fname)) { + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + overwrite = true; + } else { + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } + } else { + cytnx_error_msg(true, "[ERROR] Unknown mode '%c' for writing to HDF5 file.", mode); + } + // create group + std::filesystem::path grouppath(path); + std::filesystem::path subpath; + std::string groupfolder = "/"; + for (const auto &part : grouppath) { + if (part.empty()) continue; + subpath /= part; + groupfolder = subpath.generic_string(); + if (!h5file.nameExists(groupfolder)) h5file.createGroup(groupfolder); + } + H5::Group group = h5file.openGroup(groupfolder); + // write data + this->to_hdf5(group, "", overwrite); + h5file.close(); + return; + } else { // create binary file + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fname), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fname.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", + mode); + } + f.open(fname, std::ios::out | std::ios::trunc | std::ios::binary); + } + } else { // create binary file with standard extension + std::filesystem::path fnameext = fname; + fnameext += ".cybd"; + cytnx_warning_msg(true, + "Missing file extension in fname '%s'. I am adding the extension '.cybd'. " + "This is deprecated, please provide the file extension in the future.\n", + fname.string().c_str()); + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fnameext), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fnameext.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", mode); + } + f.open(fnameext, std::ios::out | std::ios::trunc | std::ios::binary); } + // write binary if (!f.is_open()) { cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); } - this->_Save(f); + this->to_binary(f); f.close(); } - void Bond::Save(const char *fname) const { this->Save(string(fname)); } + void Bond::Save(const char *fname, const std::string &path, const char mode) const { + this->Save(std::filesystem::path(fname), path, mode); + } - Bond Bond::Load(const std::string &fname) { + Bond Bond::Load(const std::filesystem::path &fname, const std::string &path) { Bond out; - fstream f; - f.open(fname, ios::in | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.c_str()); - } - out._Load(f); - f.close(); + out.Load_(fname, path); return out; } - Bond Bond::Load(const char *fname) { return Bond::Load(string(fname)); } + Bond Bond::Load(const char *fname, const std::string &path) { + return Bond::Load(std::filesystem::path(fname), path); + } + + void Bond::Load_(const std::filesystem::path &fname, const std::string &path) { + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { // load HDF5 + H5::H5File h5file(fname, H5F_ACC_RDONLY); + // open group + H5::Group group; + try { + group = h5file.openGroup(path.empty() ? "/" : path); + } catch (const H5::Exception &e) { + std::cerr << e.getDetailMsg() << std::endl; + cytnx_error_msg(true, "[ERROR] HDF5 path '%s' not found or is not a group in file '%s'.", + path.c_str(), fname.string().c_str()); + } + // read data + this->from_hdf5(group, ""); + h5file.close(); + } else { // load binary + fstream f; + f.open(fname, std::ios::in | std::ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); + } + this->from_binary(f); + f.close(); + } + } + void Bond::Load_(const char *fname, const std::string &path) { + this->Load_(std::filesystem::path(fname), path); + } + + void Bond::to_hdf5(H5::Group &container, const std::string &name, const bool overwrite) const { + H5::Group rootgroup; + if (name.empty()) + rootgroup = container; + else { + rootgroup = io::create_group(container, name, false); + } + + // write attributes + io::save_attribute(this->_impl->_dim, rootgroup, "dimension", overwrite); + io::save_attribute(bondtype_to_string.at(this->_impl->_type), rootgroup, "type", overwrite); + + hsize_t sectordim = this->_impl->_qnums.size(); + if (sectordim < 1) { // no symmetries + io::unlink(rootgroup, "degeneracies", overwrite); + io::unlink(rootgroup, "quantum_numbers", overwrite); + io::unlink(rootgroup, "symmetries", overwrite); + return; + } + // degs; write vector + H5::DataSet dataset = + io::save_dataset(this->_impl->_degs, rootgroup, "degeneracies", overwrite); + // label axis + io::save_attribute("sector", dataset, "axis_label", overwrite); + + // qnums; write matrix (dim x qnumdim) + dataset = io::save_dataset(this->_impl->_qnums, rootgroup, "quantum_numbers", overwrite); + // label axes + io::save_attribute(std::vector{"sector", "symmetry"}, dataset, "axis_labels", + overwrite); + + // symmetries + H5::Group symloc = io::create_group(rootgroup, "symmetries", false); + hsize_t symnum = 0; + for (; symnum < this->_impl->_syms.size(); symnum++) { + this->_impl->_syms[symnum].to_hdf5(symloc, "Symmetry" + std::to_string(symnum), overwrite); + } + // delete further symmetries if they exist + while (io::remove_attribute(symloc, "Symmetry" + std::to_string(symnum), overwrite)) symnum++; + } + + void Bond::from_hdf5(H5::Group &container, const std::string &name) { + H5::Group rootgroup = (name.empty() ? container : container.openGroup(name)); + + // type from string + std::string typestr; + io::load_attribute(typestr, rootgroup, "type"); + this->_impl->_type = string_to_bondtype.at(typestr); + + // degs; read vector + bool symmetric = false; + if (rootgroup.nameExists("degeneracies")) { + io::load_dataset(this->_impl->_degs, rootgroup, "degeneracies"); + symmetric = true; + this->_impl->_dim = + std::accumulate(this->_impl->_degs.begin(), this->_impl->_degs.end(), cytnx_uint64(0)); + } else { + this->_impl->_degs = {}; + } - void Bond::_Save(fstream &f) const { - cytnx_error_msg(!f.is_open(), "[ERROR][Bond] invalid fstream%s", "\n"); + // qnums; read matrix (dim x qnumdim) + hsize_t qnumdim = 0; + if (rootgroup.nameExists("quantum_numbers")) { + cytnx_error_msg(!symmetric, + "[ERROR] 'degeneracies' were not found, but " + "'quantum_numbers' exist. The HDF5 data seems corrupt!%s", + "\n"); + io::load_dataset(this->_impl->_qnums, rootgroup, "quantum_numbers"); + cytnx_error_msg(this->_impl->_qnums.size() != this->_impl->_degs.size(), + "[ERROR] Length of 'degeneracies' = %zu, but first dimension of " + "'quantum_numbers' is %zu. The HDF5 data seems corrupt!\n", + this->_impl->_degs.size(), this->_impl->_qnums.size()); + if (!(this->_impl->_qnums.empty())) qnumdim = this->_impl->_qnums[0].size(); + } else { + this->_impl->_qnums.clear(); + } + + // symmetries + if (rootgroup.nameExists("symmetries")) { + H5::Group symloc = rootgroup.openGroup("symmetries"); + this->_impl->_syms.clear(); + hsize_t idx = 0; + while (true) { + std::string symmname = "Symmetry" + std::to_string(idx); + if (!symloc.attrExists(symmname) && !symloc.nameExists(symmname)) { + break; + } + Symmetry sym; + sym.from_hdf5(symloc, symmname); + this->_impl->_syms.push_back(sym); + idx++; + } + if (symmetric) { + cytnx_error_msg(idx != qnumdim, + "[ERROR] %zu symmetries were found, but second dimension of " + "'quantum_numbers' is %zu. The HDF5 data seems corrupt!\n", + idx, qnumdim); + } else { + cytnx_error_msg(idx > 0, + "[ERROR] 'degeneracies' and 'quantum_numbers' were not found, but " + "'symmetries' exist. The HDF5 data seems corrupt!%s", + "\n"); + this->_impl->_syms.clear(); + } + } else { + cytnx_error_msg(symmetric, + "[ERROR] 'degeneracies' and 'quantum_numbers' exist, but " + "'symmetries' are missing. The HDF5 data seems corrupt!%s", + "\n"); + this->_impl->_syms.clear(); + } + + // dim; from attribute + if (rootgroup.attrExists("dimension")) { + cytnx_uint64 dimension; + io::load_attribute(dimension, rootgroup, "dimension"); + if (symmetric) { + cytnx_error_msg(dimension != this->_impl->_dim, + "[ERROR] 'dimension' read from HDF5 file is %llu, but the sum of all " + "degeneracies is %llu. The HDF5 data seems corrupt!\n", + dimension, this->_impl->_dim); + } else { + this->_impl->_dim = dimension; + } + } else { + cytnx_error_msg(!symmetric, + "[ERROR] Could not find 'dimension' or 'degeneracies' in HDF5 file. The HDF5 " + "data seems corrupt!%s", + "\n"); + } + } + + void Bond::to_binary(std::ostream &f) const { unsigned int IDDs = 666; f.write((char *)&IDDs, sizeof(unsigned int)); @@ -545,17 +781,20 @@ namespace cytnx { if (Nsym != 0) { for (int j = 0; j < Nsym; j++) { - this->_impl->_syms[j]._Save(f); + this->_impl->_syms[j].to_binary(f); } } } - void Bond::_Load(fstream &f) { - cytnx_error_msg(!f.is_open(), "[ERROR][Bond] invalid fstream%s", "\n"); + void Bond::from_binary(std::istream &f) { unsigned int tmpIDDs; f.read((char *)&tmpIDDs, sizeof(unsigned int)); cytnx_error_msg(tmpIDDs != 666, "[ERROR] the object is not a cytnx Bond!%s", "\n"); + this->_impl->_qnums.clear(); + this->_impl->_degs.clear(); + this->_impl->_syms.clear(); + int ver; f.read((char *)&ver, sizeof(int)); @@ -604,8 +843,10 @@ namespace cytnx { if (Nsym_in != 0) { this->_impl->_syms.resize(Nsym_in); for (int j = 0; j < Nsym_in; j++) { - this->_impl->_syms[j]._Load(f); + this->_impl->_syms[j].from_binary(f); } + } else { + this->_impl->_syms.clear(); } } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 99556a8a8..6e108383c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,6 +26,7 @@ target_sources(cytnx Physics.cpp Accessor.cpp LinOp.cpp + io.cpp Type.cpp Tensor.cpp diff --git a/src/DenseUniTensor.cpp b/src/DenseUniTensor.cpp index efe78caec..115c7f659 100644 --- a/src/DenseUniTensor.cpp +++ b/src/DenseUniTensor.cpp @@ -2,10 +2,12 @@ #include "utils/utils.hpp" #include "Generator.hpp" +#include "io.hpp" #include "linalg.hpp" #include #include #include + typedef cytnx::Accessor ac; using namespace std; @@ -1268,8 +1270,62 @@ namespace cytnx { void DenseUniTensor::normalize_() { this->_block /= linalg::Norm(this->_block); } - void DenseUniTensor::_save_dispatch(std::fstream &f) const { this->_block._Save(f); } - void DenseUniTensor::_load_dispatch(std::fstream &f) { this->_block._Load(f); } + void DenseUniTensor::to_hdf5_dispatch(H5::Group &container, const bool overwrite) const { + // delete all entries that could be written by one of the UniTensor implementations; + io::unlink(container, "block_to_sectors", overwrite); + io::unlink(container, "blocks", overwrite); + + // is_tag, write as attribute + io::save_attribute(this->_is_tag, container, "directed", overwrite); + + this->_block.to_hdf5(container, "Tensor", overwrite); + } + + void DenseUniTensor::from_hdf5_dispatch(H5::Group &container, bool restore_device) { + this->_block.from_hdf5(container, "Tensor", restore_device); + // check data consistency + auto shape = this->_block.shape(); + if (this->_bonds.empty()) { + size_t Nelem = std::accumulate(shape.begin(), shape.end(), 1, std::multiplies()); + cytnx_error_msg( + Nelem > 1, + "[ERROR] No bonds exit, but the Tensor has %d > 1 elements. The HDF5 data seems corrupt!\n", + Nelem); + } else if (!this->_is_diag) { + cytnx_error_msg( + this->_bonds.size() != shape.size(), + "[ERROR] %d bonds exit, but the Tensor has rank %d. The HDF5 data seems corrupt!\n", + this->_bonds.size(), shape.size()); + for (cytnx_uint64 i = 0; i < shape.size(); i++) { + cytnx_error_msg(shape[i] != this->_bonds[i].dim(), + "[ERROR] Tensor has dimension %d on index %d, but the corresponding bond " + "has dimension %d. The HDF5 data seems corrupt!\n", + shape[i], i, this->_bonds[i].dim()); + } + } + + // is_tag, read from attribute or reproduce + if (container.attrExists("directed")) { + H5::Attribute attr = container.openAttribute("directed"); + H5::DataType datatype = attr.getDataType(); + cytnx_error_msg( + datatype.getSize() != Type.get_hdf5_type(this->_is_tag).getSize(), + "[ERROR] 'directed' bit-length mismatch. File: %zu bytes, expected: %zu bytes.\n", + datatype.getSize(), Type.get_hdf5_type(this->_is_tag).getSize()); + attr.read(datatype, &this->_is_tag); + } else { + if (this->_bonds.empty()) { + this->_is_tag = false; + } else { + this->_is_tag = (this->_bonds[0].type() != bondType::BD_REG); + } + } + } + + void DenseUniTensor::to_binary_dispatch(std::ostream &f) const { this->_block.to_binary(f); } + void DenseUniTensor::from_binary_dispatch(std::istream &f, bool restore_device) { + this->_block.from_binary(f, restore_device); + } void DenseUniTensor::truncate_(const std::string &bond_label, const cytnx_uint64 &dim) { // if it is diagonal tensor, truncate will be done on both index! diff --git a/src/Gncon_base.cpp b/src/Gncon_base.cpp index 334783a0b..6766231a4 100644 --- a/src/Gncon_base.cpp +++ b/src/Gncon_base.cpp @@ -1,6 +1,7 @@ +#include #include -#include "Gncon.hpp" +#include "Gncon.hpp" #include "linalg.hpp" using namespace std; @@ -14,13 +15,13 @@ namespace cytnx { const std::string &contract_order) { cytnx_error_msg(true, "[ERROR][Gncon][Contract_plan] call from uninitialize Gncon.%s", "\n"); } - void Gncon_base::Fromfile(const std::string &fname) { + void Gncon_base::Fromfile(const std::filesystem::path &fname) { cytnx_error_msg(true, "[ERROR][Gncon][Fromfile] call from uninitialize Gncon.%s", "\n"); } void Gncon_base::FromString(const std::vector &fname) { cytnx_error_msg(true, "[ERROR][Gncon][FromString] call from uninitialize Gncon.%s", "\n"); } - void Gncon_base::Savefile(const std::string &fname) { + void Gncon_base::Savefile(const std::filesystem::path &fname) { cytnx_error_msg(true, "[ERROR][Gncon][Savefile] call from uninitialize Gncon.%s", "\n"); } void Gncon_base::PutUniTensor(const std::string &name, const UniTensor &utensor) { diff --git a/src/Network_base.cpp b/src/Network_base.cpp index 6877fc8c7..e04cd7b85 100644 --- a/src/Network_base.cpp +++ b/src/Network_base.cpp @@ -1,6 +1,8 @@ -#include #include "Network.hpp" +#include +#include + #include "linalg.hpp" using namespace std; @@ -15,13 +17,13 @@ namespace cytnx { cytnx_error_msg(true, "[ERROR][Network][Contract_plan] call from uninitialized network.%s", "\n"); } - void Network_base::Fromfile(const std::string &fname) { + void Network_base::Fromfile(const std::filesystem::path &fname) { cytnx_error_msg(true, "[ERROR][Network][Fromfile] call from uninitialized network.%s", "\n"); } void Network_base::FromString(const std::vector &fname) { cytnx_error_msg(true, "[ERROR][Network][FromString] call from uninitialized network.%s", "\n"); } - void Network_base::Savefile(const std::string &fname) { + void Network_base::Savefile(const std::filesystem::path &fname) { cytnx_error_msg(true, "[ERROR][Network][Savefile] call from uninitialized network.%s", "\n"); } void Network_base::PutUniTensor(const std::string &name, const UniTensor &utensor) { diff --git a/src/RegularGncon.cpp b/src/RegularGncon.cpp index 7a9358ca0..ea2d964e4 100644 --- a/src/RegularGncon.cpp +++ b/src/RegularGncon.cpp @@ -1,10 +1,12 @@ -#include #include "Gncon.hpp" -#include "search_tree.hpp" -#include #include +#include #include +#include +#include + +#include "search_tree.hpp" using namespace std; @@ -395,7 +397,7 @@ namespace cytnx { // print_gn(this->table, this->names, this->name2pos); } - void RegularGncon::Fromfile(const std::string &fname) { + void RegularGncon::Fromfile(const std::filesystem::path &fname) { const cytnx_uint64 MAXLINES = 1024; // empty all @@ -403,9 +405,9 @@ namespace cytnx { // open file std::ifstream infile; - infile.open(fname.c_str()); + infile.open(fname.string().c_str()); if (!(infile.is_open())) { - cytnx_error_msg(true, "[Gncon] Error in opening file \'", fname.c_str(), "\'.\n"); + cytnx_error_msg(true, "[Gncon] Error in opening file \'", fname.string().c_str(), "\'.\n"); } filename = fname; @@ -475,16 +477,16 @@ namespace cytnx { } } - void RegularGncon::Savefile(const std::string &fname) { + void RegularGncon::Savefile(const std::filesystem::path &fname) { cytnx_error_msg(this->label_arr.size() == 0, "[ERROR][RegularGncon][Savefile] Cannot save empty Gncon to Gncon file!%s", "\n"); fstream fo; - fo.open(fname + ".net", ios::out | ios::trunc); + fo.open(std::filesystem::path(fname) += ".net", ios::out | ios::trunc); if (!fo.is_open()) { cytnx_error_msg(true, "[ERROR][RegularGncon][Savefile] Cannot open/create file:%s\n", - fname.c_str()); + fname.string().c_str()); } for (int i = 0; i < this->label_arr.size(); i++) { diff --git a/src/RegularNetwork.cpp b/src/RegularNetwork.cpp index 065a354ac..c89fc9db4 100644 --- a/src/RegularNetwork.cpp +++ b/src/RegularNetwork.cpp @@ -569,7 +569,7 @@ namespace cytnx { this->einsum_path = CtTreeToEinsumpathInternal(CtTree, names); } // end of FromString - void RegularNetwork::Fromfile(const string &fname) { + void RegularNetwork::Fromfile(const std::filesystem::path &fname) { const cytnx_uint64 MAXLINES = 1024; // empty all @@ -577,9 +577,9 @@ namespace cytnx { // open file ifstream infile; - infile.open(fname.c_str()); + infile.open(fname.string().c_str()); if (!(infile.is_open())) { - cytnx_error_msg(true, "[Network] Error in opening file \'", fname.c_str(), "\'.\n"); + cytnx_error_msg(true, "[Network] Error in opening file \'", fname.string().c_str(), "\'.\n"); } filename = fname; @@ -657,16 +657,16 @@ namespace cytnx { } } - void RegularNetwork::Savefile(const string &fname) { + void RegularNetwork::Savefile(const std::filesystem::path &fname) { cytnx_error_msg( this->label_arr.size() == 0, "[ERROR][RegularNetwork][Savefile] Cannot save empty network to network file!%s", "\n"); fstream fo; - fo.open(fname + ".net", ios::out | ios::trunc); + fo.open(std::filesystem::path(fname) += ".net", ios::out | ios::trunc); if (!fo.is_open()) { cytnx_error_msg(true, "[ERROR][RegularNetwork][Savefile] Cannot open/create file:%s\n", - fname.c_str()); + fname.string().c_str()); } for (int i = 0; i < this->label_arr.size(); i++) { diff --git a/src/SparseUniTensor.cpp b/src/SparseUniTensor.cpp index 66cb1c94f..58f331a86 100644 --- a/src/SparseUniTensor.cpp +++ b/src/SparseUniTensor.cpp @@ -2324,18 +2324,26 @@ namespace cytnx { cytnx_error_msg(true, "[ERROR] truncate for SparseUniTensor is under developing!!%s", "\n"); } - void SparseUniTensor::_save_dispatch(std::fstream &f) const { + void SparseUniTensor::to_hdf5_dispatch(H5::Group &container, const bool overwrite) const { + cytnx_error_msg(true, "[ERROR] Saving SparseUniTensor to HDF5 is not implemented!%s", "\n"); + } + + void SparseUniTensor::from_hdf5_dispatch(H5::Group &container, bool restore_device) { + cytnx_error_msg(true, "[ERROR] Loading SparseUniTensor from HDF5 is not implemented!%s", "\n"); + } + + void SparseUniTensor::to_binary_dispatch(std::ostream &f) const { // cytnx_error_msg(true,"[ERROR] Save for SparseUniTensor is under developing!!%s","\n"); cytnx_uint64 Nblocks = this->_blocks.size(); f.write((char *)&Nblocks, sizeof(cytnx_uint64)); for (unsigned int i = 0; i < this->_blocks.size(); i++) { - this->_blocks[i]._Save(f); + this->_blocks[i].to_binary(f); } } - void SparseUniTensor::_load_dispatch(std::fstream &f) { + void SparseUniTensor::from_binary_dispatch(std::istream &f, bool restore_device) { // cytnx_error_msg(true,"[ERROR] Save for SparseUniTensor is under developing!!%s","\n"); cytnx_uint64 Nblocks; @@ -2346,7 +2354,7 @@ namespace cytnx { "\n"); for (unsigned int i = 0; i < this->_blocks.size(); i++) { - this->_blocks[i]._Load(f); + this->_blocks[i].from_binary(f, restore_device); } } diff --git a/src/Symmetry.cpp b/src/Symmetry.cpp index 23119747f..4f473b886 100644 --- a/src/Symmetry.cpp +++ b/src/Symmetry.cpp @@ -6,6 +6,10 @@ #include #include +#include "H5Cpp.h" + +#include "io.hpp" + using namespace std; namespace cytnx { @@ -77,6 +81,10 @@ namespace cytnx { cytnx_error_msg(1, "%s", "[ERROR][Internal] should not call Symmerty base!"); } + std::string cytnx::Symmetry_base::getname() const { + cytnx_error_msg(1, "%s", "[ERROR][Internal] should not call Symmerty base!"); + } + std::string cytnx::Symmetry_base::stype_str() const { cytnx_error_msg(1, "%s", "[ERROR][Internal] should not call Symmerty base!"); } @@ -105,7 +113,7 @@ namespace cytnx { void cytnx::U1Symmetry::print_info() const { cout << "--------------------\n"; cout << "[Symmetry]" << endl; - cout << "type : Abelian, U1" << endl; + cout << "type : Abelian, " << this->getname() << endl; cout << "combine rule : Q1 + Q2" << endl; cout << "reverse rule : Q*(-1) " << endl; cout << "--------------------\n"; @@ -155,7 +163,7 @@ namespace cytnx { void cytnx::ZnSymmetry::print_info() const { cout << "--------------------\n"; cout << "[Symmetry]" << endl; - cout << "type : Abelian, Z(" << this->n << ")" << endl; + cout << "type : Abelian, " << this->getname() << endl; cout << "combine rule : (Q1 + Q2)\%" << this->n << endl; cout << "reverse rule : Q*(-1) " << endl; cout << "--------------------\n"; @@ -209,8 +217,8 @@ namespace cytnx { void cytnx::FermionParitySymmetry::print_info() const { cout << "--------------------\n"; cout << "[Symmetry]" << endl; - cout << "type : fermionic, FermionParity" << endl; - cout << "combine rule : (Q1 + Q2)\2" << endl; + cout << "type : Fermionic, " << this->getname() << endl; + cout << "combine rule : (Q1 + Q2)\%2" << endl; cout << "reverse rule : Q*(-1) " << endl; cout << "--------------------\n"; } @@ -251,7 +259,7 @@ namespace cytnx { void cytnx::FermionNumberSymmetry::print_info() const { cout << "--------------------\n"; cout << "[Symmetry]" << endl; - cout << "type : fermionic, FermionNumber" << endl; + cout << "type : fermionic, " << this->getname() << endl; cout << "combine rule : Q1 + Q2" << endl; cout << "reverse rule : Q*(-1) " << endl; cout << "--------------------\n"; @@ -259,52 +267,166 @@ namespace cytnx { //================================================== - void cytnx::Symmetry::Save(const std::string &fname) const { - fstream f; - if (std::filesystem::path(fname).has_extension()) { + void cytnx::Symmetry::Save(const std::filesystem::path &fname, const std::string &path, + const char mode) const { + fstream f; // only for binary saving, not used for HDF5 + if (fname.has_extension()) { // filename extension is given - f.open(fname, ios::out | ios::trunc | ios::binary); - } else { - // add filename extension + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { + // save as HDF5 + H5::H5File h5file; + // Enable reuse of space after data is deleted; + // Set the strategy: FSM_AGGR is standard for free-space management + // Parameters: strategy, persist (true), threshold (default 1: track all free-space + // sections) + H5::FileCreatPropList fcpl; + fcpl.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_FSM_AGGR, true, 1); + // Persistent free space requires HDF5 1.10.x format or later + H5::FileAccPropList fapl; + fapl.setLibverBounds(H5F_LIBVER_V110, H5F_LIBVER_LATEST); + // open file + bool overwrite = false; + if (mode == 'w') { // Write new file + h5file = H5::H5File(fname, H5F_ACC_TRUNC, fcpl, fapl); + } else if (mode == 'x') { // eXclusive create + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'a') { // Append data + if (std::filesystem::exists(fname)) + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + else + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'u') { // Update data + if (std::filesystem::exists(fname)) { + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + overwrite = true; + } else { + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } + } else { + cytnx_error_msg(true, "[ERROR] Unknown mode '%c' for writing to HDF5 file.", mode); + } + // split path into group and name + std::filesystem::path p(path); + std::filesystem::path grouppath = p.parent_path(); + std::string datasetname = p.filename().string(); + if (datasetname.empty()) datasetname = "Symmetry"; + // create group + std::filesystem::path subpath; + std::string groupfolder = "/"; + for (const auto &part : grouppath) { + if (part.empty()) continue; + subpath /= part; + groupfolder = subpath.generic_string(); + if (!h5file.nameExists(groupfolder)) h5file.createGroup(groupfolder); + } + H5::Group group = h5file.openGroup(groupfolder); + // write data + this->to_hdf5(group, datasetname, overwrite); + h5file.close(); + return; + } else { // create binary file + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fname), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fname.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", + mode); + } + f.open(fname, std::ios::out | std::ios::trunc | std::ios::binary); + } + } else { // create binary file with standard extension + std::filesystem::path fnameext = fname; + fnameext += ".cysym"; cytnx_warning_msg(true, "Missing file extension in fname '%s'. I am adding the extension '.cysym'. " "This is deprecated, please provide the file extension in the future.\n", - fname.c_str()); - f.open((fname + ".cysym"), ios::out | ios::trunc | ios::binary); + fname.string().c_str()); + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fnameext), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fnameext.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", mode); + } + f.open(fnameext, std::ios::out | std::ios::trunc | std::ios::binary); } + // write binary if (!f.is_open()) { cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); } - this->_Save(f); + this->to_binary(f); f.close(); } - void cytnx::Symmetry::Save(const char *fname) const { this->Save(string(fname)); } + void cytnx::Symmetry::Save(const char *fname, const std::string &path, const char mode) const { + this->Save(std::filesystem::path(fname), path, mode); + } - cytnx::Symmetry cytnx::Symmetry::Load(const std::string &fname) { + cytnx::Symmetry cytnx::Symmetry::Load(const std::filesystem::path &fname, + const std::string &path) { Symmetry out; - fstream f; - f.open(fname, ios::in | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.c_str()); - } - out._Load(f); - f.close(); + out.Load_(fname, path); return out; } - cytnx::Symmetry cytnx::Symmetry::Load(const char *fname) { - return cytnx::Symmetry::Load(string(fname)); + cytnx::Symmetry cytnx::Symmetry::Load(const char *fname, const std::string &path) { + return cytnx::Symmetry::Load(std::filesystem::path(fname), path); + } + + void cytnx::Symmetry::Load_(const std::filesystem::path &fname, const std::string &path) { + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { // load HDF5 + H5::H5File h5file(fname, H5F_ACC_RDONLY); + // split path into group and name + std::filesystem::path p(path); + std::string grouppath = p.parent_path().generic_string(); + std::string datasetname = p.filename().string(); + if (datasetname.empty()) datasetname = "Symmetry"; + // open group + H5::Group group; + try { + group = h5file.openGroup(grouppath.empty() ? "/" : grouppath); + } catch (const H5::Exception &e) { + std::cerr << e.getDetailMsg() << std::endl; + cytnx_error_msg(true, "[ERROR] HDF5 path '%s' not found or is not a group in file '%s'.", + grouppath.c_str(), fname.string().c_str()); + } + // read data + this->from_hdf5(group, datasetname); + h5file.close(); + } else { // load binary + fstream f; + f.open(fname, ios::in | ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); + } + this->from_binary(f); + f.close(); + } + } + void cytnx::Symmetry::Load_(const char *fname, const std::string &path) { + this->Load_(std::filesystem::path(fname), path); + } + + void cytnx::Symmetry::to_hdf5(H5::Group &container, const std::string &name, + const bool overwrite) const { + io::save_attribute(this->getname(), container, name, overwrite); + } + void cytnx::Symmetry::from_hdf5(H5::Group &container, const std::string &name) { + std::string symname; + io::load_attribute(symname, container, name); + this->Init(symname); } - //================== - void cytnx::Symmetry::_Save(fstream &f) const { - cytnx_error_msg(!f.is_open(), "[ERROR][Symmetry] invalid fstream%s", "\n"); + void cytnx::Symmetry::to_binary(std::ostream &f) const { unsigned int IDDs = 777; f.write((char *)&IDDs, sizeof(unsigned int)); f.write((char *)&this->_impl->stype_id, sizeof(int)); f.write((char *)&this->_impl->n, sizeof(int)); } - void cytnx::Symmetry::_Load(fstream &f) { - cytnx_error_msg(!f.is_open(), "[ERROR][Symmetry] invalid fstream%s", "\n"); + void cytnx::Symmetry::from_binary(std::istream &f) { unsigned int tmpIDDs; f.read((char *)&tmpIDDs, sizeof(unsigned int)); cytnx_error_msg(tmpIDDs != 777, "[ERROR] the object is not a cytnx symmetry!%s", "\n"); diff --git a/src/Tensor.cpp b/src/Tensor.cpp index 7d5d51544..206cdcbb0 100644 --- a/src/Tensor.cpp +++ b/src/Tensor.cpp @@ -3,9 +3,11 @@ #include #include +#include "H5Cpp.h" +#include "io.hpp" +#include "Type.hpp" #include "linalg.hpp" #include "utils/is.hpp" -#include "Type.hpp" using namespace std; @@ -66,9 +68,7 @@ namespace cytnx { ( [&]() { if (dtype == Is) { - using TargetType = - std::variant_alternative_t::type>; + using TargetType = std::variant_alternative_t; result = static_cast(p); } }(), @@ -93,8 +93,7 @@ namespace cytnx { ( [&]() { if (dtype == Is) { - using TargetType = std::variant_alternative_t< - Is, typename Tensor::internal::exclude_first::type>; + using TargetType = std::variant_alternative_t; result = static_cast(p); } }(), @@ -424,55 +423,200 @@ namespace cytnx { //=================================================================== // wrapper - void Tensor::Tofile(const std::string &fname) const { - if (!this->is_contiguous()) { - auto A = this->contiguous(); - A.storage().Tofile(fname); - } else { - this->_impl->_storage.Tofile(fname); - } - } - void Tensor::Tofile(const char *fname) const { - if (!this->is_contiguous()) { - auto A = this->contiguous(); - A.storage().Tofile(fname); - } else { - this->_impl->_storage.Tofile(fname); - } - } - void Tensor::Tofile(fstream &f) const { - if (!this->is_contiguous()) { - auto A = this->contiguous(); - A.storage().Tofile(f); - } else { - this->_impl->_storage.Tofile(f); - } - } - void Tensor::Save(const std::string &fname) const { - fstream f; - if (std::filesystem::path(fname).has_extension()) { + void cytnx::Tensor::Save(const std::filesystem::path &fname, const std::string &path, + const char mode) const { + fstream f; // only for binary saving, not used for HDF5 + if (fname.has_extension()) { // filename extension is given - f.open(fname, ios::out | ios::trunc | ios::binary); - } else { - // add filename extension + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { + // save as HDF5 + H5::H5File h5file; + // Enable reuse of space after data is deleted; + // Set the strategy: FSM_AGGR is standard for free-space management + // Parameters: strategy, persist (true), threshold (default 1: track all free-space + // sections) + H5::FileCreatPropList fcpl; + fcpl.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_FSM_AGGR, true, 1); + // Persistent free space requires HDF5 1.10.x format or later + H5::FileAccPropList fapl; + fapl.setLibverBounds(H5F_LIBVER_V110, H5F_LIBVER_LATEST); + // open file + bool overwrite = false; + if (mode == 'w') { // Write new file + h5file = H5::H5File(fname, H5F_ACC_TRUNC, fcpl, fapl); + } else if (mode == 'x') { // eXclusive create + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'a') { // Append data + if (std::filesystem::exists(fname)) + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + else + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'u') { // Update data + if (std::filesystem::exists(fname)) { + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + overwrite = true; + } else { + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } + } else { + cytnx_error_msg(true, "[ERROR] Unknown mode '%c' for writing to HDF5 file.", mode); + } + // split path into group and name + std::filesystem::path p(path); + std::filesystem::path grouppath = p.parent_path(); + std::string datasetname = p.filename().string(); + if (datasetname.empty()) datasetname = "Tensor"; + // create group + std::filesystem::path subpath; + std::string groupfolder = "/"; + for (const auto &part : grouppath) { + if (part.empty()) continue; + subpath /= part; + groupfolder = subpath.generic_string(); + if (!h5file.nameExists(groupfolder)) h5file.createGroup(groupfolder); + } + H5::Group group = h5file.openGroup(groupfolder); + // write data + this->to_hdf5(group, datasetname, overwrite); + h5file.close(); + return; + } else { // create binary file + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fname), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fname.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", + mode); + } + f.open(fname, std::ios::out | std::ios::trunc | std::ios::binary); + } + } else { // create binary file with standard extension + std::filesystem::path fnameext = fname; + fnameext += ".cytn"; cytnx_warning_msg(true, "Missing file extension in fname '%s'. I am adding the extension '.cytn'. " "This is deprecated, please provide the file extension in the future.\n", - fname.c_str()); - f.open((fname + ".cytn"), ios::out | ios::trunc | ios::binary); + fname.string().c_str()); + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fnameext), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fnameext.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", mode); + } + f.open(fnameext, std::ios::out | std::ios::trunc | std::ios::binary); } + // write binary if (!f.is_open()) { cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); } - this->_Save(f); + this->to_binary(f); f.close(); } - void Tensor::Save(const char *fname) const { this->Save(string(fname)); } - void Tensor::_Save(fstream &f) const { - // header - // check: - cytnx_error_msg(!f.is_open(), "[ERROR] invalid fstream!.%s", "\n"); + void cytnx::Tensor::Save(const char *fname, const std::string &path, const char mode) const { + this->Save(std::filesystem::path(fname), path, mode); + } + + cytnx::Tensor cytnx::Tensor::Load(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + Tensor out; + out.Load_(fname, path, restore_device); + return out; + } + cytnx::Tensor cytnx::Tensor::Load(const char *fname, const std::string &path, + bool restore_device) { + return cytnx::Tensor::Load(std::filesystem::path(fname), path, restore_device); + } + + void cytnx::Tensor::Load_(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { // load HDF5 + H5::H5File h5file(fname, H5F_ACC_RDONLY); + // split path into group and name + std::filesystem::path p(path); + std::string grouppath = p.parent_path().generic_string(); + std::string datasetname = p.filename().string(); + if (datasetname.empty()) datasetname = "Tensor"; + // open group + H5::Group group; + try { + group = h5file.openGroup(grouppath.empty() ? "/" : grouppath); + } catch (const H5::Exception &e) { + std::cerr << e.getDetailMsg() << std::endl; + cytnx_error_msg(true, "[ERROR] HDF5 path '%s' not found or is not a group in file '%s'.", + grouppath.c_str(), fname.string().c_str()); + } + // read data + this->from_hdf5(group, datasetname, restore_device); + h5file.close(); + } else { // load binary + fstream f; + f.open(fname, ios::in | ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); + } + this->from_binary(f, restore_device); + f.close(); + } + } + void cytnx::Tensor::Load_(const char *fname, const std::string &path, bool restore_device) { + this->Load_(std::filesystem::path(fname), path, restore_device); + } + void Tensor::to_hdf5(H5::Group &container, const std::string &name, const bool overwrite) const { + Tensor ten = this->contiguous(); + + H5::DataType datatype = Type.dtype_to_hdf5_type(this->dtype()); + H5::DataSet dataset = + io::internal::create_dataset(datatype, this->shape(), container, name, overwrite); + ten.storage().data_to_hdf5(dataset, datatype); + + if (this->device() != Device.cpu) { + io::save_attribute(this->device(), dataset, "device", overwrite); + } else { + io::remove_attribute(dataset, "device", overwrite); + } + } + + void Tensor::from_hdf5(H5::Group &container, const std::string &name, bool restore_device) { + H5::DataSet dataset = container.openDataSet(name); + H5::DataType datatype = dataset.getDataType(); + unsigned int dtype = Type.from_hdf5_type(datatype); + H5::DataSpace dataspace = dataset.getSpace(); + int rank = dataspace.getSimpleExtentNdims(); + std::vector dims(rank); + dataspace.getSimpleExtentDims(dims.data()); + auto Nelem = dataspace.getSimpleExtentNpoints(); + + this->_impl->_shape = std::vector(dims.begin(), dims.end()); + this->_impl->_mapper = vec_range(this->_impl->_shape.size()); + this->_impl->_invmapper = this->_impl->_mapper; + this->_impl->_contiguous = true; // HDF5 data is always stored in contiguous layout + + int device = Device.cpu; + if (restore_device && dataset.attrExists("device")) { + io::load_attribute(device, dataset, "device"); + #ifndef UNI_GPU + if (device != Device.cpu) { + cytnx_warning_msg(true, + "Cytnx was compiled without CPU support, but Tensor in HDF5 file " + "requests to load to GPU with id %d. Loading Tensor to CPU memory " + "instead. Use restore_device=false to disable this message.", + device); + device = Device.cpu; + } + #endif + } + + this->_impl->_storage.data_from_hdf5(dataset, Nelem, dtype, datatype, device); + } + + void Tensor::to_binary(std::ostream &f) const { unsigned int IDDs = 888; f.write((char *)&IDDs, sizeof(unsigned int)); cytnx_uint64 shp = this->shape().size(); @@ -485,33 +629,10 @@ namespace cytnx { f.write((char *)&this->_impl->_invmapper[0], sizeof(cytnx_uint64) * shp); // pass to storage for save: - this->_impl->_storage._Save(f); - } - - Tensor Tensor::Fromfile(const std::string &fname, const unsigned int &dtype, - const cytnx_int64 &count) { - return Tensor::from_storage(Storage::Fromfile(fname, dtype, count)); - } - Tensor Tensor::Fromfile(const char *fname, const unsigned int &dtype, const cytnx_int64 &count) { - return Tensor::from_storage(Storage::Fromfile(fname, dtype, count)); + this->storage().to_binary(f); } - Tensor Tensor::Load(const std::string &fname) { - Tensor out; - fstream f; - f.open(fname, ios::in | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.c_str()); - } - out._Load(f); - f.close(); - return out; - } - Tensor Tensor::Load(const char *fname) { return Tensor::Load(string(fname)); } - void Tensor::_Load(fstream &f) { - // header - // check: - cytnx_error_msg(!f.is_open(), "[ERROR] invalid fstream!.%s", "\n"); + void Tensor::from_binary(std::istream &f, bool restore_device) { unsigned int tmpIDDs; f.read((char *)&tmpIDDs, sizeof(unsigned int)); cytnx_error_msg(tmpIDDs != 888, "[ERROR] the object is not a cytnx tensor!%s", "\n"); @@ -530,7 +651,41 @@ namespace cytnx { f.read((char *)&this->_impl->_invmapper[0], sizeof(cytnx_uint64) * shp); // pass to storage for save: - this->_impl->_storage._Load(f); + this->_impl->_storage.from_binary(f, restore_device); + } + + void Tensor::Tofile(const std::filesystem::path &fname) const { + if (!this->is_contiguous()) { + auto A = this->contiguous(); + A.storage().Tofile(fname); + } else { + this->storage().Tofile(fname); + } + } + void Tensor::Tofile(const char *fname) const { + if (!this->is_contiguous()) { + auto A = this->contiguous(); + A.storage().Tofile(fname); + } else { + this->storage().Tofile(fname); + } + } + void Tensor::Tofile(fstream &f) const { + if (!this->is_contiguous()) { + auto A = this->contiguous(); + A.storage().Tofile(f); + } else { + this->storage().Tofile(f); + } + } + + Tensor Tensor::Fromfile(const std::filesystem::path &fname, const unsigned int &dtype, + const cytnx_int64 &count, const int device) { + return Tensor::from_storage(Storage::Fromfile(fname, dtype, count, device)); + } + Tensor Tensor::Fromfile(const char *fname, const unsigned int &dtype, const cytnx_int64 &count, + const int device) { + return Tensor::from_storage(Storage::Fromfile(fname, dtype, count, device)); } Tensor Tensor::real() { diff --git a/src/Type.cpp b/src/Type.cpp index 99aef3e0d..66389003b 100644 --- a/src/Type.cpp +++ b/src/Type.cpp @@ -25,11 +25,9 @@ namespace cytnx { using namespace std; // global debug flag! -namespace cytnx { - bool User_debug = false; -} namespace cytnx { + bool User_debug = false; // Construct an array of typeid(T).name() for each type in Type_list. // This is complicated by Type_list containing 'void', which means we can't use an ordinary diff --git a/src/UniTensor.cpp b/src/UniTensor.cpp index f37071e6f..d3102b6d9 100644 --- a/src/UniTensor.cpp +++ b/src/UniTensor.cpp @@ -3,9 +3,12 @@ #include #include -#include "utils/utils.hpp" +#include "H5Cpp.h" + +#include "io.hpp" #include "linalg.hpp" #include "random.hpp" +#include "utils/utils.hpp" using namespace std; @@ -14,6 +17,26 @@ using namespace std; namespace cytnx { + void UniTensor::Init(const std::string name) { + if (name == UTenType.getname(UTenType.Block)) { + boost::intrusive_ptr tmp(new BlockUniTensor); + this->_impl = tmp; + } else if (name == UTenType.getname(UTenType.BlockFermionic)) { + boost::intrusive_ptr tmp(new BlockFermionicUniTensor); + this->_impl = tmp; + } else if (name == UTenType.getname(UTenType.Dense)) { + boost::intrusive_ptr tmp(new DenseUniTensor); + this->_impl = tmp; + } else if (name == UTenType.getname(UTenType.Void)) { + boost::intrusive_ptr tmp(new UniTensor_base); + this->_impl = tmp; + } else if (name == UTenType.getname(UTenType.Sparse)) { + cytnx_error_msg(true, "[ERROR] SparseUniTensor is deprecated.\s", "\n"); + } else { + cytnx_error_msg(true, "[ERROR] No UniTensor type matches the string '%s'.\n", name.c_str()); + } + } + UniTensor UniTensor::Pow(const double &p) const { return cytnx::linalg::Pow(*this, p); } UniTensor &UniTensor::Pow_(const double &p) { cytnx::linalg::Pow_(*this, p); @@ -38,8 +61,245 @@ namespace cytnx { UniTensor UniTensor::Mul(const UniTensor &rhs) const { return cytnx::linalg::Mul(*this, rhs); } UniTensor UniTensor::Mul(const Scalar &rhs) const { return cytnx::linalg::Mul(*this, rhs); } - void UniTensor::_Save(std::fstream &f) const { - cytnx_error_msg(!f.is_open(), "[ERROR][UniTensor] invalid fstream!.%s", "\n"); + void UniTensor::Save(const std::filesystem::path &fname, const std::string &path, + const char mode) const { + fstream f; // only for binary saving, not used for HDF5 + if (fname.has_extension()) { + // filename extension is given + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { + // save as HDF5 + H5::H5File h5file; + // Enable reuse of space after data is deleted; + // Set the strategy: FSM_AGGR is standard for free-space management + // Parameters: strategy, persist (true), threshold (default 1: track all free-space + // sections) + H5::FileCreatPropList fcpl; + fcpl.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_FSM_AGGR, true, 1); + // Persistent free space requires HDF5 1.10.x format or later + H5::FileAccPropList fapl; + fapl.setLibverBounds(H5F_LIBVER_V110, H5F_LIBVER_LATEST); + // open file + bool overwrite = false; + if (mode == 'w') { // Write new file + h5file = H5::H5File(fname, H5F_ACC_TRUNC, fcpl, fapl); + } else if (mode == 'x') { // eXclusive create + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'a') { // Append data + if (std::filesystem::exists(fname)) + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + else + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'u') { // Update data + if (std::filesystem::exists(fname)) { + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + overwrite = true; + } else { + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } + } else { + cytnx_error_msg(true, "[ERROR] Unknown mode '%c' for writing to HDF5 file.", mode); + } + // create group + std::filesystem::path grouppath(path); + std::filesystem::path subpath; + std::string groupfolder = "/"; + for (const auto &part : grouppath) { + if (part.empty()) continue; + subpath /= part; + groupfolder = subpath.generic_string(); + if (!h5file.nameExists(groupfolder)) h5file.createGroup(groupfolder); + } + H5::Group group = h5file.openGroup(groupfolder); + // write data + this->to_hdf5(group, "", overwrite); + h5file.close(); + return; + } else { // create binary file + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fname), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fname.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", + mode); + } + f.open(fname, std::ios::out | std::ios::trunc | std::ios::binary); + } + } else { // create binary file with standard extension + std::filesystem::path fnameext = fname; + fnameext += ".cytnx"; + cytnx_warning_msg(true, + "Missing file extension in fname '%s'. I am adding the extension '.cytnx'. " + "This is deprecated, please provide the file extension in the future.\n", + fname.string().c_str()); + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fnameext), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fnameext.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", mode); + } + f.open(fnameext, std::ios::out | std::ios::trunc | std::ios::binary); + } + // write binary + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + } + this->to_binary(f); + f.close(); + } + void UniTensor::Save(const char *fname, const std::string &path, const char mode) const { + this->Save(std::filesystem::path(fname), path, mode); + } + + UniTensor UniTensor::Load(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + UniTensor out; + out.Load_(fname, path, restore_device); + return out; + } + UniTensor UniTensor::Load(const char *fname, const std::string &path, bool restore_device) { + return UniTensor::Load(std::filesystem::path(fname), path, restore_device); + } + + void UniTensor::Load_(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { // load HDF5 + H5::H5File h5file(fname, H5F_ACC_RDONLY); + // open group + H5::Group group; + try { + group = h5file.openGroup(path.empty() ? "/" : path); + } catch (const H5::Exception &e) { + std::cerr << e.getDetailMsg() << std::endl; + cytnx_error_msg(true, "[ERROR] HDF5 path '%s' not found or is not a group in file '%s'.", + path.c_str(), fname.string().c_str()); + } + // read data + this->from_hdf5(group, "", restore_device); + h5file.close(); + } else { // load binary + fstream f; + f.open(fname, std::ios::in | std::ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); + } + this->from_binary(f, restore_device); + f.close(); + } + } + void UniTensor::Load_(const char *fname, const std::string &path, bool restore_device) { + this->Load_(std::filesystem::path(fname), path, restore_device); + } + + void UniTensor::to_hdf5(H5::Group &container, const std::string &name, + const bool overwrite) const { + H5::Group rootgroup; + if (name.empty()) + rootgroup = container; + else { + rootgroup = io::create_group(container, name, false); + } + + // write attributes + io::save_attribute(UTenType.getname(this->_impl->uten_type_id), rootgroup, "type", overwrite); + if (this->_impl->_is_diag) { + io::save_attribute(this->_impl->_is_diag, rootgroup, "diagonal", overwrite); + } else { + io::remove_attribute(rootgroup, "diagonal", overwrite); + } + io::save_attribute(this->_impl->_rowrank, rootgroup, "rowrank", overwrite); + io::save_attribute(this->_impl->_name, rootgroup, "name", overwrite); + + // write labels + if (this->_impl->_labels.empty()) { + io::unlink(rootgroup, "labels", overwrite); + } else { + io::save_dataset(this->_impl->_labels, rootgroup, "labels", overwrite); + } + + // write bonds + if (this->_impl->_bonds.empty()) { + io::unlink(rootgroup, "bonds", overwrite); + } else { + H5::Group bondloc = io::create_group(rootgroup, "bonds", false); + for (int i = 0; i < this->_impl->_bonds.size(); i++) { + this->_impl->_bonds[i].to_hdf5(bondloc, "Bond" + std::to_string(i), overwrite); + } + } + + this->_impl->to_hdf5_dispatch(rootgroup, overwrite); + } + + void UniTensor::from_hdf5(H5::Group &container, const std::string &name, bool restore_device) { + H5::Group rootgroup = (name.empty() ? container : container.openGroup(name)); + + // type, read from attribute + std::string utenname; + io::load_attribute(utenname, rootgroup, "type"); + this->Init(utenname); + + // is_diag, read from attribute + if (rootgroup.attrExists("diagonal")) { + io::load_attribute(this->_impl->_is_diag, rootgroup, "diagonal"); + } else { + this->_impl->_is_diag = false; + } + + // rowrank, read from attribute + io::load_attribute(this->_impl->_rowrank, rootgroup, "rowrank"); + + // name, read from string attribute + if (rootgroup.attrExists("name")) { + io::load_attribute(this->_impl->_name, rootgroup, "name"); + } else { + this->_impl->_name.clear(); + } + + // labels; read from string vector + if (rootgroup.nameExists("labels")) { + io::load_dataset(this->_impl->_labels, rootgroup, "labels"); + } else { + this->_impl->_labels.clear(); + } + + // bonds; read from group + this->_impl->_bonds.clear(); + if (rootgroup.nameExists("bonds")) { + H5::Group dir = rootgroup.openGroup("bonds"); + hsize_t idx = 0; + while (true) { + std::string bondname = "Bond" + std::to_string(idx); + if (!dir.nameExists(bondname)) { + break; + } + Bond bond; + bond.from_hdf5(dir, bondname); + this->_impl->_bonds.push_back(bond); + idx++; + } + cytnx_error_msg( + idx != this->_impl->_labels.size(), + "[ERROR] %zu bonds were found, but %zu labels exist. The HDF5 data seems corrupt!\n", idx, + this->_impl->_labels.size()); + } else { + cytnx_error_msg( + !this->_impl->_labels.empty(), + "[ERROR] %zu labels exist, but no bonds were found. The HDF5 data seems corrupt!\n", + this->_impl->_labels.size()); + this->_impl->_bonds.clear(); + } + + // dispatch + this->_impl->from_hdf5_dispatch(rootgroup, restore_device); + this->_impl->_is_braket_form = this->_impl->_update_braket(); + } + + void UniTensor::to_binary(std::ostream &f) const { cytnx_error_msg(this->_impl->uten_type_id == UTenType.Void, "[ERROR][UniTensor] Cannot save an uninitialized UniTensor.%s", "\n"); @@ -82,15 +342,14 @@ namespace cytnx { } // f.write((char *)&(this->_impl->_labels[0]), sizeof(cytnx_int64) * rank); for (cytnx_uint64 i = 0; i < rank; i++) { - this->_impl->_bonds[i]._Save(f); + this->_impl->_bonds[i].to_binary(f); } // second, let dispatch to do remaining saving. - this->_impl->_save_dispatch(f); + this->_impl->to_binary_dispatch(f); } - void UniTensor::_Load(std::fstream &f) { - cytnx_error_msg(!f.is_open(), "[ERROR][UniTensor] invalid fstream%s", "\n"); + void UniTensor::from_binary(std::istream &f, bool restore_device) { unsigned int tmpIDDs; f.read((char *)&tmpIDDs, sizeof(unsigned int)); cytnx_error_msg(tmpIDDs != 555, "[ERROR] the object is not a cytnx UniTensor!%s", "\n"); @@ -129,6 +388,8 @@ namespace cytnx { f.read(cname, sizeof(char) * len_name); this->_impl->_name = std::string(cname); free(cname); + } else { + this->_impl->_name.clear(); } cytnx_uint64 rank; @@ -146,46 +407,12 @@ namespace cytnx { } // f.read((char *)&(this->_impl->_labels[0]), sizeof(cytnx_int64) * rank); for (cytnx_uint64 i = 0; i < rank; i++) { - this->_impl->_bonds[i]._Load(f); + this->_impl->_bonds[i].from_binary(f); } // second, let dispatch to do remaining loading. - this->_impl->_load_dispatch(f); - } - - void UniTensor::Save(const std::string &fname) const { - fstream f; - if (std::filesystem::path(fname).has_extension()) { - // filename extension is given - f.open(fname, ios::out | ios::trunc | ios::binary); - } else { - // add filename extension - cytnx_warning_msg(true, - "Missing file extension in fname '%s'. I am adding the extension '.cytnx'. " - "This is deprecated, please provide the file extension in the future.\n", - fname.c_str()); - f.open((fname + ".cytnx"), ios::out | ios::trunc | ios::binary); - } - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); - } - this->_Save(f); - f.close(); - } - void UniTensor::Save(const char *fname) const { Save(string(fname)); } - - UniTensor UniTensor::Load(const std::string &fname) { - UniTensor out; - fstream f; - f.open(fname, ios::in | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] invalid file path for load. >> %s\n", fname.c_str()); - } - out._Load(f); - f.close(); - return out; + this->_impl->from_binary_dispatch(f, restore_device); } - UniTensor UniTensor::Load(const char *fname) { return UniTensor::Load(string(fname)); } // Random Generators: UniTensor UniTensor::normal(const cytnx_uint64 &Nelem, const double &mean, const double &std, diff --git a/src/UniTensor_base.cpp b/src/UniTensor_base.cpp index 1cb1e9beb..d7655aaf5 100644 --- a/src/UniTensor_base.cpp +++ b/src/UniTensor_base.cpp @@ -11,20 +11,19 @@ namespace cytnx { //==================================================== std::string UniTensorType_class::getname(const int &ut_type) const { if (ut_type == this->Void) { - return std::string("Void (un-initialize UniTensor)"); + return std::string("Void (uninitialized UniTensor)"); } else if (ut_type == this->Dense) { - return std::string("Dense"); + return std::string("DenseUniTensor"); } else if (ut_type == this->Sparse) { - return std::string("Sparse"); + return std::string("SparseUniTensor"); } else if (ut_type == this->Block) { - return std::string("Block"); + return std::string("BlockUniTensor"); } else if (ut_type == this->BlockFermionic) { - return std::string("Block Fermionic"); + return std::string("BlockFermionicUniTensor"); } else { cytnx_error_msg(true, "%s\n", "[ERROR] invalid ut_type"); return std::string(""); } - // extend more in here!! } UniTensorType_class UTenType; //=================================================== @@ -692,12 +691,22 @@ namespace cytnx { true, "[ERROR] fatal internal, cannot call on an un-initialized UniTensor_base%s", "\n"); } - void UniTensor_base::_save_dispatch(std::fstream &f) const { + void UniTensor_base::to_hdf5_dispatch(H5::Group &container, const bool overwrite) const { cytnx_error_msg( true, "[ERROR] fatal internal, cannot call on an un-initialized UniTensor_base%s", "\n"); } - void UniTensor_base::_load_dispatch(std::fstream &f) { + void UniTensor_base::from_hdf5_dispatch(H5::Group &container, bool restore_device) { + cytnx_error_msg(true, "[ERROR] Loading BlockUniTensor from HDF5 is not implemented yet!%s", + "\n"); + } + + void UniTensor_base::to_binary_dispatch(std::ostream &f) const { + cytnx_error_msg( + true, "[ERROR] fatal internal, cannot call on an un-initialized UniTensor_base%s", "\n"); + } + + void UniTensor_base::from_binary_dispatch(std::istream &f, bool restore_device) { cytnx_error_msg( true, "[ERROR] fatal internal, cannot call on an un-initialized UniTensor_base%s", "\n"); } diff --git a/src/backend/Storage.cpp b/src/backend/Storage.cpp index 8415d62a3..e6544fef1 100644 --- a/src/backend/Storage.cpp +++ b/src/backend/Storage.cpp @@ -3,6 +3,10 @@ #include #include +#include "H5Cpp.h" + +#include "io.hpp" + using namespace std; namespace cytnx { @@ -82,57 +86,235 @@ namespace cytnx { } bool Storage::operator!=(const Storage &rhs) { return !(*this == rhs); } - void Storage::Save(const std::string &fname) const { - fstream f; - if (std::filesystem::path(fname).has_extension()) { + void Storage::Save(const std::filesystem::path &fname, const std::string &path, + const char mode) const { + fstream f; // only for binary saving, not used for HDF5 + if (fname.has_extension()) { // filename extension is given - f.open(fname, ios::out | ios::trunc | ios::binary); - } else { - // add filename extension + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { + // save as HDF5 + H5::H5File h5file; + // Enable reuse of space after data is deleted; + // Set the strategy: FSM_AGGR is standard for free-space management + // Parameters: strategy, persist (true), threshold (default 1: track all free-space + // sections) + H5::FileCreatPropList fcpl; + fcpl.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_FSM_AGGR, true, 1); + // Persistent free space requires HDF5 1.10.x format or later + H5::FileAccPropList fapl; + fapl.setLibverBounds(H5F_LIBVER_V110, H5F_LIBVER_LATEST); + // open file + bool overwrite = false; + if (mode == 'w') { // Write new file + h5file = H5::H5File(fname, H5F_ACC_TRUNC, fcpl, fapl); + } else if (mode == 'x') { // eXclusive create + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'a') { // Append data + if (std::filesystem::exists(fname)) + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + else + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'u') { // Update data + if (std::filesystem::exists(fname)) { + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + overwrite = true; + } else { + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } + } else { + cytnx_error_msg(true, "[ERROR] Unknown mode '%c' for writing to HDF5 file.", mode); + } + // split path into group and name + std::filesystem::path p(path); + std::filesystem::path grouppath = p.parent_path(); + std::string datasetname = p.filename().string(); + if (datasetname.empty()) datasetname = "Storage"; + // create group + std::filesystem::path subpath; + std::string groupfolder = "/"; + for (const auto &part : grouppath) { + if (part.empty()) continue; + subpath /= part; + groupfolder = subpath.generic_string(); + if (!h5file.nameExists(groupfolder)) h5file.createGroup(groupfolder); + } + H5::Group group = h5file.openGroup(groupfolder); + // write data + this->to_hdf5(group, datasetname, overwrite); + h5file.close(); + return; + } else { // create binary file + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fname), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fname.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", + mode); + } + f.open(fname, std::ios::out | std::ios::trunc | std::ios::binary); + } + } else { // create binary file with standard extension + std::filesystem::path fnameext = fname; + fnameext += ".cyst"; cytnx_warning_msg(true, "Missing file extension in fname '%s'. I am adding the extension '.cyst'. " "This is deprecated, please provide the file extension in the future.\n", - fname.c_str()); - f.open((fname + ".cyst"), ios::out | ios::trunc | ios::binary); + fname.string().c_str()); + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fnameext), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fnameext.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", mode); + } + f.open(fnameext, std::ios::out | std::ios::trunc | std::ios::binary); } + // write binary if (!f.is_open()) { cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); } - this->_Save(f); + this->to_binary(f); f.close(); } - void Storage::Save(const char *fname) const { this->Save(string(fname)); } - void Storage::Tofile(const std::string &fname) const { - fstream f; - f.open(fname, ios::out | ios::trunc | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + void Storage::Save(const char *fname, const std::string &path, const char mode) const { + this->Save(std::filesystem::path(fname), path, mode); + } + + Storage Storage::Load(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + Storage out; + out.Load_(fname, path, restore_device); + return out; + } + Storage Storage::Load(const char *fname, const std::string &path, bool restore_device) { + return Storage::Load(std::filesystem::path(fname), path, restore_device); + } + + void Storage::Load_(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { // load HDF5 + H5::H5File h5file(fname, H5F_ACC_RDONLY); + // split path into group and name + std::filesystem::path p(path); + std::string grouppath = p.parent_path().generic_string(); + std::string datasetname = p.filename().string(); + if (datasetname.empty()) datasetname = "Storage"; + // open group + H5::Group group; + try { + group = h5file.openGroup(grouppath.empty() ? "/" : grouppath); + } catch (const H5::Exception &e) { + std::cerr << e.getDetailMsg() << std::endl; + cytnx_error_msg(true, "[ERROR] HDF5 path '%s' not found or is not a group in file '%s'.", + grouppath.c_str(), fname.string().c_str()); + } + // read data + this->from_hdf5(group, datasetname, restore_device); + h5file.close(); + } else { // load binary + fstream f; + f.open(fname, ios::in | ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); + } + this->from_binary(f, restore_device); + f.close(); } - this->_Savebinary(f); - f.close(); } - void Storage::Tofile(const char *fname) const { - fstream f; - string ffname = string(fname); - f.open(ffname, ios::out | ios::trunc | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + void Storage::Load_(const char *fname, const std::string &path, bool restore_device) { + this->Load_(std::filesystem::path(fname), path, restore_device); + } + + void Storage::to_hdf5(H5::Group &container, const std::string &name, const bool overwrite) const { + H5::DataType datatype = Type.dtype_to_hdf5_type(this->dtype()); + H5::DataSet dataset = + io::internal::create_dataset(datatype, {this->size()}, container, name, overwrite); + this->data_to_hdf5(dataset, datatype); + + if (this->device() != Device.cpu) { + io::save_attribute(this->device(), dataset, "device", overwrite); + } else { + io::remove_attribute(dataset, "device", overwrite); } - this->_Savebinary(f); - f.close(); } - void Storage::Tofile(fstream &f) const { - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + + void Storage::from_hdf5(H5::Group &container, const std::string &name, bool restore_device) { + H5::DataSet dataset = container.openDataSet(name); + H5::DataType datatype = dataset.getDataType(); + unsigned int dtype = Type.from_hdf5_type(datatype); + H5::DataSpace dataspace = dataset.getSpace(); + auto Nelem = dataspace.getSimpleExtentNpoints(); + + int device = Device.cpu; + if (restore_device && dataset.attrExists("device")) { + io::load_attribute(device, dataset, "device"); +#ifndef UNI_GPU + if (device != Device.cpu) { + cytnx_warning_msg(true, + "Cytnx was compiled without CPU support, but Storage in HDF5 file " + "requests to load to GPU with id %d. Loading Storage to CPU memory " + "instead. Use restore_device=false to disable this message.", + device); + device = Device.cpu; + } +#endif } - this->_Savebinary(f); + + this->data_from_hdf5(dataset, Nelem, dtype, datatype, device); } - void Storage::_Save(fstream &f) const { - // header - // check: - cytnx_error_msg(!f.is_open(), "[ERROR] invalid fstream!.%s", "\n"); + void Storage::data_to_hdf5(H5::DataSet &dataset, H5::DataType &hdf5type) const { + if (this->device() == Device.cpu) { + dataset.write(this->data(), hdf5type); + } else { +#ifdef UNI_GPU + checkCudaErrors(cudaSetDevice(this->device())); + void *htmp = malloc(Type.typeSize(this->dtype()) * this->size()); + checkCudaErrors(cudaMemcpy(htmp, this->_impl->data(), + Type.typeSize(this->dtype()) * this->size(), + cudaMemcpyDeviceToHost)); + dataset.write(htmp, hdf5type); + free(htmp); +#else + cytnx_error_msg(true, + "[ERROR][Storage] Trying to access data from GPU without GPU support. " + "Internal fatal error!%s", + "\n"); +#endif + } + } + + void Storage::data_from_hdf5(H5::DataSet &dataset, const cytnx_uint64 &Nelem, + const unsigned int &dtype, H5::DataType &hdf5type, + const int &device) { + this->_impl = __SII.USIInit[dtype](); + this->_impl->Init(Nelem, device, false); + if (device == Device.cpu) { + dataset.read(this->_impl->data(), hdf5type); + } else { +#ifdef UNI_GPU + checkCudaErrors(cudaSetDevice(device)); + void *htmp = malloc(Type.typeSize(dtype) * Nelem); + dataset.read(htmp, hdf5type); + checkCudaErrors(cudaMemcpy(this->_impl->data(), htmp, Type.typeSize(dtype) * Nelem, + cudaMemcpyHostToDevice)); + free(htmp); +#else + cytnx_error_msg(true, + "[ERROR] Trying to load Storage from HDF5 file to the GPU with id %d, but " + "cytnx was compiled without GPU support!\n", + device); +#endif + } + } + + void Storage::to_binary(std::ostream &f) const { unsigned int IDDs = 999; f.write((char *)&IDDs, sizeof(unsigned int)); auto write_number = [&f](auto number) { @@ -142,7 +324,39 @@ namespace cytnx { write_number(this->dtype()); write_number(this->device()); - // data: + this->data_to_binary(f); + } + + void Storage::from_binary(std::istream &f, bool restore_device) { + unsigned long long Nelem; + unsigned int dtype; + int device; + // checking IDD + unsigned int tmpIDDs; + f.read((char *)&tmpIDDs, sizeof(unsigned int)); + if (tmpIDDs != 999) { + cytnx_error_msg(true, "[ERROR] the Load file is not the Storage object!\n", "%s"); + } + + f.read((char *)&Nelem, sizeof(unsigned long long)); + f.read((char *)&dtype, sizeof(unsigned int)); + f.read((char *)&device, sizeof(int)); + + if (restore_device) { + if (device != Device.cpu && device >= Device.Ngpus) { + cytnx_warning_msg(true, + "[Warning!!] the original device ID does not exists. the tensor will be " + "put on CPU, please use .to() or .to_() to move to desire devices.%s", + "\n"); + device = Device.cpu; + } + } else { + device = Device.cpu; + } + this->data_from_binary(f, Nelem, dtype, device); + } + + void Storage::data_to_binary(std::ostream &f) const { if (this->device() == Device.cpu) { f.write((char *)this->_impl->data(), Type.typeSize(this->dtype()) * this->size()); } else { @@ -154,42 +368,67 @@ namespace cytnx { cudaMemcpyDeviceToHost)); f.write((char *)htmp, Type.typeSize(this->dtype()) * this->size()); free(htmp); - #else cytnx_error_msg(true, "ERROR internal fatal error in Save Storage%s", "\n"); #endif } } - void Storage::_Savebinary(fstream &f) const { - // header - // check: - cytnx_error_msg(!f.is_open(), "[ERROR] invalid fstream!.%s", "\n"); - // data: - if (this->device() == Device.cpu) { - f.write((char *)this->_impl->data(), Type.typeSize(this->dtype()) * this->size()); + void Storage::data_from_binary(std::istream &f, const cytnx_uint64 &Nelem, + const unsigned int &dtype, const int &device) { + // before enter this func, make sure + // 1. dtype is not void. + // 2. the Nelement is consistent and smaller than the file size, and should not be zero! + this->_impl = __SII.USIInit[dtype](); + this->_impl->Init(Nelem, device, false); + if (device == Device.cpu) { + f.read((char *)this->_impl->data(), Type.typeSize(dtype) * Nelem); } else { #ifdef UNI_GPU - checkCudaErrors(cudaSetDevice(this->device())); - void *htmp = malloc(Type.typeSize(this->dtype()) * this->size()); - checkCudaErrors(cudaMemcpy(htmp, this->_impl->data(), - Type.typeSize(this->dtype()) * this->size(), - cudaMemcpyDeviceToHost)); - f.write((char *)htmp, Type.typeSize(this->dtype()) * this->size()); + checkCudaErrors(cudaSetDevice(device)); + void *htmp = malloc(Type.typeSize(dtype) * Nelem); + f.read((char *)htmp, Type.typeSize(dtype) * Nelem); + checkCudaErrors(cudaMemcpy(this->_impl->data(), htmp, Type.typeSize(dtype) * Nelem, + cudaMemcpyHostToDevice)); free(htmp); - #else - cytnx_error_msg(true, "ERROR internal fatal error in Save Storage%s", "\n"); + cytnx_error_msg(true, "ERROR internal fatal error in Load Storage%s", "\n"); #endif } } - Storage Storage::Fromfile(const char *fname, const unsigned int &dtype, - const cytnx_int64 &count) { - return Storage::Fromfile(string(fname), dtype, count); + void Storage::Tofile(const std::filesystem::path &fname) const { + fstream f; + f.open(fname, ios::out | ios::trunc | ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + } + this->data_to_binary(f); + f.close(); + } + void Storage::Tofile(const char *fname) const { + fstream f; + string ffname = string(fname); + f.open(ffname, ios::out | ios::trunc | ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + } + this->data_to_binary(f); + f.close(); + } + void Storage::Tofile(fstream &f) const { + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + } + this->data_to_binary(f); + } + + Storage Storage::Fromfile(const char *fname, const unsigned int &dtype, const cytnx_int64 &count, + const int device) { + return Storage::Fromfile(std::filesystem::path(fname), dtype, count, device); } - Storage Storage::Fromfile(const std::string &fname, const unsigned int &dtype, - const cytnx_int64 &count) { + Storage Storage::Fromfile(const std::filesystem::path &fname, const unsigned int &dtype, + const cytnx_int64 &count, const int device) { cytnx_error_msg(dtype == Type.Void, "[ERROR] Cannot have Void dtype.%s", "\n"); cytnx_error_msg(count == 0, "[ERROR] count cannot be zero!%s", "\n"); @@ -201,7 +440,7 @@ namespace cytnx { ifstream jf; jf.open(fname, ios::ate | ios::binary); if (!jf.is_open()) { - cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.c_str()); + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); } Nbytes = jf.tellg(); jf.close(); @@ -212,98 +451,21 @@ namespace cytnx { "[ERROR] the total size of file is not an interval of assigned dtype.%s", "\n"); // check count smaller than Nelem: - if (count < 0) - Nelem = Nbytes / Type.typeSize(dtype); - else { - cytnx_error_msg(count > Nelem, "[ERROR] count exceed the total # of elements %d in file.\n", - Nelem); + Nelem = Nbytes / Type.typeSize(dtype); // total elements in the file + if (count >= 0) { + cytnx_error_msg(static_cast(count) > Nelem, + "[ERROR] count exceed the total # of elements %d in file.\n", Nelem); Nelem = count; } f.open(fname, ios::in | ios::binary); if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.c_str()); - } - out._Loadbinary(f, dtype, Nelem); - f.close(); - return out; - } - Storage Storage::Load(const std::string &fname) { - Storage out; - fstream f; - f.open(fname, ios::in | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.c_str()); + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); } - out._Load(f); + out.data_from_binary(f, Nelem, dtype, device); f.close(); return out; } - Storage Storage::Load(const char *fname) { return Storage::Load(string(fname)); } - void Storage::_Load(fstream &f) { - // header - unsigned long long sz; - unsigned int dt; - int dv; - - // check: - cytnx_error_msg(!f.is_open(), "[ERROR] invalid fstream!.%s", "\n"); - - // checking IDD - unsigned int tmpIDDs; - f.read((char *)&tmpIDDs, sizeof(unsigned int)); - if (tmpIDDs != 999) { - cytnx_error_msg(true, "[ERROR] the Load file is not the Storage object!\n", "%s"); - } - - f.read((char *)&sz, sizeof(unsigned long long)); - f.read((char *)&dt, sizeof(unsigned int)); - f.read((char *)&dv, sizeof(int)); - - if (dv != Device.cpu) { - if (dv >= Device.Ngpus) { - cytnx_warning_msg(true, - "[Warning!!] the original device ID does not exists. the tensor will be " - "put on CPU, please use .to() or .to_() to move to desire devices.%s", - "\n"); - dv = -1; - } - } - - this->_impl = __SII.USIInit[dt](); - this->_impl->Init(sz, dv); - - // data: - if (dv == Device.cpu) { - f.read((char *)this->_impl->data(), Type.typeSize(dt) * sz); - } else { -#ifdef UNI_GPU - checkCudaErrors(cudaSetDevice(dv)); - void *htmp = malloc(Type.typeSize(dt) * sz); - f.read((char *)htmp, Type.typeSize(dt) * sz); - checkCudaErrors( - cudaMemcpy(this->_impl->data(), htmp, Type.typeSize(dt) * sz, cudaMemcpyHostToDevice)); - free(htmp); - -#else - cytnx_error_msg(true, "ERROR internal fatal error in Load Storage%s", "\n"); -#endif - } - } - - void Storage::_Loadbinary(fstream &f, const unsigned int &dtype, const cytnx_uint64 &Nelem) { - // before enter this func, makesure - // 1. dtype is not void. - // 2. the Nelement is consistent and smaller than the file size, and should not be zero! - - // check: - cytnx_error_msg(!f.is_open(), "[ERROR] invalid fstream!.%s", "\n"); - - this->_impl = __SII.USIInit[dtype](); - this->_impl->Init(Nelem, Device.cpu); - - f.read((char *)this->_impl->data(), Type.typeSize(dtype) * Nelem); - } Scalar::Sproxy Storage::operator()(const cytnx_uint64 &idx) { Scalar::Sproxy out(this->_impl, idx); diff --git a/src/backend/Tensor_impl.cpp b/src/backend/Tensor_impl.cpp index 5baefe16c..fa9aa05fb 100644 --- a/src/backend/Tensor_impl.cpp +++ b/src/backend/Tensor_impl.cpp @@ -248,7 +248,6 @@ namespace cytnx { vector get_shape(acc.size()); - // vector new_shape; std::vector> locators(this->_shape.size()); for (cytnx_uint32 i = 0; i < acc.size(); i++) { acc[i].get_len_pos(this->_shape[i], get_shape[i], locators[i]); diff --git a/src/backend/algo_internal_gpu/cuSort_internal.cuh b/src/backend/algo_internal_gpu/cuSort_internal.cuh index 266c1b2ca..afa6fd01b 100644 --- a/src/backend/algo_internal_gpu/cuSort_internal.cuh +++ b/src/backend/algo_internal_gpu/cuSort_internal.cuh @@ -11,7 +11,7 @@ namespace cytnx { namespace algo_internal { - // handle flaot2, double2 + // handle float2, double2 template concept Vec2Like = requires(T t) { t.x; diff --git a/src/io.cpp b/src/io.cpp new file mode 100644 index 000000000..1969964d9 --- /dev/null +++ b/src/io.cpp @@ -0,0 +1,368 @@ +#include "io.hpp" + +#include +#include +#include + +#include "H5Cpp.h" + +namespace cytnx { + namespace io { + + namespace internal { + H5::DataSet create_dataset(H5::DataType &datatype, const std::vector &dimensions, + H5::Group &container, const std::string &name, bool overwrite) { + H5::DataSet dataset; + // check if old dataset can be overwritten + bool create_new = true; + if (container.nameExists(name)) { // we can overwrite the data if type and size match + cytnx_error_msg(!overwrite, + "[ERROR] Dataset (or group) '%s' already exists. Use argument overwrite " + "= true to overwrite.\n", + name.c_str()); + if (container.childObjType(name) == H5O_TYPE_DATASET) { + dataset = container.openDataSet(name); + H5::DataSpace olddataspace = dataset.getSpace(); + auto rank = olddataspace.getSimpleExtentNdims(); + if (olddataspace.getSimpleExtentType() == H5S_SIMPLE && + dataset.getDataType() == datatype && rank == dimensions.size()) { + create_new = false; + std::vector olddims(rank); + olddataspace.getSimpleExtentDims(olddims.data()); + for (int i = 0; i < rank; i++) { + if (olddims[i] != dimensions[i]) { + create_new = true; + break; + } + } + } + if (!create_new) return dataset; + // otherwise, create a new dataset one now + dataset.close(); + } + container.unlink(name); + } + // create a new dataset + H5::DataSpace dataspace(dimensions.size(), dimensions.data()); + dataset = container.createDataSet(name, datatype, dataspace); + return dataset; + } + } // namespace internal + + H5::Group create_group(H5::Group &container, const std::string &path, bool recursive) { + if (recursive) { + std::filesystem::path grouppath(path); + std::filesystem::path subpath; + std::string groupfolder = "/"; + for (const auto &part : grouppath) { + if (part.empty()) continue; + subpath /= part; + groupfolder = subpath.generic_string(); // use generic_string() to avoid + // incompatibilities between file systems + if (!container.nameExists(groupfolder)) container.createGroup(groupfolder); + } + H5::Group group = container.openGroup(groupfolder); + return group; + } else { // simple version without recursion + H5::Group group; + if (container.nameExists(path)) { + group = container.openGroup(path); + } else { + group = container.createGroup(path); + } + return group; + } + } + + bool unlink(H5::Group &container, const std::string &name, bool overwrite) { + if (container.nameExists(name)) { + cytnx_error_msg( + !overwrite, + "[ERROR] Dataset or group '%s' exists. Use argument overwrite = true to remove it.\n", + name.c_str()); + container.unlink(name); + return true; + } + return false; + } + + void save_attribute(const Scalar_list &object, H5::H5Object &container, const std::string &name, + bool overwrite) { + std::visit( + [&](const auto &scalar) { + H5::DataType datatype = Type.get_hdf5_type(scalar); + if (container.attrExists(name)) { + cytnx_error_msg(!overwrite, + "[ERROR] Attribute '%s' already exists. Use argument overwrite = true " + "to overwrite.\n", + name.c_str()); + H5::Attribute oldattr = container.openAttribute(name); + if (oldattr.getSpace().getSimpleExtentType() == H5S_SCALAR && + oldattr.getDataType() == datatype) { + oldattr.write(datatype, &scalar); + return; + } // else: remove and create again + container.removeAttr(name); + } + H5::Attribute attr = container.createAttribute(name, datatype, H5::DataSpace(H5S_SCALAR)); + attr.write(datatype, &scalar); + }, + object); + } + + void save_attribute(const std::string &object, H5::H5Object &container, const std::string &name, + bool overwrite) { + H5::StrType str_type = + H5::StrType(H5::PredType::C_S1, object.length() + 1); // include NULL terminator + if (container.attrExists(name)) { + cytnx_error_msg( + !overwrite, + "[ERROR] Attribute '%s' already exists. Use argument overwrite = true to overwrite.\n", + name.c_str()); + H5::Attribute oldattr = container.openAttribute(name); + if (oldattr.getSpace().getSimpleExtentType() == H5S_SCALAR && + oldattr.getStrType() == str_type) { + oldattr.write(str_type, object); + return; + } // else: remove and create again + container.removeAttr(name); + } + H5::Attribute attr = container.createAttribute(name, str_type, H5::DataSpace(H5S_SCALAR)); + attr.write(str_type, object); + } + + void save_attribute(const std::vector &object, H5::H5Object &container, + const std::string &name, bool overwrite) { + hsize_t vecdims[1] = {object.size()}; + H5::DataSpace dataspace = H5::DataSpace(1, vecdims); + H5::StrType str_type = H5::StrType(H5::PredType::C_S1, H5T_VARIABLE); + if (container.attrExists(name)) { // simply delete old dataset + cytnx_error_msg( + !overwrite, + "[ERROR] Attribute '%s' already exists. Use argument overwrite = true to overwrite.\n", + name.c_str()); + container.removeAttr(name); + } + // create a new dataset + H5::Attribute attr = container.createAttribute(name, str_type, dataspace); + std::vector c_strings; // H5 needs cstrings + c_strings.reserve(object.size()); + for (const auto &elem : object) { + c_strings.push_back(elem.c_str()); + } + attr.write(str_type, c_strings.data()); + } + + void load_attribute(std::string &object, H5::H5Object &container, const std::string &name) { + H5::Attribute attr = container.openAttribute(name); + H5::StrType str_type = attr.getStrType(); + size_t size = str_type.getSize(); + object.resize(size); + attr.read(str_type, object.data()); + + // remove the null terminator + if (!object.empty() && object.back() == '\0') { + object.pop_back(); + } + } + + bool remove_attribute(H5::H5Object &container, const std::string &name, bool overwrite) { + if (container.attrExists(name)) { + cytnx_error_msg( + !overwrite, + "[ERROR] Attribute '%s' exists. Use argument overwrite = true to remove it.\n", + name.c_str()); + container.removeAttr(name); + return true; + } + return false; + } + + H5::DataSet save_dataset(const Vector_list &object, H5::Group &container, + const std::string &name, bool overwrite) { + H5::DataSet dataset; + std::visit( + [&](const auto &vec) { + cytnx_error_msg(vec.empty(), "[ERROR] Cannot write an empty vector to dataset '%s'!\n", + name.c_str()); + + H5::DataType datatype = Type.get_hdf5_type(vec[0]); + dataset = internal::create_dataset(datatype, {vec.size()}, container, name, overwrite); + + using Value_type = typename std::decay_t::value_type; + if constexpr (std::is_same_v) { + // H5cpp writes bool as unsigned char + std::vector flat_bools(vec.begin(), vec.end()); + dataset.write(flat_bools.data(), datatype); + } else { + dataset.write(vec.data(), datatype); + } + }, + object); + return dataset; + } + + H5::DataSet save_dataset(const std::vector &object, H5::Group &container, + const std::string &name, bool overwrite) { + hsize_t vecdims[1] = {object.size()}; + H5::DataSpace dataspace = H5::DataSpace(1, vecdims); + H5::StrType str_type = H5::StrType(H5::PredType::C_S1, H5T_VARIABLE); + if (container.nameExists(name)) { // simply delete old dataset + cytnx_error_msg( + !overwrite, + "Dataset (or group) '%s' already exists. Use argument overwrite = true to overwrite.\n", + name.c_str()); + container.unlink(name); + } + // create a new dataset + H5::DataSet dataset = container.createDataSet(name, str_type, dataspace); + std::vector c_strings; // H5 needs cstrings + c_strings.reserve(object.size()); + for (const auto &elem : object) { + c_strings.push_back(elem.c_str()); + } + dataset.write(c_strings.data(), str_type); + return dataset; + } + + H5::DataSet save_dataset(const Matrix_list &object, H5::Group &container, + const std::string &name, bool overwrite) { + H5::DataSet dataset; + std::visit( + [&](const auto &vec) { + cytnx_error_msg(vec.empty() || vec[0].empty(), + "[ERROR] Cannot write an empty vector to dataset '%s'!\n", name.c_str()); + hsize_t rownum = vec.size(); + hsize_t colnum = vec[0].size(); + for (hsize_t i = 1; i < vec.size(); ++i) { + cytnx_error_msg(vec[i].size() != colnum, + "[ERROR] object[%zu] has different size than object[0]. A rectanglular " + "matrix is expected as input!\n", + i); + } + + H5::DataType datatype = Type.get_hdf5_type(vec[0][0]); + dataset = + internal::create_dataset(datatype, {rownum, colnum}, container, name, overwrite); + + using Inner_vec_type = typename std::decay_t::value_type; + using Scalar_type = typename Inner_vec_type::value_type; + if constexpr (std::is_same_v) { + std::vector flat(rownum * colnum); // flatten vector + for (hsize_t i = 0; i < vec.size(); ++i) { + std::copy(vec[i].begin(), vec[i].end(), flat.begin() + i * colnum); + } + dataset.write(flat.data(), datatype); + } else { + std::vector flat(rownum * colnum); // flatten vector + for (hsize_t i = 0; i < vec.size(); ++i) { + std::copy(vec[i].begin(), vec[i].end(), flat.begin() + i * colnum); + } + dataset.write(flat.data(), datatype); + } + }, + object); + return dataset; + } + + void load_dataset(std::vector &object, H5::H5Object &container, + const std::string &name) { + H5::DataSet dataset = container.openDataSet(name); + H5::DataSpace dataspace = dataset.getSpace(); + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims); + // H5T_VARIABLE requires reading into an array of char pointers (char**) + H5::StrType str_type(H5::PredType::C_S1, H5T_VARIABLE); + std::vector c_strings(dims[0]); + dataset.read(c_strings.data(), str_type); + object.reserve(dims[0]); + for (size_t i = 0; i < dims[0]; ++i) { + object.push_back(std::string(c_strings[i])); + } + // free the space of each char* that was allocated in dataset.read() + dataset.vlenReclaim(c_strings.data(), str_type, dataspace); + } + + H5::H5File open(const std::filesystem::path &fname, IoMode mode) { + H5::H5File h5file; + + // Enable reuse of space after data is deleted; + // Set the strategy: FSM_AGGR is standard for free-space management + // Parameters: strategy, persist (true), threshold (default 1: track all free-space + // sections) + H5::FileCreatPropList fcpl; + fcpl.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_FSM_AGGR, true, 1); + + // Persistent free space requires HDF5 1.10.x format or later + H5::FileAccPropList fapl; + fapl.setLibverBounds(H5F_LIBVER_V110, H5F_LIBVER_LATEST); + + // open file + switch (mode) { + case ACC_TRUNC: + h5file = H5::H5File(fname, H5F_ACC_TRUNC, fcpl, fapl); + break; + case ACC_NOREPLACE: + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + break; + case ACC_IN: + h5file = H5::H5File(fname, H5F_ACC_RDONLY, fcpl, fapl); + break; + case ACC_INOUT: + if (std::filesystem::exists(fname)) { + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + } else { + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } + break; + default: + cytnx_error_msg(true, "[ERROR] Unknown mode '%d' for writing to HDF5 file.", mode); + } + + return h5file; + } + + // void close(H5::H5File &container) { + // container.close(); + // } + + void Save(const savable_class &object, H5::Group &container, const std::string &name, + const std::string &path, bool overwrite) { + std::visit( + [&](const auto &concreteObj) { + H5::Group group = create_group(container, path); + concreteObj.to_hdf5(group, name, overwrite); + }, + object); + } + + void Load(savable_class &object, H5::Group &container, const std::string &name, + const std::string &path) { + std::visit( + [&](auto &concreteObj) { + if (path.empty()) { + concreteObj.from_hdf5(container, name); + } else { + H5::Group group = container.openGroup(path); + concreteObj.from_hdf5(group, name); + } + }, + object); + } + + void Load(loadable_to_device &object, H5::Group &container, const std::string &name, + const std::string &path, bool restore_device) { + std::visit( + [&](auto &concreteObj) { + if (path.empty()) { + concreteObj.from_hdf5(container, name, restore_device); + } else { + H5::Group group = container.openGroup(path); + concreteObj.from_hdf5(group, name, restore_device); + } + }, + object); + } + + } // namespace io +} // namespace cytnx diff --git a/src/tn_algo/MPS.cpp b/src/tn_algo/MPS.cpp index e94e360c6..98e1d16fa 100644 --- a/src/tn_algo/MPS.cpp +++ b/src/tn_algo/MPS.cpp @@ -4,6 +4,8 @@ #include #include +#include "H5Cpp.h" + using namespace std; #ifdef BACKEND_TORCH @@ -12,38 +14,180 @@ using namespace std; namespace cytnx { namespace tn_algo { - std::ostream& operator<<(std::ostream& os, const MPS& in) { + std::ostream &operator<<(std::ostream &os, const MPS &in) { in._impl->Print(os); return os; } - void MPS::_Save(std::fstream& f) const { - cytnx_error_msg(!f.is_open(), "[ERROR][MPS] invalid fstream!.%s", "\n"); + void MPS::Save(const std::filesystem::path &fname, const std::string &path, + const char mode) const { + fstream f; // only for binary saving, not used for HDF5 + if (fname.has_extension()) { + // filename extension is given + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { + // save as HDF5 + H5::H5File h5file; + // Enable reuse of space after data is deleted; + // Set the strategy: FSM_AGGR is standard for free-space management + // Parameters: strategy, persist (true), threshold (default 1: track all free-space + // sections) + H5::FileCreatPropList fcpl; + fcpl.setFileSpaceStrategy(H5F_FSPACE_STRATEGY_FSM_AGGR, true, 1); + // Persistent free space requires HDF5 1.10.x format or later + H5::FileAccPropList fapl; + fapl.setLibverBounds(H5F_LIBVER_V110, H5F_LIBVER_LATEST); + // open file + bool overwrite = false; + if (mode == 'w') { // Write new file + h5file = H5::H5File(fname, H5F_ACC_TRUNC, fcpl, fapl); + } else if (mode == 'x') { // eXclusive create + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'a') { // Append data + if (std::filesystem::exists(fname)) + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + else + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } else if (mode == 'u') { // Update data + if (std::filesystem::exists(fname)) { + h5file = H5::H5File(fname, H5F_ACC_RDWR, H5::FileCreatPropList::DEFAULT, fapl); + overwrite = true; + } else { + h5file = H5::H5File(fname, H5F_ACC_EXCL, fcpl, fapl); + } + } else { + cytnx_error_msg(true, "[ERROR] Unknown mode '%c' for writing to HDF5 file.", mode); + } + // create group + std::filesystem::path grouppath(path); + std::filesystem::path subpath; + std::string groupfolder = "/"; + for (const auto &part : grouppath) { + if (part.empty()) continue; + subpath /= part; + groupfolder = subpath.generic_string(); + if (!h5file.nameExists(groupfolder)) h5file.createGroup(groupfolder); + } + H5::Group group = h5file.openGroup(groupfolder); + // write data + this->to_hdf5(group, "MPS", overwrite); + h5file.close(); + return; + } else { // create binary file + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fname), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fname.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", + mode); + } + f.open(fname, std::ios::out | std::ios::trunc | std::ios::binary); + } + } else { // create binary file with standard extension + std::filesystem::path fnameext = fname; + fnameext += ".cymps"; + cytnx_warning_msg( + true, + "Missing file extension in fname '%s'. I am adding the extension '.cymps'. This is " + "deprecated, please provide the file extension in the future.\n", + fname.string().c_str()); + if (mode == 'x') { + cytnx_error_msg(std::filesystem::exists(fnameext), + "[ERROR] File %s already exists. Use mode 'w' to overwrite.", + fnameext.string().c_str()); + } else { + cytnx_error_msg(mode != 'w', "[ERROR] Unknown mode '%c' for writing to binary file.", + mode); + } + f.open(fnameext, std::ios::out | std::ios::trunc | std::ios::binary); + } + // write binary + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); + } + this->to_binary(f); + f.close(); + } + void MPS::Save(const char *fname, const std::string &path, const char mode) const { + this->Save(std::filesystem::path(fname), path, mode); + } + + MPS MPS::Load(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + MPS out; + out.Load_(fname, path, restore_device); + return out; + } + MPS MPS::Load(const char *fname, const std::string &path, bool restore_device) { + return MPS::Load(std::filesystem::path(fname), path, restore_device); + } + + void MPS::Load_(const std::filesystem::path &fname, const std::string &path, + bool restore_device) { + std::string ext = fname.extension().string(); + if (ext == ".h5" || ext == ".hdf5" || ext == ".H5" || ext == ".HDF5" || ext == ".hdf" || + ext == ".HDF") { // load HDF5 + H5::H5File h5file(fname, H5F_ACC_RDONLY); + // open group + H5::Group group; + try { + group = h5file.openGroup(path.empty() ? "/" : path); + } catch (const H5::Exception &e) { + std::cerr << e.getDetailMsg() << std::endl; + cytnx_error_msg(true, "[ERROR] HDF5 path '%s' not found or is not a group in file '%s'.", + path.c_str(), fname.string().c_str()); + } + // read data + this->from_hdf5(group, "", restore_device); + h5file.close(); + } else { // load binary + fstream f; + f.open(fname, std::ios::in | std::ios::binary); + if (!f.is_open()) { + cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.string().c_str()); + } + this->from_binary(f, restore_device); + f.close(); + } + } + void MPS::Load_(const char *fname, const std::string &path, bool restore_device) { + this->Load_(std::filesystem::path(fname), path, restore_device); + } + + void MPS::to_hdf5(H5::Group &container, const std::string &name, const bool overwrite) const { + cytnx_error_msg(true, "[ERROR] Saving MPS to HDF5 is not implemented yet!%s", "\n"); + } + void MPS::from_hdf5(H5::Group &container, const std::string &name, bool restore_device) { + cytnx_error_msg(true, "[ERROR] Loading MPS from HDF5 is not implemented yet!%s", "\n"); + } + + void MPS::to_binary(std::ostream &f) const { cytnx_error_msg(this->_impl->mps_type_id == MPSType.Void, "[ERROR][MPS] Cannot save an uninitialize MPS.%s", "\n"); unsigned int IDDs = 109; - f.write((char*)&IDDs, sizeof(unsigned int)); + f.write((char *)&IDDs, sizeof(unsigned int)); - f.write((char*)&this->_impl->mps_type_id, + f.write((char *)&this->_impl->mps_type_id, sizeof(int)); // mps type, this is used to determine Regular/iMPS upon load // first, save common meta data: - f.write((char*)&this->_impl->virt_dim, sizeof(this->_impl->virt_dim)); - f.write((char*)&this->_impl->S_loc, sizeof(this->_impl->S_loc)); + f.write((char *)&this->_impl->virt_dim, sizeof(this->_impl->virt_dim)); + f.write((char *)&this->_impl->S_loc, sizeof(this->_impl->S_loc)); // second, dispatch to do remaining saving: - this->_impl->_save_dispatch(f); + this->_impl->to_binary_dispatch(f); } - void MPS::_Load(std::fstream& f) { - cytnx_error_msg(!f.is_open(), "[ERROR][MPS] invalid fstream%s", "\n"); + void MPS::from_binary(std::istream &f, bool restore_device) { unsigned int tmpIDDs; - f.read((char*)&tmpIDDs, sizeof(unsigned int)); + f.read((char *)&tmpIDDs, sizeof(unsigned int)); cytnx_error_msg(tmpIDDs != 109, "[ERROR] the object is not a cytnx MPS!%s", "\n"); int mpstype; - f.read((char*)&mpstype, + f.read((char *)&mpstype, sizeof(int)); // mps type, this is used to determine Sparse/Dense upon load if (mpstype == MPSType.RegularMPS) { @@ -55,57 +199,12 @@ namespace cytnx { } // first, load common meta data: - f.read((char*)&this->_impl->virt_dim, sizeof(this->_impl->virt_dim)); - f.read((char*)&this->_impl->S_loc, sizeof(this->_impl->S_loc)); + f.read((char *)&this->_impl->virt_dim, sizeof(this->_impl->virt_dim)); + f.read((char *)&this->_impl->S_loc, sizeof(this->_impl->S_loc)); // second, let dispatch to do remaining loading. - this->_impl->_load_dispatch(f); + this->_impl->from_binary_dispatch(f, restore_device); } - - void MPS::Save(const std::string& fname) const { - fstream f; - if (std::filesystem::path(fname).has_extension()) { - // filename extension is given - f.open(fname, ios::out | ios::trunc | ios::binary); - } else { - // add filename extension - cytnx_warning_msg( - true, - "Missing file extension in fname '%s'. I am adding the extension '.cymps'. This is " - "deprecated, please provide the file extension in the future.\n", - fname.c_str()); - f.open((fname + ".cymps"), ios::out | ios::trunc | ios::binary); - } - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); - } - this->_Save(f); - f.close(); - } - void MPS::Save(const char* fname) const { - fstream f; - string ffname = string(fname) + ".cymps"; - f.open((ffname), ios::out | ios::trunc | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] invalid file path for save.%s", "\n"); - } - this->_Save(f); - f.close(); - } - - MPS MPS::Load(const std::string& fname) { - MPS out; - fstream f; - f.open(fname, ios::in | ios::binary); - if (!f.is_open()) { - cytnx_error_msg(true, "[ERROR] Cannot open file '%s'.\n", fname.c_str()); - } - out._Load(f); - f.close(); - return out; - } - MPS MPS::Load(const char* fname) { return MPS::Load(string(fname)); } - } // namespace tn_algo } // namespace cytnx diff --git a/src/tn_algo/MPS_base.cpp b/src/tn_algo/MPS_base.cpp index 820d89c65..b305fd966 100644 --- a/src/tn_algo/MPS_base.cpp +++ b/src/tn_algo/MPS_base.cpp @@ -74,12 +74,12 @@ namespace cytnx { true, "[ERROR] MPS_Base should not be called. Please initialize the MPS first.%s", "\n"); } - void MPS_impl::_save_dispatch(fstream &f) { + void MPS_impl::to_binary_dispatch(std::ostream &f) { cytnx_error_msg( true, "[ERROR] MPS_Base should not be called. Please initialize the MPS first.%s", "\n"); } - void MPS_impl::_load_dispatch(fstream &f) { + void MPS_impl::from_binary_dispatch(std::istream &f, bool restore_device) { cytnx_error_msg( true, "[ERROR] MPS_Base should not be called. Please initialize the MPS first.%s", "\n"); } diff --git a/src/tn_algo/RegularMPS.cpp b/src/tn_algo/RegularMPS.cpp index 1459fe253..940783dae 100644 --- a/src/tn_algo/RegularMPS.cpp +++ b/src/tn_algo/RegularMPS.cpp @@ -261,16 +261,16 @@ namespace cytnx { } } - void RegularMPS::_save_dispatch(fstream &f) { + void RegularMPS::to_binary_dispatch(std::ostream &f) { cytnx_uint64 N = this->_TNs.size(); f.write((char *)&N, sizeof(cytnx_uint64)); // save UniTensor one by one: for (cytnx_uint64 i = 0; i < N; i++) { - this->_TNs[i]._Save(f); + this->_TNs[i].to_binary(f); } } - void RegularMPS::_load_dispatch(fstream &f) { + void RegularMPS::from_binary_dispatch(std::istream &f, bool restore_device) { cytnx_uint64 N; f.read((char *)&N, sizeof(cytnx_uint64)); @@ -278,7 +278,7 @@ namespace cytnx { // Load UniTensor one by one: for (cytnx_uint64 i = 0; i < N; i++) { - this->_TNs[i]._Load(f); + this->_TNs[i].from_binary(f, restore_device); } } diff --git a/src/tn_algo/iMPS.cpp b/src/tn_algo/iMPS.cpp index adb52275e..9492dfa04 100644 --- a/src/tn_algo/iMPS.cpp +++ b/src/tn_algo/iMPS.cpp @@ -60,16 +60,16 @@ namespace cytnx { } } - void iMPS::_save_dispatch(fstream &f) { + void iMPS::to_binary_dispatch(std::ostream &f) { cytnx_uint64 N = this->_TNs.size(); f.write((char *)&N, sizeof(cytnx_uint64)); // save UniTensor one by one: for (cytnx_uint64 i = 0; i < N; i++) { - this->_TNs[i]._Save(f); + this->_TNs[i].to_binary(f); } } - void iMPS::_load_dispatch(fstream &f) { + void iMPS::from_binary_dispatch(std::istream &f, bool restore_device) { cytnx_uint64 N; f.read((char *)&N, sizeof(cytnx_uint64)); @@ -77,7 +77,7 @@ namespace cytnx { // Load UniTensor one by one: for (cytnx_uint64 i = 0; i < N; i++) { - this->_TNs[i]._Load(f); + this->_TNs[i].from_binary(f, restore_device); } } diff --git a/tests/Bond_test.cpp b/tests/Bond_test.cpp index bb686d83f..85f266415 100644 --- a/tests/Bond_test.cpp +++ b/tests/Bond_test.cpp @@ -209,6 +209,32 @@ TEST(Bond, Clear_type) { EXPECT_THROW(bd_sym.set_type(BD_REG), std::logic_error); } +// In-place Load_ must not inherit stale qnums/degs/syms from a previously Bond +TEST(Bond, Load_ResetsStaleMetadata) { + const std::string fpath = std::string(std::tmpnam(nullptr)) + ".cytnx"; + + // 1) save a plain non-symmetric Bond (no qnums/degs/syms) + Bond bd_plain(5); + bd_plain.Save(fpath); + + // 2) start with a symmetric Bond that has non-empty _qnums, _degs, _syms + Bond bd_sym = Bond(BD_KET, {{0}, {1}, {2}}, {3, 3, 3}); + EXPECT_FALSE(bd_sym.qnums().empty()); + EXPECT_FALSE(bd_sym.getDegeneracies().empty()); + EXPECT_NE(bd_sym.Nsym(), 0); + + // 3) Load_ the plain payload into the symmetric Bond; the result must equal the saved Bond. + bd_sym.Load_(fpath); + EXPECT_EQ(bd_sym, bd_plain) << "Load_ should fully reconstruct the saved Bond"; + EXPECT_EQ(bd_sym.dim(), 5); + EXPECT_EQ(bd_sym.type(), BD_REG); + EXPECT_EQ(bd_sym.Nsym(), 0); + EXPECT_TRUE(bd_sym.qnums().empty()) << "stale qnums leaked into a non-symmetric Bond"; + EXPECT_TRUE(bd_sym.getDegeneracies().empty()) << "stale degs leaked into a non-symmetric Bond"; + + std::filesystem::remove(fpath); +} + // TEST(Bond, ConstructorTypeQnums){ // // Bond(bondType tp, const std::vector& qnums); // diff --git a/tests/Bond_test.h b/tests/Bond_test.h index 8625be037..d843d3877 100644 --- a/tests/Bond_test.h +++ b/tests/Bond_test.h @@ -2,6 +2,7 @@ // #ifndef __CYTNX_TEST_BOND_H__ // #define __CYTNX_TEST_BOND_H__ #include +#include #include #include #include diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a90f537ce..ef66f5836 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,6 +49,11 @@ add_executable( algo_test/Vsplit_test.cpp algo_test/Vstack_test.cpp random_test/uniform_test.cpp + io_test/Storage_test.cpp + io_test/Tensor_test.cpp + io_test/Bond_test.cpp + io_test/Symmetry_test.cpp + io_test/UniTensor_test.cpp ) if(USE_CUDA) diff --git a/tests/DenseUniTensor_test.cpp b/tests/DenseUniTensor_test.cpp index 348d00ed8..d74c8a45a 100644 --- a/tests/DenseUniTensor_test.cpp +++ b/tests/DenseUniTensor_test.cpp @@ -343,13 +343,15 @@ TEST_F(DenseUniTensorTest, dtype_str_uninit) { EXPECT_ANY_THROW(ut_uninit.dtype_ /*=====test info===== describe:test uten_type_str for dense tensor. ====================*/ -TEST_F(DenseUniTensorTest, uten_type_str) { EXPECT_EQ(utzero345.uten_type_str(), "Dense"); } +TEST_F(DenseUniTensorTest, uten_type_str) { + EXPECT_EQ(utzero345.uten_type_str(), "DenseUniTensor"); +} /*=====test info===== describe:test uten_type for uninitialized tensor. ====================*/ TEST_F(DenseUniTensorTest, uten_type_str_uninit) { - EXPECT_EQ(ut_uninit.uten_type_str(), "Void (un-initialize UniTensor)"); + EXPECT_EQ(ut_uninit.uten_type_str(), "Void (uninitialized UniTensor)"); } TEST_F(DenseUniTensorTest, device_str) { EXPECT_EQ(Spf.device_str(), "cytnx device: CPU"); } @@ -4894,6 +4896,29 @@ describe:test Save uninitialized UniTensor ====================*/ TEST_F(DenseUniTensorTest, Save_uninit) { EXPECT_ANY_THROW(ut_uninit.Save(temp_file_path)); } +/*=====test info===== +describe:In-place Load_ must not inherit a stale name from a previous UniTensor. +====================*/ +TEST_F(DenseUniTensorTest, Load_ResetsStaleName) { + auto row_rank = 1u; + std::vector bonds = {Bond(3), Bond(2)}; + + // 1) save a UniTensor with an empty name + UniTensor ut_anon = UniTensor(bonds, {"a", "b"}, row_rank); + ASSERT_TRUE(ut_anon.name().empty()); + ut_anon.Save(temp_file_path); + + // 2) build a UniTensor with a non-empty name, then Load_ the anonymous payload into it. + UniTensor ut_named = UniTensor(bonds, {"a", "b"}, row_rank); + ut_named.set_name("stale_name"); + ASSERT_EQ(ut_named.name(), "stale_name"); + ut_named.Load_(temp_file_path); + EXPECT_TRUE(ut_named.name().empty()) + << "stale UniTensor name leaked when loading a payload with empty name"; + EXPECT_TRUE(AreEqUniTensor(ut_named, ut_anon)) + << "Load_ should fully reconstruct the saved UniTensor"; +} + /*=====test info===== describe:test truncate by label ====================*/ diff --git a/tests/io_test/Bond_test.cpp b/tests/io_test/Bond_test.cpp new file mode 100644 index 000000000..0b857f447 --- /dev/null +++ b/tests/io_test/Bond_test.cpp @@ -0,0 +1,92 @@ +#include "io_test_tools.h" + +using namespace cytnx; +using namespace cytnx::IOTest; + +// A plain (untagged, no-symmetry) bond. +TEST(IOBondTest, RoundTripRegular) { + Bond b = Bond(7); + Bond loaded = RoundTrip(b); + EXPECT_TRUE(loaded == b); +} + +// Directional (tagged) bonds without symmetry. +TEST(IOBondTest, RoundTripDirectional) { + for (auto bt : {BD_IN, BD_OUT}) { + Bond b = Bond(5, bt); + Bond loaded = RoundTrip(b); + EXPECT_TRUE(loaded == b); + } +} + +// A single-U1-symmetry bond. +TEST(IOBondTest, RoundTripU1) { + Bond b = Bond(BD_IN, {Qs(0) >> 1, Qs(1) >> 2, Qs(-1) >> 3}); + Bond loaded = RoundTrip(b); + EXPECT_TRUE(loaded == b); +} + +// A bond carrying multiple symmetries (U1 x Z2). +TEST(IOBondTest, RoundTripMultiSymmetry) { + Bond b = + Bond(BD_KET, {{0, 2}, {1, 5}, {1, 6}, {0, 1}}, {4, 7, 2, 3}, {Symmetry::Zn(2), Symmetry::U1()}); + Bond loaded = RoundTrip(b); + EXPECT_TRUE(loaded == b); +} + +// Bonds carrying a fermionic-parity symmetry. +TEST(IOBondTest, RoundTripFermionic) { + Bond b = Bond(BD_IN, {Qs(0) >> 2, Qs(1) >> 3}, {Symmetry::FermionParity()}); + Bond loaded = RoundTrip(b); + EXPECT_TRUE(loaded == b); +} + +// Saving the same name twice must fail without overwrite and succeed with it. +TEST(IOBondTest, Overwrite) { + TempH5File tmp; + Bond a = Bond(BD_IN, {Qs(0) >> 1, Qs(1) >> 2}); + Bond b = Bond(BD_OUT, {Qs(0) >> 2, Qs(2) >> 3, Qs(-1) >> 1}); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + Bond loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_TRUE(loaded == b); +} + +// Store under a nested group path and read it back. +TEST(IOBondTest, NestedPath) { + TempH5File tmp; + Bond b = Bond(BD_IN, {Qs(0) >> 2, Qs(1) >> 2}); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(b, file, "obj", "/bonds/a"); + file.close(); + } + Bond loaded = LoadFromFile(tmp.str(), "obj", "/bonds/a"); + EXPECT_TRUE(loaded == b); +} + +// Multiple bonds coexist in one file under different names. +TEST(IOBondTest, MultipleObjectsInOneFile) { + TempH5File tmp; + Bond b1 = Bond(4); + Bond b2 = Bond(BD_IN, {Qs(0) >> 1, Qs(1) >> 1}); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(b1, file, "first"); + io::Save(b2, file, "second"); + file.close(); + } + EXPECT_TRUE(LoadFromFile(tmp.str(), "first") == b1); + EXPECT_TRUE(LoadFromFile(tmp.str(), "second") == b2); +} + +// Load committed reference files (regression coverage for the on-disk format). +TEST(IOBondTest, LoadReference) { + EXPECT_TRUE(LoadFromFile(ref_data_dir() + "bond.h5") == ref::bond()); + EXPECT_TRUE(LoadFromFile(ref_data_dir() + "bond_fermionic.h5") == ref::bond_fermionic()); +} diff --git a/tests/io_test/Storage_test.cpp b/tests/io_test/Storage_test.cpp new file mode 100644 index 000000000..5165c928d --- /dev/null +++ b/tests/io_test/Storage_test.cpp @@ -0,0 +1,142 @@ +#include "io_test_tools.h" + +#include "test_tools.h" + +using namespace cytnx; +using namespace cytnx::IOTest; + +namespace { + + // Build a deterministic Storage of the requested dtype. + Storage MakeStorage(unsigned int dtype, cytnx_uint64 n = 12) { + std::vector v(n); + for (cytnx_uint64 i = 0; i < n; ++i) v[i] = static_cast(i % 5); + return Storage(v).astype(dtype); + } + +} // namespace + +// Round-trip a Storage of every supported dtype through the io module. +TEST(IOStorageTest, RoundTripAllDtypes) { + for (auto dtype : TestTools::dtype_list) { + Storage s = MakeStorage(dtype); + Storage loaded = RoundTrip(s); + EXPECT_EQ(loaded.dtype(), s.dtype()) << "dtype: " << Type.getname(dtype); + EXPECT_TRUE(loaded == s) << "dtype: " << Type.getname(dtype); + } +} + +// Saving the same name twice must fail without overwrite and succeed with it. +TEST(IOStorageTest, Overwrite) { + TempH5File tmp; + Storage a = MakeStorage(Type.Double); + Storage b = MakeStorage(Type.Double, 20); // different size -> dataset gets recreated + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + Storage loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_TRUE(loaded == b); +} + +// Overwrite with identical type and shape reuses the existing dataset. +TEST(IOStorageTest, OverwriteSameShape) { + TempH5File tmp; + Storage a = MakeStorage(Type.Double); + Storage b = MakeStorage(Type.Double); + b.fill(7.0); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + Storage loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_TRUE(loaded == b); +} + +// Overwrite must replace data stored with a different dtype (dataset recreated). +TEST(IOStorageTest, OverwriteDifferentDtype) { + TempH5File tmp; + Storage a = MakeStorage(Type.Double); + Storage b = MakeStorage(Type.Int64, 7); // different dtype and size + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + Storage loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_EQ(loaded.dtype(), Type.Int64); + EXPECT_TRUE(loaded == b); +} + +// Re-saving the identical object with overwrite=true keeps the data intact. +TEST(IOStorageTest, OverwriteIdentical) { + TempH5File tmp; + Storage a = MakeStorage(Type.Double); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_NO_THROW(io::Save(a, file, "obj", "", true)); + file.close(); + } + EXPECT_TRUE(LoadFromFile(tmp.str(), "obj") == a); +} + +// Objects can be stored under a nested group path and read back from it. +TEST(IOStorageTest, NestedPath) { + TempH5File tmp; + Storage s = MakeStorage(Type.Int64); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(s, file, "obj", "/group/sub"); + file.close(); + } + Storage loaded = LoadFromFile(tmp.str(), "obj", "/group/sub"); + EXPECT_TRUE(loaded == s); +} + +// Several independent objects can live in the same file under different names. +TEST(IOStorageTest, MultipleObjectsInOneFile) { + TempH5File tmp; + Storage s1 = MakeStorage(Type.Double); + Storage s2 = MakeStorage(Type.ComplexDouble, 8); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(s1, file, "first"); + io::Save(s2, file, "second"); + file.close(); + } + Storage l1 = LoadFromFile(tmp.str(), "first"); + Storage l2 = LoadFromFile(tmp.str(), "second"); + EXPECT_TRUE(l1 == s1); + EXPECT_TRUE(l2 == s2); +} + +// The device-restoring Load overload keeps CPU data on the CPU either way. +TEST(IOStorageTest, LoadDeviceRestore) { + TempH5File tmp; + Storage s = MakeStorage(Type.Float); + SaveToFile(s, tmp.str()); + + Storage keep = LoadFromFileDevice(tmp.str(), true); + EXPECT_EQ(keep.device(), Device.cpu); + EXPECT_TRUE(keep == s); + + Storage cpu = LoadFromFileDevice(tmp.str(), false); + EXPECT_EQ(cpu.device(), Device.cpu); + EXPECT_TRUE(cpu == s); +} + +// Load committed reference file (regression coverage for the on-disk format). +TEST(IOStorageTest, LoadReference) { + Storage loaded = LoadFromFile(ref_data_dir() + "storage.h5"); + Storage expected = ref::storage(); + EXPECT_EQ(loaded.dtype(), expected.dtype()); + EXPECT_TRUE(loaded == expected); +} diff --git a/tests/io_test/Symmetry_test.cpp b/tests/io_test/Symmetry_test.cpp new file mode 100644 index 000000000..50f9735f4 --- /dev/null +++ b/tests/io_test/Symmetry_test.cpp @@ -0,0 +1,77 @@ +#include "io_test_tools.h" + +using namespace cytnx; +using namespace cytnx::IOTest; + +// Round-trip the U1 and Zn symmetry generators. +TEST(IOSymmetryTest, RoundTripU1) { + Symmetry s = Symmetry::U1(); + Symmetry loaded = RoundTrip(s); + EXPECT_TRUE(loaded == s); +} + +TEST(IOSymmetryTest, RoundTripZn) { + for (int n : {2, 3, 5}) { + Symmetry s = Symmetry::Zn(n); + Symmetry loaded = RoundTrip(s); + EXPECT_TRUE(loaded == s) << "Zn n=" << n; + } +} + +// Round-trip the fermionic symmetry generators. +TEST(IOSymmetryTest, RoundTripFermionic) { + for (Symmetry s : {Symmetry::FermionParity(), Symmetry::FermionNumber()}) { + Symmetry loaded = RoundTrip(s); + EXPECT_TRUE(loaded == s); + } +} + +// Saving the same name twice must fail without overwrite and succeed with it. +TEST(IOSymmetryTest, Overwrite) { + TempH5File tmp; + Symmetry a = Symmetry::U1(); + Symmetry b = Symmetry::Zn(2); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + Symmetry loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_TRUE(loaded == b); +} + +// Store under a nested group path and read it back. +TEST(IOSymmetryTest, NestedPath) { + TempH5File tmp; + Symmetry s = Symmetry::Zn(4); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(s, file, "obj", "/sym/inner"); + file.close(); + } + Symmetry loaded = LoadFromFile(tmp.str(), "obj", "/sym/inner"); + EXPECT_TRUE(loaded == s); +} + +// Multiple symmetries coexist in one file under different names. +TEST(IOSymmetryTest, MultipleObjectsInOneFile) { + TempH5File tmp; + Symmetry s1 = Symmetry::U1(); + Symmetry s2 = Symmetry::Zn(3); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(s1, file, "first"); + io::Save(s2, file, "second"); + file.close(); + } + EXPECT_TRUE(LoadFromFile(tmp.str(), "first") == s1); + EXPECT_TRUE(LoadFromFile(tmp.str(), "second") == s2); +} + +// Load committed reference files (regression coverage for the on-disk format). +TEST(IOSymmetryTest, LoadReference) { + EXPECT_TRUE(LoadFromFile(ref_data_dir() + "symmetry.h5") == ref::symmetry()); + EXPECT_TRUE(LoadFromFile(ref_data_dir() + "symmetry_fpar.h5") == ref::symmetry_fpar()); +} diff --git a/tests/io_test/Tensor_test.cpp b/tests/io_test/Tensor_test.cpp new file mode 100644 index 000000000..da7c4a9f0 --- /dev/null +++ b/tests/io_test/Tensor_test.cpp @@ -0,0 +1,143 @@ +#include "io_test_tools.h" + +#include "test_tools.h" + +using namespace cytnx; +using namespace cytnx::IOTest; +using namespace cytnx::TestTools; + +// Round-trip a Tensor of every supported dtype. +TEST(IOTensorTest, RoundTripAllDtypes) { + for (auto dtype : dtype_list) { + Tensor t = Tensor({3, 4, 2}, dtype); + InitTensorUniform(t, /*seed=*/dtype); + Tensor loaded = RoundTrip(t); + EXPECT_TRUE(AreEqTensor(loaded, t)) << "dtype: " << Type.getname(dtype); + } +} + +// Round-trip tensors of several ranks/shapes. +TEST(IOTensorTest, RoundTripShapes) { + std::vector> shapes = {{6}, {3, 4}, {2, 3, 4}, {2, 2, 2, 2}}; + for (const auto& shape : shapes) { + Tensor t = Tensor(shape, Type.Double); + InitTensorUniform(t, 7); + Tensor loaded = RoundTrip(t); + EXPECT_TRUE(AreEqTensor(loaded, t)); + } +} + +// A non-contiguous (permuted) tensor must round-trip to its logical (contiguous) values. +TEST(IOTensorTest, RoundTripNonContiguous) { + Tensor t = Tensor({3, 4, 5}, Type.ComplexDouble); + InitTensorUniform(t, 11); + t.permute_({2, 0, 1}); + ASSERT_FALSE(t.is_contiguous()); + Tensor loaded = RoundTrip(t); + EXPECT_TRUE(loaded.is_contiguous()); + EXPECT_TRUE(AreEqTensor(loaded, t.contiguous())); +} + +// Saving the same name twice must fail without overwrite and succeed with it. +TEST(IOTensorTest, Overwrite) { + TempH5File tmp; + Tensor a = Tensor({3, 4}, Type.Double); + InitTensorUniform(a, 1); + Tensor b = Tensor({2, 5}, Type.Double); // different shape -> dataset recreated + InitTensorUniform(b, 2); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + Tensor loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_TRUE(AreEqTensor(loaded, b)); +} + +// Overwrite with identical shape/dtype but different values replaces the data. +TEST(IOTensorTest, OverwriteSameShape) { + TempH5File tmp; + Tensor a = Tensor({3, 4}, Type.Double); + Tensor b = Tensor({3, 4}, Type.Double); + InitTensorUniform(a, 1); + InitTensorUniform(b, 99); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + EXPECT_TRUE(AreEqTensor(LoadFromFile(tmp.str(), "obj"), b)); +} + +// Overwrite must replace data stored with a different dtype (dataset recreated). +TEST(IOTensorTest, OverwriteDifferentDtype) { + TempH5File tmp; + Tensor a = Tensor({3, 4}, Type.Double); + Tensor b = Tensor({3, 4}, Type.ComplexDouble); + InitTensorUniform(a, 1); + InitTensorUniform(b, 2); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + EXPECT_TRUE(AreEqTensor(LoadFromFile(tmp.str(), "obj"), b)); +} + +// Store under a nested group path and read it back. +TEST(IOTensorTest, NestedPath) { + TempH5File tmp; + Tensor t = Tensor({4, 4}, Type.Float); + InitTensorUniform(t, 3); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(t, file, "obj", "/deeply/nested/group"); + file.close(); + } + Tensor loaded = LoadFromFile(tmp.str(), "obj", "/deeply/nested/group"); + EXPECT_TRUE(AreEqTensor(loaded, t)); +} + +// Multiple tensors coexist in one file under different names. +TEST(IOTensorTest, MultipleObjectsInOneFile) { + TempH5File tmp; + Tensor t1 = Tensor({3, 3}, Type.Double); + Tensor t2 = Tensor({2, 4}, Type.Int32); + InitTensorUniform(t1, 4); + InitTensorUniform(t2, 5); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(t1, file, "alpha"); + io::Save(t2, file, "beta"); + file.close(); + } + EXPECT_TRUE(AreEqTensor(LoadFromFile(tmp.str(), "alpha"), t1)); + EXPECT_TRUE(AreEqTensor(LoadFromFile(tmp.str(), "beta"), t2)); +} + +// The device-restoring Load overload keeps CPU data on the CPU either way. +TEST(IOTensorTest, LoadDeviceRestore) { + TempH5File tmp; + Tensor t = Tensor({3, 4}, Type.Double); + InitTensorUniform(t, 6); + SaveToFile(t, tmp.str()); + + Tensor keep = LoadFromFileDevice(tmp.str(), true); + EXPECT_EQ(keep.device(), Device.cpu); + EXPECT_TRUE(AreEqTensor(keep, t)); + + Tensor cpu = LoadFromFileDevice(tmp.str(), false); + EXPECT_EQ(cpu.device(), Device.cpu); + EXPECT_TRUE(AreEqTensor(cpu, t)); +} + +// Load committed reference file (regression coverage for the on-disk format). +TEST(IOTensorTest, LoadReference) { + Tensor loaded = LoadFromFile(ref_data_dir() + "tensor.h5"); + EXPECT_TRUE(AreEqTensor(loaded, ref::tensor())); +} diff --git a/tests/io_test/UniTensor_test.cpp b/tests/io_test/UniTensor_test.cpp new file mode 100644 index 000000000..ef182a1f4 --- /dev/null +++ b/tests/io_test/UniTensor_test.cpp @@ -0,0 +1,186 @@ +#include "io_test_tools.h" + +#include "test_tools.h" + +using namespace cytnx; +using namespace cytnx::IOTest; +using namespace cytnx::TestTools; + +namespace { + + // Round-trip a UniTensor and check both metadata and element values survive. + void ExpectRoundTrip(const UniTensor& ut) { + UniTensor loaded = RoundTrip(ut); + EXPECT_TRUE(AreEqUniTensorMeta(loaded, ut)) << "metadata mismatch after round-trip"; + EXPECT_TRUE(AreEqUniTensor(loaded, ut)) << "element mismatch after round-trip"; + } + +} // namespace + +// Untagged dense UniTensor across all supported dtypes. +TEST(IOUniTensorTest, RoundTripDenseUntaggedAllDtypes) { + for (auto dtype : dtype_list) { + UniTensor ut = + UniTensor({Bond(3), Bond(4), Bond(2)}, {"a", "b", "c"}, 1, dtype).set_name("dense"); + InitUniTensorUniform(ut, /*seed=*/dtype); + ExpectRoundTrip(ut); + } +} + +// Tagged (directional, no-symmetry) dense UniTensor. +TEST(IOUniTensorTest, RoundTripDenseTagged) { + UniTensor ut = UniTensor({Bond(3, BD_KET), Bond(3, BD_BRA)}, {"i", "j"}, 1, Type.ComplexDouble) + .set_name("tagged"); + ASSERT_TRUE(ut.is_tag()); + InitUniTensorUniform(ut, 21); + ExpectRoundTrip(ut); +} + +// Diagonal UniTensor (is_diag = true). +TEST(IOUniTensorTest, RoundTripDiagonal) { + UniTensor ut = + UniTensor({Bond(5), Bond(5)}, {"i", "j"}, 1, Type.Double, Device.cpu, /*is_diag=*/true) + .set_name("diag"); + ASSERT_TRUE(ut.is_diag()); + InitUniTensorUniform(ut, 22); + ExpectRoundTrip(ut); +} + +// Symmetric (block) UniTensor with a single U1 symmetry, across numeric dtypes. +TEST(IOUniTensorTest, RoundTripSymmetricU1) { + // Bool is excluded: block construction (Hstack) does not support it. + for (auto dtype : {Type.Double, Type.ComplexDouble, Type.Float, Type.Int64}) { + Bond bk = Bond(BD_KET, {Qs(0) >> 2, Qs(1) >> 3, Qs(-1) >> 1}); + Bond bb = Bond(BD_BRA, {Qs(0) >> 2, Qs(1) >> 3, Qs(-1) >> 1}); + UniTensor ut = UniTensor({bk, bb}, {"a", "b"}, -1, dtype).set_name("sym_u1"); + ASSERT_EQ(ut.uten_type(), UTenType.Block); + InitUniTensorUniform(ut, 23); + ExpectRoundTrip(ut); + } +} + +// Symmetric UniTensor with multiple symmetries (U1 x Z2). +TEST(IOUniTensorTest, RoundTripSymmetricMulti) { + Bond bk = Bond(BD_KET, {{0, 0}, {1, 1}, {0, 1}}, {2, 3, 1}, {Symmetry::U1(), Symmetry::Zn(2)}); + Bond bb = Bond(BD_BRA, {{0, 0}, {1, 1}, {0, 1}}, {2, 3, 1}, {Symmetry::U1(), Symmetry::Zn(2)}); + UniTensor ut = UniTensor({bk, bb}, {"a", "b"}, -1, Type.ComplexDouble).set_name("sym_u1z2"); + ASSERT_EQ(ut.uten_type(), UTenType.Block); + InitUniTensorUniform(ut, 24); + ExpectRoundTrip(ut); +} + +// Fermionic symmetric (BlockFermionic) UniTensor. +TEST(IOUniTensorTest, RoundTripFermionic) { + Bond bk = Bond(BD_IN, {Qs(0) >> 2, Qs(1) >> 3}, {Symmetry::FermionParity()}); + Bond bb = Bond(BD_OUT, {Qs(0) >> 2, Qs(1) >> 3}, {Symmetry::FermionParity()}); + UniTensor ut = UniTensor({bk, bb}, {"a", "b"}, -1, Type.ComplexDouble).set_name("ferm"); + ASSERT_EQ(ut.uten_type(), UTenType.BlockFermionic); + InitUniTensorUniform(ut, 25); + ExpectRoundTrip(ut); +} + +// Saving the same name twice must fail without overwrite and succeed with it. +TEST(IOUniTensorTest, Overwrite) { + TempH5File tmp; + UniTensor a = UniTensor({Bond(3), Bond(2)}, {"a", "b"}, 1, Type.Double).set_name("A"); + UniTensor b = + UniTensor({Bond(4), Bond(5), Bond(2)}, {"x", "y", "z"}, 2, Type.Double).set_name("B"); + InitUniTensorUniform(a, 1); + InitUniTensorUniform(b, 2); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + UniTensor loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_TRUE(AreEqUniTensorMeta(loaded, b)); + EXPECT_TRUE(AreEqUniTensor(loaded, b)); +} + +// Overwrite with identical structure but different values replaces the data. +TEST(IOUniTensorTest, OverwriteSameStructure) { + TempH5File tmp; + UniTensor a = UniTensor({Bond(3), Bond(2)}, {"a", "b"}, 1, Type.Double).set_name("A"); + UniTensor b = UniTensor({Bond(3), Bond(2)}, {"a", "b"}, 1, Type.Double).set_name("A"); + InitUniTensorUniform(a, 1); + InitUniTensorUniform(b, 77); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(a, file, "obj"); + EXPECT_THROW(io::Save(b, file, "obj", "", false), std::logic_error); + EXPECT_NO_THROW(io::Save(b, file, "obj", "", true)); + file.close(); + } + UniTensor loaded = LoadFromFile(tmp.str(), "obj"); + EXPECT_TRUE(AreEqUniTensorMeta(loaded, b)); + EXPECT_TRUE(AreEqUniTensor(loaded, b)); +} + +// Store under a nested group path and read it back. +TEST(IOUniTensorTest, NestedPath) { + TempH5File tmp; + UniTensor ut = UniTensor({Bond(3), Bond(3)}, {"a", "b"}, 1, Type.Double).set_name("nested"); + InitUniTensorUniform(ut, 31); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(ut, file, "obj", "/group/sub"); + file.close(); + } + UniTensor loaded = LoadFromFile(tmp.str(), "obj", "/group/sub"); + EXPECT_TRUE(AreEqUniTensorMeta(loaded, ut)); + EXPECT_TRUE(AreEqUniTensor(loaded, ut)); +} + +// Multiple UniTensors coexist in one file under different names. +TEST(IOUniTensorTest, MultipleObjectsInOneFile) { + TempH5File tmp; + UniTensor u1 = UniTensor({Bond(2), Bond(3)}, {"a", "b"}, 1, Type.Double).set_name("u1"); + UniTensor u2 = UniTensor({Bond(4), Bond(2)}, {"c", "d"}, 1, Type.ComplexDouble).set_name("u2"); + InitUniTensorUniform(u1, 41); + InitUniTensorUniform(u2, 42); + { + H5::H5File file = io::open(tmp.str(), io::ACC_TRUNC); + io::Save(u1, file, "first"); + io::Save(u2, file, "second"); + file.close(); + } + EXPECT_TRUE(AreEqUniTensor(LoadFromFile(tmp.str(), "first"), u1)); + EXPECT_TRUE(AreEqUniTensor(LoadFromFile(tmp.str(), "second"), u2)); +} + +// The device-restoring Load overload keeps CPU data on the CPU either way. +TEST(IOUniTensorTest, LoadDeviceRestore) { + TempH5File tmp; + UniTensor ut = UniTensor({Bond(3), Bond(4)}, {"a", "b"}, 1, Type.Double).set_name("dev"); + InitUniTensorUniform(ut, 51); + SaveToFile(ut, tmp.str()); + + UniTensor keep = LoadFromFileDevice(tmp.str(), true); + EXPECT_EQ(keep.device(), Device.cpu); + EXPECT_TRUE(AreEqUniTensor(keep, ut)); + + UniTensor cpu = LoadFromFileDevice(tmp.str(), false); + EXPECT_EQ(cpu.device(), Device.cpu); + EXPECT_TRUE(AreEqUniTensor(cpu, ut)); +} + +// Load committed reference files (regression coverage for the on-disk format). +TEST(IOUniTensorTest, LoadReference) { + struct Ref { + std::string file; + UniTensor expected; + }; + std::vector refs = { + {"unitensor_dense.h5", ref::unitensor_dense()}, + {"unitensor_diag.h5", ref::unitensor_diag()}, + {"unitensor_sym.h5", ref::unitensor_sym()}, + {"unitensor_fermionic.h5", ref::unitensor_fermionic()}, + }; + for (const auto& r : refs) { + UniTensor loaded = LoadFromFile(ref_data_dir() + r.file); + EXPECT_TRUE(AreEqUniTensorMeta(loaded, r.expected)) << r.file; + EXPECT_TRUE(AreEqUniTensor(loaded, r.expected)) << r.file; + } +} diff --git a/tests/io_test/io_test_tools.h b/tests/io_test/io_test_tools.h new file mode 100644 index 000000000..32e34806c --- /dev/null +++ b/tests/io_test/io_test_tools.h @@ -0,0 +1,146 @@ +#ifndef CYTNX_TESTS_IO_TEST_TOOLS_H_ +#define CYTNX_TESTS_IO_TEST_TOOLS_H_ + +#include +#include +#include +#include + +#include + +#include "cytnx.hpp" +#include "io.hpp" +#include "test_tools.h" + +// Shared helpers for the cytnx::io HDF5 Save/Load tests. Saving and loading go +// through io::Save / io::Load only; io::open provides the file handle, which is +// passed directly as the container (an H5File is a valid H5::Group). +namespace cytnx { + namespace IOTest { + + // Directory holding the committed reference data produced by the generator + // in developer_tools/io_data_generator.cpp. + inline std::string ref_data_dir() { return std::string(CYTNX_TEST_DATA_DIR) + "/io/"; } + + // Unique temp file path ending in ".h5", removed again on destruction. + class TempH5File { + public: + TempH5File() : path_(std::string(std::tmpnam(nullptr)) + ".h5") {} + ~TempH5File() { + std::error_code ec; + std::filesystem::remove(path_, ec); + } + const std::string& str() const { return path_; } + + private: + std::string path_; + }; + + // Save a single object into a freshly truncated file. + template + void SaveToFile(const T& object, const std::string& fname, const std::string& name = "obj", + const std::string& path = "") { + H5::H5File file = io::open(fname, io::ACC_TRUNC); + io::Save(object, file, name, path); + file.close(); + } + + // Load an object of type T from a file. + template + T LoadFromFile(const std::string& fname, const std::string& name = "obj", + const std::string& path = "") { + io::savable_class holder = T(); + H5::H5File file = io::open(fname, io::ACC_IN); + io::Load(holder, file, name, path); + file.close(); + return std::get(holder); + } + + // Load an object of type T, choosing whether to restore the stored device. + // Only valid for the loadable_to_device alternatives (Storage, Tensor, UniTensor, MPS). + template + T LoadFromFileDevice(const std::string& fname, bool restore_device, + const std::string& name = "obj", const std::string& path = "") { + io::loadable_to_device holder = T(); + H5::H5File file = io::open(fname, io::ACC_IN); + io::Load(holder, file, name, path, restore_device); + file.close(); + return std::get(holder); + } + + // Save then load through a temporary file and return the loaded copy. + template + T RoundTrip(const T& object) { + TempH5File tmp; + SaveToFile(object, tmp.str()); + return LoadFromFile(tmp.str()); + } + + // Deterministic reference objects shared by the data generator + // (developer_tools side) and the load-from-reference tests, so both build + // exactly the same object. Each reference object is stored in its own file + // under the name "obj" in ref_data_dir(). + namespace ref { + + inline Storage storage() { + std::vector v(12); + for (size_t i = 0; i < v.size(); ++i) v[i] = static_cast(i % 5); + return Storage(v).astype(Type.ComplexDouble); + } + + inline Tensor tensor() { + Tensor t = Tensor({3, 4, 2}, Type.Double); + TestTools::InitTensorUniform(t, 1234); + return t; + } + + inline Bond bond() { + return Bond(BD_KET, {{0, 2}, {1, 5}, {1, 6}, {0, 1}}, {4, 7, 2, 3}, + {Symmetry::Zn(2), Symmetry::U1()}); + } + + inline Symmetry symmetry() { return Symmetry::Zn(3); } + + inline Symmetry symmetry_fpar() { return Symmetry::FermionParity(); } + + inline Bond bond_fermionic() { + return Bond(BD_IN, {Qs(0) >> 2, Qs(1) >> 3}, {Symmetry::FermionParity()}); + } + + inline UniTensor unitensor_dense() { + UniTensor ut = UniTensor({Bond(3), Bond(4), Bond(2)}, {"a", "b", "c"}, 1, Type.Double) + .set_name("ref_dense"); + TestTools::InitUniTensorUniform(ut, 5678); + return ut; + } + + inline UniTensor unitensor_diag() { + UniTensor ut = + UniTensor({Bond(5), Bond(5)}, {"i", "j"}, 1, Type.Double, Device.cpu, /*is_diag=*/true) + .set_name("ref_diag"); + TestTools::InitUniTensorUniform(ut, 8765); + return ut; + } + + inline UniTensor unitensor_sym() { + Bond bk = Bond(BD_KET, {Qs(0) >> 2, Qs(1) >> 3, Qs(-1) >> 1}); + Bond bb = Bond(BD_BRA, {Qs(0) >> 2, Qs(1) >> 3, Qs(-1) >> 1}); + UniTensor ut = UniTensor({bk, bb}, {"a", "b"}, -1, Type.ComplexDouble).set_name("ref_sym"); + TestTools::InitUniTensorUniform(ut, 4321); + return ut; + } + + inline UniTensor unitensor_fermionic() { + Bond bk = Bond(BD_IN, {Qs(0) >> 2, Qs(1) >> 3}, {Symmetry::FermionParity()}); + Bond bb = Bond(BD_OUT, {Qs(0) >> 2, Qs(1) >> 3}, {Symmetry::FermionParity()}); + UniTensor ut = UniTensor({bk, bb}, {"a", "b"}, -1, Type.ComplexDouble).set_name("ref_ferm"); + TestTools::InitUniTensorUniform(ut, 1357); + return ut; + } + + } // namespace ref + + } // namespace IOTest +} // namespace cytnx + +#endif // CYTNX_TESTS_IO_TEST_TOOLS_H_ diff --git a/tests/test_data_base/io/bond.h5 b/tests/test_data_base/io/bond.h5 new file mode 100644 index 000000000..ef287281b Binary files /dev/null and b/tests/test_data_base/io/bond.h5 differ diff --git a/tests/test_data_base/io/bond_fermionic.h5 b/tests/test_data_base/io/bond_fermionic.h5 new file mode 100644 index 000000000..5c56d1e39 Binary files /dev/null and b/tests/test_data_base/io/bond_fermionic.h5 differ diff --git a/tests/test_data_base/io/storage.h5 b/tests/test_data_base/io/storage.h5 new file mode 100644 index 000000000..138b41511 Binary files /dev/null and b/tests/test_data_base/io/storage.h5 differ diff --git a/tests/test_data_base/io/symmetry.h5 b/tests/test_data_base/io/symmetry.h5 new file mode 100644 index 000000000..e04c37550 Binary files /dev/null and b/tests/test_data_base/io/symmetry.h5 differ diff --git a/tests/test_data_base/io/symmetry_fpar.h5 b/tests/test_data_base/io/symmetry_fpar.h5 new file mode 100644 index 000000000..5d1e599de Binary files /dev/null and b/tests/test_data_base/io/symmetry_fpar.h5 differ diff --git a/tests/test_data_base/io/tensor.h5 b/tests/test_data_base/io/tensor.h5 new file mode 100644 index 000000000..b531450d2 Binary files /dev/null and b/tests/test_data_base/io/tensor.h5 differ diff --git a/tests/test_data_base/io/unitensor_dense.h5 b/tests/test_data_base/io/unitensor_dense.h5 new file mode 100644 index 000000000..754825c2c Binary files /dev/null and b/tests/test_data_base/io/unitensor_dense.h5 differ diff --git a/tests/test_data_base/io/unitensor_diag.h5 b/tests/test_data_base/io/unitensor_diag.h5 new file mode 100644 index 000000000..fffb264f8 Binary files /dev/null and b/tests/test_data_base/io/unitensor_diag.h5 differ diff --git a/tests/test_data_base/io/unitensor_fermionic.h5 b/tests/test_data_base/io/unitensor_fermionic.h5 new file mode 100644 index 000000000..a35a1921b Binary files /dev/null and b/tests/test_data_base/io/unitensor_fermionic.h5 differ diff --git a/tests/test_data_base/io/unitensor_sym.h5 b/tests/test_data_base/io/unitensor_sym.h5 new file mode 100644 index 000000000..b8c9b746f Binary files /dev/null and b/tests/test_data_base/io/unitensor_sym.h5 differ diff --git a/tests/utils/io_data_generator.cpp b/tests/utils/io_data_generator.cpp new file mode 100644 index 000000000..0107f75f0 --- /dev/null +++ b/tests/utils/io_data_generator.cpp @@ -0,0 +1,41 @@ +// Generator for the committed HDF5 reference data used by the io load tests. +// +// This file does not contain real tests; it only (re)generates the reference +// files in "tests/test_data_base/io" using io::Save, so that the io load tests +// can verify that previously-saved files remain readable (format-stability / +// regression coverage). +// +// To (re)generate the reference data, set NEED_GEN_IO_DATA to 1, rebuild, and +// run the test binary once (e.g. `./test_main --gtest_filter='IODataGen.*'`). +// Then set NEED_GEN_IO_DATA back to 0 and commit the generated files. +#define NEED_GEN_IO_DATA 0 +#if NEED_GEN_IO_DATA + + #include + + #include "cytnx.hpp" + #include "io.hpp" + #include "io_test/io_test_tools.h" + +using namespace cytnx; +using namespace cytnx::IOTest; + +namespace { + void ensure_dir() { std::filesystem::create_directories(ref_data_dir()); } +} // namespace + +TEST(IODataGen, GenerateAll) { + ensure_dir(); + SaveToFile(ref::storage(), ref_data_dir() + "storage.h5"); + SaveToFile(ref::tensor(), ref_data_dir() + "tensor.h5"); + SaveToFile(ref::bond(), ref_data_dir() + "bond.h5"); + SaveToFile(ref::bond_fermionic(), ref_data_dir() + "bond_fermionic.h5"); + SaveToFile(ref::symmetry(), ref_data_dir() + "symmetry.h5"); + SaveToFile(ref::symmetry_fpar(), ref_data_dir() + "symmetry_fpar.h5"); + SaveToFile(ref::unitensor_dense(), ref_data_dir() + "unitensor_dense.h5"); + SaveToFile(ref::unitensor_diag(), ref_data_dir() + "unitensor_diag.h5"); + SaveToFile(ref::unitensor_sym(), ref_data_dir() + "unitensor_sym.h5"); + SaveToFile(ref::unitensor_fermionic(), ref_data_dir() + "unitensor_fermionic.h5"); +} + +#endif // NEED_GEN_IO_DATA diff --git a/tools/cibuildwheel_before_all.sh b/tools/cibuildwheel_before_all.sh index 38b75db5c..e59efbb13 100644 --- a/tools/cibuildwheel_before_all.sh +++ b/tools/cibuildwheel_before_all.sh @@ -2,11 +2,11 @@ set -xe # Install required packages for manylinux_2_28+ (AlmaLinux/RHEL). if command -v dnf >/dev/null 2>&1; then - dnf install -y boost-devel openblas-devel arpack-devel ccache + dnf install -y boost-devel openblas-devel arpack-devel hdf5-devel ccache # musllinux_1_2 images are Alpine-based, so use apk there. elif command -v apk >/dev/null 2>&1; then apk update - apk add boost-dev openblas-dev arpack-dev ccache + apk add boost-dev openblas-dev arpack-dev hdf5-dev ccache else echo "Unsupported package manager: expected dnf or apk" >&2 exit 1 diff --git a/tools/cibuildwheel_before_all_macos.sh b/tools/cibuildwheel_before_all_macos.sh index 7372c9268..24dab1099 100755 --- a/tools/cibuildwheel_before_all_macos.sh +++ b/tools/cibuildwheel_before_all_macos.sh @@ -6,7 +6,7 @@ set -euo pipefail # encode a minimum supported macOS version (minos). We inspect that metadata # and persist both a report and an effective deployment target for cibuildwheel. -probe_packages=(arpack boost openblas) +probe_packages=(arpack boost hdf5 openblas) install_only_packages=(ccache libomp) install_packages=("${probe_packages[@]}" "${install_only_packages[@]}")