From 7abb9e26e4b17cc1f8588996a859bc79d96b8b1f Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 9 Feb 2026 20:58:12 +0800 Subject: [PATCH 01/51] fix some --- cmake/utils.cmake | 2 +- src/ailego/buffer/buffer_manager.cc | 4 +-- src/ailego/math/inner_product_matrix_fp16.cc | 2 +- src/ailego/math/inner_product_matrix_fp32.cc | 3 +- .../mips_euclidean_distance_matrix_fp32.cc | 2 +- .../math_batch/inner_product_distance_batch.h | 1 + src/core/algorithm/flat/flat_builder.h | 1 + .../algorithm/flat/flat_streamer_dumper.h | 1 + .../flat_sparse_streamer_entity.cc | 1 + src/include/zvec/ailego/internal/platform.h | 2 +- src/include/zvec/ailego/io/file.h | 1 + src/include/zvec/ailego/logger/logger.h | 4 +-- src/include/zvec/ailego/utility/file_helper.h | 1 + thirdparty/arrow/CMakeLists.txt | 36 +++++++++++++------ 14 files changed, 41 insertions(+), 20 deletions(-) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index faccff08b..cda8a055a 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -12,7 +12,7 @@ function(apply_patch_once patch_name target_dir patch_file) #message(STATUS "Applying patch '${patch_name}' to ${target_dir} ...") execute_process( - COMMAND patch -p1 -i "${patch_file}" + COMMAND git apply --ignore-space-change --ignore-whitespace "${patch_file}" WORKING_DIRECTORY "${target_dir}" RESULT_VARIABLE patch_result OUTPUT_VARIABLE patch_stdout diff --git a/src/ailego/buffer/buffer_manager.cc b/src/ailego/buffer/buffer_manager.cc index ac2945b08..e42e58d9f 100644 --- a/src/ailego/buffer/buffer_manager.cc +++ b/src/ailego/buffer/buffer_manager.cc @@ -256,8 +256,8 @@ bool BufferManager::BufferContext::read_vector() { } AILEGO_DEFER([this] { file.close(); }); uint32_t len = id.vector().length; - auto ret = posix_memalign((void **)&vector, 64, len); // 64-byte alignment - if (ret != 0 || vector == nullptr) { + vector = (uint8_t *)ailego_aligned_malloc(len, 64); // 64-byte alignment + if (vector == nullptr) { LOG_ERROR("Failed to allocate buffer for file[%s]", file_name.c_str()); return false; } diff --git a/src/ailego/math/inner_product_matrix_fp16.cc b/src/ailego/math/inner_product_matrix_fp16.cc index 682cc9181..7968e63ec 100644 --- a/src/ailego/math/inner_product_matrix_fp16.cc +++ b/src/ailego/math/inner_product_matrix_fp16.cc @@ -1881,7 +1881,7 @@ float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count, _mm_loadu_ph(val_start_2 + k))); } - Float16 __attribute__((aligned(16))) tmp_res[8]; + AILEGO_ALIGNED(16) Float16 tmp_res[8]; _mm_store_ph(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3] + tmp_res[4] + tmp_res[5] + tmp_res[6] + tmp_res[7]); diff --git a/src/ailego/math/inner_product_matrix_fp32.cc b/src/ailego/math/inner_product_matrix_fp32.cc index 78e260d0c..4aae107e7 100644 --- a/src/ailego/math/inner_product_matrix_fp32.cc +++ b/src/ailego/math/inner_product_matrix_fp32.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include "distance_matrix_accum_fp32.i" #include "inner_product_matrix.h" @@ -379,7 +380,7 @@ float InnerProductSparseInSegmentSSE(uint32_t m_sparse_count, _mm_loadu_ps(val_start_2 + k))); } - float __attribute__((aligned(16))) tmp_res[4]; + AILEGO_ALIGNED(16) float tmp_res[4]; _mm_store_ps(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); } diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc index f14117a5a..800b659ee 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc @@ -592,7 +592,7 @@ float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count, _mm_loadu_ps(val_start_2 + k))); } - float __attribute__((aligned(16))) tmp_res[4]; + AILEGO_ALIGNED(16) float tmp_res[4]; _mm_store_ps(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); } diff --git a/src/ailego/math_batch/inner_product_distance_batch.h b/src/ailego/math_batch/inner_product_distance_batch.h index f5799497a..26f706a2a 100644 --- a/src/ailego/math_batch/inner_product_distance_batch.h +++ b/src/ailego/math_batch/inner_product_distance_batch.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include diff --git a/src/core/algorithm/flat/flat_builder.h b/src/core/algorithm/flat/flat_builder.h index 5a48c6e50..665179bcb 100644 --- a/src/core/algorithm/flat/flat_builder.h +++ b/src/core/algorithm/flat/flat_builder.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once +#include #include #include #include "flat_utility.h" diff --git a/src/core/algorithm/flat/flat_streamer_dumper.h b/src/core/algorithm/flat/flat_streamer_dumper.h index 7d3044bc0..30b2994bf 100644 --- a/src/core/algorithm/flat/flat_streamer_dumper.h +++ b/src/core/algorithm/flat/flat_streamer_dumper.h @@ -14,6 +14,7 @@ #pragma once +#include #include "flat_streamer.h" #include "flat_utility.h" diff --git a/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc b/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc index 53d8a24ce..1930d4325 100644 --- a/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc +++ b/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc @@ -15,6 +15,7 @@ #include "flat_sparse_streamer_entity.h" #include #include +#include #include #include #include diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index ecfe673fe..5260690e2 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -174,7 +174,7 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_likely(x) (x) #define ailego_unlikely(x) (x) #ifdef __SSE__ -#define ailego_prefetch(p) _mm_prefetch((p), 0) +#define ailego_prefetch(p) _mm_prefetch((const char *)(p), 0) #else #define ailego_prefetch(p) ((void)(p)) #endif diff --git a/src/include/zvec/ailego/io/file.h b/src/include/zvec/ailego/io/file.h index e884bdbf2..18c5f4922 100644 --- a/src/include/zvec/ailego/io/file.h +++ b/src/include/zvec/ailego/io/file.h @@ -14,6 +14,7 @@ #pragma once +#include #include namespace zvec { diff --git a/src/include/zvec/ailego/logger/logger.h b/src/include/zvec/ailego/logger/logger.h index 872d529f2..e92aa3aa5 100644 --- a/src/include/zvec/ailego/logger/logger.h +++ b/src/include/zvec/ailego/logger/logger.h @@ -150,8 +150,8 @@ class LoggerBroker { } //! Log Message - __attribute__((format(printf, 4, 5))) static void Log( - int level, const char *file, int line, const char *format, ...) { + AILEGO_PRINTF_FORMAT(4, 5) + static void Log(int level, const char *file, int line, const char *format, ...) { if (IsLevelEnabled(level)) { va_list args; va_start(args, format); diff --git a/src/include/zvec/ailego/utility/file_helper.h b/src/include/zvec/ailego/utility/file_helper.h index 00f4f5d0d..36b5fa940 100644 --- a/src/include/zvec/ailego/utility/file_helper.h +++ b/src/include/zvec/ailego/utility/file_helper.h @@ -16,6 +16,7 @@ #include #include +#include namespace zvec { namespace ailego { diff --git a/thirdparty/arrow/CMakeLists.txt b/thirdparty/arrow/CMakeLists.txt index 4326ea92a..a8129821a 100644 --- a/thirdparty/arrow/CMakeLists.txt +++ b/thirdparty/arrow/CMakeLists.txt @@ -1,4 +1,4 @@ -set(ARROW_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0) +set(ARROW_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0) if(ANDROID) set(ARROW_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.android.patch) apply_patch_once("arrow_android_fix" "${ARROW_SRC_DIR}" "${ARROW_PATCH}") @@ -12,12 +12,26 @@ include(ProcessorCount) ProcessorCount(NPROC) -set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/libparquet.a) -set(LIB_ARROW ${EXTERNAL_LIB_DIR}/libarrow.a) -set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/libarrow_compute.a) -set(LIB_ACERO ${EXTERNAL_LIB_DIR}/libarrow_acero.a) -set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/libarrow_bundled_dependencies.a) -set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/libarrow_dataset.a) +if(MSVC) + set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/parquet_static.lib) + set(LIB_ARROW ${EXTERNAL_LIB_DIR}/arrow_static.lib) + set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/arrow_compute_static.lib) + set(LIB_ACERO ${EXTERNAL_LIB_DIR}/arrow_acero_static.lib) + set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/arrow_bundled_dependencies.lib) + set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/arrow_dataset_static.lib) +else() + set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/libparquet.a) + set(LIB_ARROW ${EXTERNAL_LIB_DIR}/libarrow.a) + set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/libarrow_compute.a) + set(LIB_ACERO ${EXTERNAL_LIB_DIR}/libarrow_acero.a) + set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/libarrow_bundled_dependencies.a) + set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/libarrow_dataset.a) +endif() + +# $ is a generator expression that evaluates to the current build configuration (e.g., Debug, Release) at build system generation time, not at CMake configure time. +# Multi-config vs. single-config generators: +# Multi-config generators (Visual Studio, Xcode, Ninja Multi-Config): $ works as expected and evaluates differently for each configuration. +# Single-config generators (Makefiles, Ninja): $ evaluates to the value of CMAKE_BUILD_TYPE (which must be set at configure time) set(CONFIGURE_ENV_LIST "") if(USE_OSS_MIRROR) @@ -54,9 +68,9 @@ else() SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 DOWNLOAD_COMMAND "" BUILD_IN_SOURCE false - CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" - BUILD_COMMAND "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC} - INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local + CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} -DCMAKE_BUILD_TYPE=$ -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" + BUILD_COMMAND "${CMAKE_COMMAND}" --build . -j ${NPROC} --config $ + INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local --config $ BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET} LOG_DOWNLOAD ON LOG_CONFIGURE ON @@ -70,7 +84,7 @@ add_dependencies(arrow ARROW.BUILD) set(Arrow_FOUND TRUE PARENT_SCOPE) set(Arrow_INCLUDE_DIR ${EXTERNAL_INC_DIR} PARENT_SCOPE) -set(Arrow_LIBRARIES ${EXTERNAL_LIB_DIR}/libarrow.a PARENT_SCOPE) +set(Arrow_LIBRARIES ${LIB_ARROW} PARENT_SCOPE) set(Arrow_DIR ${EXTERNAL_BINARY_DIR} PARENT_SCOPE) set(Arrow_LIBRARY_DIR ${EXTERNAL_BINARY_DIR} PARENT_SCOPE) From c74f276ce88cae6a780c345e4b0228600cad6464 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 16:55:30 +0800 Subject: [PATCH 02/51] fix platform related --- src/ailego/internal/cpu_features.cc | 4 +++- src/include/zvec/ailego/logger/logger.h | 7 +++++++ src/include/zvec/core/framework/index_logger.h | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 4bf139ae0..06e1444d5 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -15,7 +15,9 @@ #include "cpu_features.h" #include -#if !defined(_MSC_VER) && !defined(__ARM_ARCH) +#if defined(_MSC_VER) +#include +#elif !defined(__ARM_ARCH) #include #endif diff --git a/src/include/zvec/ailego/logger/logger.h b/src/include/zvec/ailego/logger/logger.h index e92aa3aa5..08aeae1af 100644 --- a/src/include/zvec/ailego/logger/logger.h +++ b/src/include/zvec/ailego/logger/logger.h @@ -19,6 +19,13 @@ #include #include +// Define printf format attribute for GCC/Clang, empty for MSVC +#if defined(__GNUC__) || defined(__clang__) +#define AILEGO_PRINTF_FORMAT(fmt_idx, arg_idx) __attribute__((format(printf, fmt_idx, arg_idx))) +#else +#define AILEGO_PRINTF_FORMAT(fmt_idx, arg_idx) +#endif + //! Register Index Logger #define FACTORY_REGISTER_LOGGER_ALIAS(__NAME__, __IMPL__, ...) \ AILEGO_FACTORY_REGISTER(__NAME__, zvec::ailego::Logger, __IMPL__, \ diff --git a/src/include/zvec/core/framework/index_logger.h b/src/include/zvec/core/framework/index_logger.h index 3263850ff..3a6144e9c 100644 --- a/src/include/zvec/core/framework/index_logger.h +++ b/src/include/zvec/core/framework/index_logger.h @@ -143,8 +143,8 @@ class IndexLoggerBroker { } //! Log Message - __attribute__((format(printf, 4, 5))) static void Log( - int level, const char *file, int line, const char *format, ...) { + AILEGO_PRINTF_FORMAT(4, 5) + static void Log(int level, const char *file, int line, const char *format, ...) { if (IsLevelEnabled(level)) { va_list args; va_start(args, format); From 7fb10c43c6a564ec324c35355bef60d0b1fc3079 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 16:56:47 +0800 Subject: [PATCH 03/51] hnsw VLAs --- src/core/algorithm/hnsw/hnsw_builder_entity.cc | 4 ++-- src/core/algorithm/hnsw/hnsw_chunk.h | 1 - src/core/algorithm/hnsw/hnsw_dist_calculator.h | 4 ++-- src/core/algorithm/hnsw/hnsw_entity.cc | 17 ++++++++--------- src/core/algorithm/hnsw/hnsw_searcher_entity.cc | 4 ++-- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/core/algorithm/hnsw/hnsw_builder_entity.cc b/src/core/algorithm/hnsw/hnsw_builder_entity.cc index 472d98ee5..10591ac10 100644 --- a/src/core/algorithm/hnsw/hnsw_builder_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_builder_entity.cc @@ -136,8 +136,8 @@ int HnswBuilderEntity::get_vector(const node_id_t *ids, uint32_t count, int HnswBuilderEntity::get_vector( const node_id_t *ids, uint32_t count, std::vector &vec_blocks) const { - const void *vecs[count]; - get_vector(ids, count, vecs); + std::vector vecs(count); + get_vector(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } diff --git a/src/core/algorithm/hnsw/hnsw_chunk.h b/src/core/algorithm/hnsw/hnsw_chunk.h index b19d30943..7968dff95 100644 --- a/src/core/algorithm/hnsw/hnsw_chunk.h +++ b/src/core/algorithm/hnsw/hnsw_chunk.h @@ -15,7 +15,6 @@ #include #include -#include #include #include #include diff --git a/src/core/algorithm/hnsw/hnsw_dist_calculator.h b/src/core/algorithm/hnsw/hnsw_dist_calculator.h index 84faba40b..caf6e6d15 100644 --- a/src/core/algorithm/hnsw/hnsw_dist_calculator.h +++ b/src/core/algorithm/hnsw/hnsw_dist_calculator.h @@ -145,11 +145,11 @@ class HnswDistCalculator { return dist(vec); } - dist_t operator()(id_t i) { + dist_t operator()(node_id_t i) { return dist(i); } - dist_t operator()(id_t lhs, id_t rhs) { + dist_t operator()(node_id_t lhs, node_id_t rhs) { return dist(lhs, rhs); } diff --git a/src/core/algorithm/hnsw/hnsw_entity.cc b/src/core/algorithm/hnsw/hnsw_entity.cc index 53f07dd44..563f7d75e 100644 --- a/src/core/algorithm/hnsw/hnsw_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_entity.cc @@ -172,8 +172,8 @@ int64_t HnswEntity::dump_vectors( size_t padding_size = AlignSize(vector_dump_size) - vector_dump_size; - std::vector padding(padding_size); - memset(padding.data(), 0, sizeof(char) * padding_size); + char padding[padding_size]; + memset(padding, 0, sizeof(padding)); const void *data = nullptr; uint32_t crc = 0U; size_t vecs_size = 0UL; @@ -318,7 +318,8 @@ int64_t HnswEntity::dump_upper_neighbors( ailego_assert_with(!!neighbors.data, "invalid neighbors"); ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level), "invalid neighbors"); - memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size()); + size_t buffer_bytes = buffer.size() * sizeof(node_id_t); + memset(buffer.data(), 0, buffer_bytes); buffer[0] = neighbors.size(); if (neighbor_mapping.empty()) { memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t)); @@ -327,15 +328,13 @@ int64_t HnswEntity::dump_upper_neighbors( buffer[i + 1] = neighbor_mapping[neighbors[i]]; } } - if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) != - sizeof(node_id_t) * buffer.size()) { + if (dumper->write(buffer.data(), buffer_bytes) != buffer_bytes) { LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, - sizeof(node_id_t) * buffer.size()); + buffer_bytes); return IndexError_WriteData; } - crc = ailego::Crc32c::Hash(buffer.data(), - sizeof(node_id_t) * buffer.size(), crc); - offset += sizeof(node_id_t) * buffer.size(); + crc = ailego::Crc32c::Hash(buffer.data(), buffer_bytes, crc); + offset += buffer_bytes; } } size_t padding_size = 0; diff --git a/src/core/algorithm/hnsw/hnsw_searcher_entity.cc b/src/core/algorithm/hnsw/hnsw_searcher_entity.cc index 6661c1db3..6829f71d7 100644 --- a/src/core/algorithm/hnsw/hnsw_searcher_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_searcher_entity.cc @@ -148,8 +148,8 @@ int HnswSearcherEntity::get_vector(const node_id_t *ids, uint32_t count, int HnswSearcherEntity::get_vector( const node_id_t *ids, uint32_t count, std::vector &vec_blocks) const { - const void *vecs[count]; - get_vector(ids, count, vecs); + std::vector vecs(count); + get_vector(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } From 72f7214a31fe33e7c313e15933b5c33cd6c1d206 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 16:58:17 +0800 Subject: [PATCH 04/51] hnsw sparse VLAs --- .../hnsw_sparse/hnsw_sparse_builder_entity.cc | 4 ++-- src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h | 1 - .../hnsw_sparse/hnsw_sparse_dist_calculator.h | 4 ++-- .../algorithm/hnsw_sparse/hnsw_sparse_entity.cc | 13 ++++++------- .../hnsw_sparse/hnsw_sparse_searcher_entity.cc | 4 ++-- .../hnsw_sparse/hnsw_sparse_streamer_entity.cc | 2 +- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc index 48c20d726..7d912d54b 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc @@ -168,8 +168,8 @@ int HnswSparseBuilderEntity::get_vector_metas(const node_id_t *ids, int HnswSparseBuilderEntity::get_vector_metas( const node_id_t *ids, uint32_t count, std::vector &block_vecs) const { - const void *vecs[count]; - get_vector_metas(ids, count, vecs); + std::vector vecs(count); + get_vector_metas(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { block_vecs.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h index a02698914..c0c3da08a 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h @@ -15,7 +15,6 @@ #include #include -#include #include #include #include diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h index 06cb76e4d..5bbfc1b6a 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h @@ -138,11 +138,11 @@ class HnswSparseDistCalculator { return dist(vec); } - dist_t operator()(id_t i) { + dist_t operator()(node_id_t i) { return dist(i); } - dist_t operator()(id_t lhs, id_t rhs) { + dist_t operator()(node_id_t lhs, node_id_t rhs) { return dist(lhs, rhs); } diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc index a0ecb2fc1..71ead7dd0 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc @@ -381,6 +381,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors( size_t offset = 0; uint32_t crc = 0; std::vector buffer(upper_neighbor_cnt() + 1); + size_t buffer_bytes = buffer.size() * sizeof(node_id_t); for (node_id_t id = 0; id < doc_cnt(); ++id) { node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id]; auto level = get_level(new_id); @@ -395,7 +396,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors( ailego_assert_with(!!neighbors.data, "invalid neighbors"); ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level), "invalid neighbors"); - memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size()); + memset(buffer.data(), 0, buffer_bytes); buffer[0] = neighbors.size(); if (neighbor_mapping.empty()) { memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t)); @@ -404,15 +405,13 @@ int64_t HnswSparseEntity::dump_upper_neighbors( buffer[i + 1] = neighbor_mapping[neighbors[i]]; } } - if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) != - sizeof(node_id_t) * buffer.size()) { + if (dumper->write(buffer.data(), buffer_bytes) != buffer_bytes) { LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, - sizeof(node_id_t) * buffer.size()); + buffer_bytes); return IndexError_WriteData; } - crc = ailego::Crc32c::Hash(buffer.data(), - sizeof(node_id_t) * buffer.size(), crc); - offset += sizeof(node_id_t) * buffer.size(); + crc = ailego::Crc32c::Hash(buffer.data(), buffer_bytes, crc); + offset += buffer_bytes; } } size_t padding_size = 0; diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc index 8ef79adf2..4b23ab9a6 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc @@ -155,8 +155,8 @@ int HnswSparseSearcherEntity::get_vector_metas(const node_id_t *ids, int HnswSparseSearcherEntity::get_vector_metas( const node_id_t *ids, uint32_t count, std::vector &block_vecs) const { - const void *vecs[count]; - get_vector_metas(ids, count, vecs); + std::vector vecs(count); + get_vector_metas(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { block_vecs.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc index aaffd9c12..6b429acb9 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc @@ -93,7 +93,7 @@ int HnswSparseStreamerEntity::update_neighbors( } auto loc = get_neighbor_chunk_loc(level, id); - size_t size = reinterpret_cast(&hd->neighbors[i]) - &buffer[0]; + size_t size = reinterpret_cast(&hd->neighbors[i]) - buffer.data(); size_t ret = loc.first->write(loc.second, hd, size); if (ailego_unlikely(ret != size)) { LOG_ERROR("Write neighbor header failed, ret=%zu", ret); From 89f02f55999892dd46bcd5ae98ec6d7ebec9e253 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:35:00 +0800 Subject: [PATCH 05/51] flexible array member --- src/core/algorithm/flat/flat_index_format.h | 3 +-- src/core/algorithm/flat/flat_streamer.h | 2 +- src/core/algorithm/flat/flat_streamer_entity.h | 2 +- src/core/algorithm/hnsw/hnsw_entity.h | 2 +- src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h | 2 +- src/core/algorithm/ivf/ivf_dumper.h | 2 +- src/core/algorithm/ivf/ivf_entity.h | 2 +- src/core/algorithm/ivf/ivf_index_format.h | 2 +- 8 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/core/algorithm/flat/flat_index_format.h b/src/core/algorithm/flat/flat_index_format.h index d91385abd..08d93c793 100644 --- a/src/core/algorithm/flat/flat_index_format.h +++ b/src/core/algorithm/flat/flat_index_format.h @@ -41,7 +41,6 @@ struct LinearIndexHeader { block_count(0), index_meta_size(0) { memset(reserved_, 0, sizeof(reserved_)); - memset(index_meta, 0, sizeof(index_meta)); } uint32_t header_size{0}; uint32_t total_vector_count{0}; @@ -52,7 +51,7 @@ struct LinearIndexHeader { uint32_t block_count{0}; uint32_t index_meta_size{0}; char reserved_[28] = {0}; - char index_meta[0]; + char index_meta[]; }; /*! Index Format of Linear Index Meta for each Linear list diff --git a/src/core/algorithm/flat/flat_streamer.h b/src/core/algorithm/flat/flat_streamer.h index 66ceba6a0..567b84da5 100644 --- a/src/core/algorithm/flat/flat_streamer.h +++ b/src/core/algorithm/flat/flat_streamer.h @@ -180,10 +180,10 @@ class FlatStreamer : public IndexStreamer { State state_{STATE_INIT}; mutable std::mutex mapping_mutex_{}; ailego::SharedMutex dump_mutex_{}; - FlatStreamerEntity entity_; bool column_major_order_{false}; bool use_key_info_map_{true}; uint32_t read_block_size_{0}; + FlatStreamerEntity entity_; }; } // namespace core diff --git a/src/core/algorithm/flat/flat_streamer_entity.h b/src/core/algorithm/flat/flat_streamer_entity.h index 6ac410a82..7cf915429 100644 --- a/src/core/algorithm/flat/flat_streamer_entity.h +++ b/src/core/algorithm/flat/flat_streamer_entity.h @@ -389,7 +389,6 @@ class FlatStreamerEntity { IndexStorage::Pointer storage_{}; IndexMetric::MatrixDistance row_distance_{}, column_distance_{}; mutable std::vector segments_{}; - StreamerLinearMeta meta_{}; IndexStreamer::Stats &stats_; mutable std::shared_ptr key_info_map_lock_{}; std::unordered_map key_info_map_{}; @@ -401,6 +400,7 @@ class FlatStreamerEntity { uint32_t vec_unit_size_{0}; uint32_t vec_cols_{0}; mutable std::string vec_buf_{}; + StreamerLinearMeta meta_{}; }; } // namespace core diff --git a/src/core/algorithm/hnsw/hnsw_entity.h b/src/core/algorithm/hnsw/hnsw_entity.h index e5f2077f7..816676dff 100644 --- a/src/core/algorithm/hnsw/hnsw_entity.h +++ b/src/core/algorithm/hnsw/hnsw_entity.h @@ -138,7 +138,7 @@ struct HNSWHeader { struct NeighborsHeader { uint32_t neighbor_cnt; - node_id_t neighbors[0]; + node_id_t neighbors[]; }; struct Neighbors { diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h index 37166027e..1128ae06e 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h @@ -161,7 +161,7 @@ struct HNSWSparseHeader { struct NeighborsHeader { uint32_t neighbor_cnt; - node_id_t neighbors[0]; + node_id_t neighbors[1]; }; struct Neighbors { diff --git a/src/core/algorithm/ivf/ivf_dumper.h b/src/core/algorithm/ivf/ivf_dumper.h index d8ab9935b..81c03c835 100644 --- a/src/core/algorithm/ivf/ivf_dumper.h +++ b/src/core/algorithm/ivf/ivf_dumper.h @@ -249,11 +249,11 @@ class IVFDumper { size_t block_vector_count_{kDefaultBlockCount}; std::vector inverted_lists_meta_{}; std::vector keys_{}; - InvertedIndexHeader header_{}; uint32_t cur_list_id_{0}; uint32_t dumped_feature_count_{0}; size_t dumped_features_size_{0}; mutable size_t dumped_size_{0}; + InvertedIndexHeader header_; }; } // namespace core diff --git a/src/core/algorithm/ivf/ivf_entity.h b/src/core/algorithm/ivf/ivf_entity.h index c9d3d9647..d27f53e36 100644 --- a/src/core/algorithm/ivf/ivf_entity.h +++ b/src/core/algorithm/ivf/ivf_entity.h @@ -333,7 +333,6 @@ class IVFEntity { IndexMeta meta_{}; mutable IVFReformerWrapper reformer_{}; IVFDistanceCalculator::Pointer calculator_{}; - InvertedIndexHeader header_{}; IndexStorage::Pointer container_{}; IndexStorage::Segment::Pointer inverted_{}; IndexStorage::Segment::Pointer inverted_meta_{}; @@ -345,6 +344,7 @@ class IVFEntity { mutable std::string vector_{}; // temporary buffer for colomn major order float norm_value_{0.0f}; // normalize the inverted vector to orignal score bool norm_value_sqrt_{false}; // does the norm value need to sqrt + InvertedIndexHeader header_; }; } // namespace core diff --git a/src/core/algorithm/ivf/ivf_index_format.h b/src/core/algorithm/ivf/ivf_index_format.h index d86ae52b9..9688fc01b 100644 --- a/src/core/algorithm/ivf/ivf_index_format.h +++ b/src/core/algorithm/ivf/ivf_index_format.h @@ -33,7 +33,7 @@ struct InvertedIndexHeader { uint32_t block_count{0}; uint32_t index_meta_size{0}; char reserved_[28]; - char index_meta[0]; + char index_meta[]; }; /*! Index Format of Inverted Index Meta for each Inverted list From 8dc698b4d18280edb086f8eb0c999fd773f78e1b Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:35:39 +0800 Subject: [PATCH 06/51] type define --- src/core/algorithm/ivf/ivf_index_format.h | 3 +++ src/core/utility/sparse_utility.h | 3 ++- src/core/utility/visit_filter.h | 5 +++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/core/algorithm/ivf/ivf_index_format.h b/src/core/algorithm/ivf/ivf_index_format.h index 9688fc01b..bf125fca8 100644 --- a/src/core/algorithm/ivf/ivf_index_format.h +++ b/src/core/algorithm/ivf/ivf_index_format.h @@ -19,6 +19,9 @@ namespace zvec { namespace core { +using node_id_t = uint32_t; +using key_t = uint64_t; + static constexpr uint64_t kInvalidKey = std::numeric_limits::max(); /*! Index Format of Inverted Index Header diff --git a/src/core/utility/sparse_utility.h b/src/core/utility/sparse_utility.h index 6f057c557..d99b8d82b 100644 --- a/src/core/utility/sparse_utility.h +++ b/src/core/utility/sparse_utility.h @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -25,6 +24,8 @@ namespace zvec { namespace core { +using key_t = uint64_t; + constexpr uint32_t SEGMENT_ID_BITS = 16; constexpr uint32_t SEGMENT_ID_MASK = 0xFFFF; diff --git a/src/core/utility/visit_filter.h b/src/core/utility/visit_filter.h index ea76c14c5..4f46cadd4 100644 --- a/src/core/utility/visit_filter.h +++ b/src/core/utility/visit_filter.h @@ -28,6 +28,11 @@ namespace zvec { namespace core { +// id_t is a POSIX type from ; provide a definition for Windows +#if defined(_MSC_VER) +typedef unsigned int id_t; +#endif + struct VisitFilterHeader { VisitFilterHeader() : maxDocCnt(0), maxScanNum(0) {} uint64_t maxDocCnt; From 5521744003867de3ef405c0d4748fe7f5d70b30e Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:37:30 +0800 Subject: [PATCH 07/51] vla --- tests/ailego/parallel/multi_thread_list_test.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/ailego/parallel/multi_thread_list_test.cc b/tests/ailego/parallel/multi_thread_list_test.cc index 1d3be229f..32bf955b7 100644 --- a/tests/ailego/parallel/multi_thread_list_test.cc +++ b/tests/ailego/parallel/multi_thread_list_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -68,8 +69,7 @@ TEST(MultiThreadListTest, General) { uint32_t num_of_producer = 100; uint32_t num_of_producer_done = 100; - std::vector consumer_results(num_of_consumer); - std::fill(consumer_results.begin(), consumer_results.end(), 0); + std::vector consumer_results(num_of_consumer, 0); for (uint32_t i = 0; i < num_of_consumer; i++) { consumer_pool.execute(consumer, i + 1, &consumer_results[i]); @@ -227,8 +227,7 @@ TEST(MultiThreadListTest, General_Moveable) { uint32_t num_of_producer = 100; uint32_t num_of_producer_done = 100; - std::vector consumer_results(num_of_consumer); - std::fill(consumer_results.begin(), consumer_results.end(), 0); + std::vector consumer_results(num_of_consumer, 0); for (uint32_t i = 0; i < num_of_consumer; i++) { consumer_pool.execute(consumer_moveable, i + 1, &consumer_results[i]); From 88de5da80ef815a557a023d9704c2470214f42f0 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:57:19 +0800 Subject: [PATCH 08/51] vla + uint8_t --- tests/ailego/parallel/lock_test.cc | 1 + tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ailego/parallel/lock_test.cc b/tests/ailego/parallel/lock_test.cc index 2a98e8e94..b54875964 100644 --- a/tests/ailego/parallel/lock_test.cc +++ b/tests/ailego/parallel/lock_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include diff --git a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc index d9197f1a3..38c0c8756 100644 --- a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc +++ b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include "zvec/core/framework/index_framework.h" @@ -330,7 +331,7 @@ TEST(OptKmeansCluster, IN4General) { std::random_device rd; std::mt19937 gen(rd()); - std::uniform_int_distribution dist(0, UINT8_MAX); + std::uniform_int_distribution dist(0, UINT8_MAX); for (uint32_t i = 0; i < count; ++i) { std::vector vec(dimension / 2); From 8b3b3ef0e9d64e04423bae5f80338c6f246fcd23 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:58:59 +0800 Subject: [PATCH 09/51] __VA_ARGS__ --- src/core/algorithm/ivf/ivf_utility.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/core/algorithm/ivf/ivf_utility.h b/src/core/algorithm/ivf/ivf_utility.h index 38e41a4fd..e7d213194 100644 --- a/src/core/algorithm/ivf/ivf_utility.h +++ b/src/core/algorithm/ivf/ivf_utility.h @@ -35,22 +35,22 @@ namespace core { #endif #ifndef ivf_check_with_msg -#define ivf_check_with_msg(code, fmt, args...) \ - do { \ - if (ailego_unlikely((code) != 0)) { \ - LOG_ERROR(fmt, ##args); \ - return code; \ - } \ +#define ivf_check_with_msg(code, fmt, ...) \ + do { \ + if (ailego_unlikely((code) != 0)) { \ + LOG_ERROR(fmt, ##__VA_ARGS__); \ + return code; \ + } \ } while (0) #endif #ifndef ivf_assert_with_msg -#define ivf_assert_with_msg(cond, err, fmt, args...) \ - do { \ - if (ailego_unlikely(!(cond))) { \ - LOG_ERROR(fmt, ##args); \ - return err; \ - } \ +#define ivf_assert_with_msg(cond, err, fmt, ...) \ + do { \ + if (ailego_unlikely(!(cond))) { \ + LOG_ERROR(fmt, ##__VA_ARGS__); \ + return err; \ + } \ } while (0) #endif From e8dc87187d7b294350cbedb33e6ba7f1cd5ac763 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:59:08 +0800 Subject: [PATCH 10/51] iota --- src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc index 71ead7dd0..29ec39b94 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "hnsw_sparse_entity.h" +#include namespace zvec { namespace core { From 673cf8ec8b3a9f665cfd74efc429640bb62e14ed Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:59:23 +0800 Subject: [PATCH 11/51] ssize --- src/include/zvec/ailego/internal/platform.h | 11 ++++++----- tests/ailego/utility/string_helper_test.cc | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 5260690e2..9ba9039d3 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -44,6 +44,12 @@ #endif #endif +//! Add 'ssize_t' for MSVC (before extern "C" so it's available in C++) +#if defined(_MSC_VER) && !defined(_SSIZE_T_DEFINED) +#define _SSIZE_T_DEFINED +typedef intptr_t ssize_t; +#endif + #if defined(__cplusplus) extern "C" { #endif @@ -104,11 +110,6 @@ extern "C" { #endif #endif -//! Add 'ssize_t' for MSVC -#if defined(_MSC_VER) -typedef intptr_t ssize_t; -#endif - #if defined(_MSC_VER) //! Returns the number of trailing 0-bits in x static inline int ailego_ctz32(uint32_t x) { diff --git a/tests/ailego/utility/string_helper_test.cc b/tests/ailego/utility/string_helper_test.cc index 137600de3..dde0a8a74 100644 --- a/tests/ailego/utility/string_helper_test.cc +++ b/tests/ailego/utility/string_helper_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include From 431b54148535d0b89dab5f0cc3fea6e207b15c4f Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Feb 2026 19:59:32 +0800 Subject: [PATCH 12/51] sum4 --- src/ailego/math_batch/inner_product_distance_batch_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl.h b/src/ailego/math_batch/inner_product_distance_batch_impl.h index d15a747e7..747afd783 100644 --- a/src/ailego/math_batch/inner_product_distance_batch_impl.h +++ b/src/ailego/math_batch/inner_product_distance_batch_impl.h @@ -38,7 +38,7 @@ static void compute_one_to_many_fallback( inline float sum4(__m128 v) { v = _mm_add_ps(v, _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(v), 8))); - return v[0] + v[1]; + return _mm_cvtss_f32(v) + _mm_cvtss_f32(_mm_shuffle_ps(v, v, 1)); } inline __m128 sum_top_bottom_avx(__m256 v) { From 66a8c56230a3829c74bb972b1e51fd90ddc859c9 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Tue, 10 Mar 2026 14:19:18 +0800 Subject: [PATCH 13/51] fixup! hnsw VLAs --- src/core/algorithm/hnsw/hnsw_entity.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/algorithm/hnsw/hnsw_entity.cc b/src/core/algorithm/hnsw/hnsw_entity.cc index 563f7d75e..aa0c0a3fe 100644 --- a/src/core/algorithm/hnsw/hnsw_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_entity.cc @@ -172,8 +172,8 @@ int64_t HnswEntity::dump_vectors( size_t padding_size = AlignSize(vector_dump_size) - vector_dump_size; - char padding[padding_size]; - memset(padding, 0, sizeof(padding)); + std::vector padding(padding_size); + memset(padding.data(), 0, sizeof(char) * padding_size); const void *data = nullptr; uint32_t crc = 0U; size_t vecs_size = 0UL; From 28d8c68f10db481eb2fd6578e484b003fb4ef13f Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:41:59 +0800 Subject: [PATCH 14/51] win header, file/time/sleep implement --- src/db/common/cgroup_util.cc | 2 + src/db/common/file_helper.cc | 48 +++++++++++++++++-- .../combined_vector_column_indexer.cc | 1 + src/db/index/common/doc.cc | 16 +++++++ src/db/index/storage/store_helper.h | 2 +- src/db/index/storage/wal/local_wal_file.cc | 2 + src/db/sqlengine/common/generic_node.h | 1 + src/db/sqlengine/common/util.cc | 2 + src/include/zvec/ailego/internal/platform.h | 11 ++++- .../flat_sparse/flat_sparse_streamer_test.cc | 1 + .../core/algorithm/hnsw/hnsw_streamer_test.cc | 12 +++++ .../hnsw_sparse/hnsw_sparse_streamer_test.cc | 12 +++++ tests/db/index/storage/wal_file_test.cc | 13 +++++ tools/core/bench_result.h | 25 +++++++++- tools/core/local_builder.cc | 1 - tools/core/local_builder_original.cc | 1 - 16 files changed, 141 insertions(+), 9 deletions(-) diff --git a/src/db/common/cgroup_util.cc b/src/db/common/cgroup_util.cc index 6a56dfcda..87896b04f 100644 --- a/src/db/common/cgroup_util.cc +++ b/src/db/common/cgroup_util.cc @@ -253,6 +253,8 @@ uint64_t CgroupUtil::getCurrentMemoryUsage() { return readMemoryUsageProc(); #elif defined(PLATFORM_MACOS) return getMacOSMemoryUsage(); +#else + return 0; #endif } diff --git a/src/db/common/file_helper.cc b/src/db/common/file_helper.cc index aa766f8ca..c82131681 100644 --- a/src/db/common/file_helper.cc +++ b/src/db/common/file_helper.cc @@ -13,14 +13,19 @@ // limitations under the License. #include "file_helper.h" -#include -#include #include -#include #include -#include #include #include +#ifdef _MSC_VER +#include +#include +#else +#include +#include +#include +#include +#endif #include namespace zvec { @@ -32,6 +37,18 @@ const std::string FileHelper::RECOVER_SUFFIX = ".recovering"; bool FileHelper::CopyFile(const std::string &src_file_path, const std::string &dst_file_path) { +#ifdef _MSC_VER + std::string dst_file_path_tmp = dst_file_path + ".tmp"; + std::error_code ec; + std::filesystem::copy_file(src_file_path, dst_file_path_tmp, + std::filesystem::copy_options::overwrite_existing, + ec); + if (ec) { + return false; + } + std::filesystem::rename(dst_file_path_tmp, dst_file_path, ec); + return !ec; +#else int src_fd = open(src_file_path.c_str(), O_RDONLY, 0); if (src_fd < 0) { return false; @@ -54,10 +71,19 @@ bool FileHelper::CopyFile(const std::string &src_file_path, } } return rename(dst_file_path_tmp.c_str(), dst_file_path.c_str()) == 0; +#endif } bool FileHelper::CopyDirectory(const std::string &src_dir_path, const std::string &dst_dir_path) { +#ifdef _MSC_VER + std::error_code ec; + std::filesystem::copy(src_dir_path, dst_dir_path, + std::filesystem::copy_options::recursive | + std::filesystem::copy_options::overwrite_existing, + ec); + return !ec; +#else DIR *dir = opendir(src_dir_path.c_str()); if (!dir) { return false; @@ -91,6 +117,7 @@ bool FileHelper::CopyDirectory(const std::string &src_dir_path, } } return true; +#endif } void FileHelper::CleanupDirectory(const std::string &backup_dir, @@ -100,6 +127,18 @@ void FileHelper::CleanupDirectory(const std::string &backup_dir, return; } +#ifdef _MSC_VER + size_t prefix_len = strlen(prefix_name); + std::vector candidates; + std::error_code ec; + for (const auto &entry : + std::filesystem::directory_iterator(backup_dir, ec)) { + std::string name = entry.path().filename().string(); + if (name.compare(0, prefix_len, prefix_name) == 0) { + candidates.emplace_back(name); + } + } +#else DIR *dir = opendir(backup_dir.c_str()); if (!dir) { return; @@ -115,6 +154,7 @@ void FileHelper::CleanupDirectory(const std::string &backup_dir, candidates.emplace_back(dent->d_name); } } +#endif if (candidates.size() <= max_backup_count) { return; } diff --git a/src/db/index/column/vector_column/combined_vector_column_indexer.cc b/src/db/index/column/vector_column/combined_vector_column_indexer.cc index 70c71d07b..7d04d3426 100644 --- a/src/db/index/column/vector_column/combined_vector_column_indexer.cc +++ b/src/db/index/column/vector_column/combined_vector_column_indexer.cc @@ -14,6 +14,7 @@ #include "combined_vector_column_indexer.h" #include #include +#include namespace zvec { diff --git a/src/db/index/common/doc.cc b/src/db/index/common/doc.cc index dad9bbdd1..8fd1c6266 100644 --- a/src/db/index/common/doc.cc +++ b/src/db/index/common/doc.cc @@ -16,6 +16,9 @@ #include #include #include +#ifdef _MSC_VER +#include +#endif #include #include "db/common/constants.h" #include "db/index/common/type_helper.h" @@ -116,16 +119,29 @@ template T byte_swap(T value) { if constexpr (std::is_same_v) { uint16_t val = *reinterpret_cast(&value); +#ifdef _MSC_VER + val = _byteswap_ushort(val); +#else val = __builtin_bswap16(val); +#endif return *reinterpret_cast(&val); } else if constexpr (sizeof(T) == 1) { return value; } else if constexpr (sizeof(T) == 2) { return (value << 8) | ((value >> 8) & 0xFF); } else if constexpr (sizeof(T) == 4) { +#ifdef _MSC_VER + return static_cast(_byteswap_ulong(static_cast(value))); +#else return __builtin_bswap32(value); +#endif } else if constexpr (sizeof(T) == 8) { +#ifdef _MSC_VER + return static_cast( + _byteswap_uint64(static_cast(value))); +#else return __builtin_bswap64(value); +#endif } else { T result = 0; for (size_t i = 0; i < sizeof(T); ++i) { diff --git a/src/db/index/storage/store_helper.h b/src/db/index/storage/store_helper.h index f9b3b5155..138941a47 100644 --- a/src/db/index/storage/store_helper.h +++ b/src/db/index/storage/store_helper.h @@ -40,7 +40,7 @@ namespace zvec { inline FileFormat InferFileFormat(const std::string &file_path) { - std::string ext = std::filesystem::path(file_path).extension(); + std::string ext = std::filesystem::path(file_path).extension().string(); std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); if (ext == ".parquet") { return FileFormat::PARQUET; diff --git a/src/db/index/storage/wal/local_wal_file.cc b/src/db/index/storage/wal/local_wal_file.cc index 91490b212..6a50e5c34 100644 --- a/src/db/index/storage/wal/local_wal_file.cc +++ b/src/db/index/storage/wal/local_wal_file.cc @@ -13,7 +13,9 @@ // limitations under the License. #include "local_wal_file.h" +#ifndef _MSC_VER #include +#endif #include #include #include "db/common/error_code.h" diff --git a/src/db/sqlengine/common/generic_node.h b/src/db/sqlengine/common/generic_node.h index a90de1fbc..2275c3d57 100644 --- a/src/db/sqlengine/common/generic_node.h +++ b/src/db/sqlengine/common/generic_node.h @@ -15,6 +15,7 @@ #pragma once #include +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/common/util.cc b/src/db/sqlengine/common/util.cc index 3f18bfe4d..568830ad6 100644 --- a/src/db/sqlengine/common/util.cc +++ b/src/db/sqlengine/common/util.cc @@ -13,7 +13,9 @@ // limitations under the License. #include "util.h" +#ifndef _MSC_VER #include +#endif #include #include #include diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 9ba9039d3..1bdde31f1 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -45,11 +45,20 @@ #endif //! Add 'ssize_t' for MSVC (before extern "C" so it's available in C++) -#if defined(_MSC_VER) && !defined(_SSIZE_T_DEFINED) +#if defined(_MSC_VER) + +#if !defined(S_ISDIR) +#define S_ISDIR(mode) (((mode)&S_IFMT) == S_IFDIR) +#endif + +#if !defined(_SSIZE_T_DEFINED) #define _SSIZE_T_DEFINED typedef intptr_t ssize_t; #endif +#endif + + #if defined(__cplusplus) extern "C" { #endif diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc index cad6a4d35..c6f2e99e7 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc index c04f67124..577ab258d 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc @@ -14,7 +14,19 @@ #include "hnsw_streamer.h" #include #include +#ifndef _MSC_VER #include +#include +#else +#include +#include +inline void sleep(unsigned int seconds) { + std::this_thread::sleep_for(std::chrono::seconds(seconds)); +} +inline void usleep(unsigned int microseconds) { + std::this_thread::sleep_for(std::chrono::microseconds(microseconds)); +} +#endif #include #include #include diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc index 9192fdb12..c183f9192 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc @@ -14,7 +14,19 @@ #include "hnsw_sparse_streamer.h" #include #include +#ifndef _MSC_VER #include +#include +#else +#include +#include +inline void sleep(unsigned int seconds) { + std::this_thread::sleep_for(std::chrono::seconds(seconds)); +} +inline void usleep(unsigned int microseconds) { + std::this_thread::sleep_for(std::chrono::microseconds(microseconds)); +} +#endif #include #include #include diff --git a/tests/db/index/storage/wal_file_test.cc b/tests/db/index/storage/wal_file_test.cc index e2a073f17..531f5d9fb 100644 --- a/tests/db/index/storage/wal_file_test.cc +++ b/tests/db/index/storage/wal_file_test.cc @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifdef _MSC_VER +#define _ALLOW_KEYWORD_MACROS +#endif #define private public #define protected public #include "db/index/storage/wal/wal_file.h" @@ -22,6 +25,16 @@ #include #include #include +#ifdef _MSC_VER +#include +inline int truncate(const char *path, long length) { + int fd = _open(path, _O_RDWR); + if (fd < 0) return -1; + int ret = _chsize(fd, length); + _close(fd); + return ret; +} +#endif #include #include #include diff --git a/tools/core/bench_result.h b/tools/core/bench_result.h index 9c53b13a3..177e13513 100644 --- a/tools/core/bench_result.h +++ b/tools/core/bench_result.h @@ -14,11 +14,15 @@ #pragma once -#include #include #include #include #include +#ifdef _MSC_VER +#include +#else +#include +#endif #include namespace zvec { @@ -52,15 +56,29 @@ class BenchResult { lock_.unlock(); } void mark_start() { +#ifdef _MSC_VER + start_ = std::chrono::steady_clock::now(); +#else gettimeofday(&start_, NULL); +#endif } void mark_end() { +#ifdef _MSC_VER + end_ = std::chrono::steady_clock::now(); +#else gettimeofday(&end_, NULL); +#endif } long get_duration_by_ms() { +#ifdef _MSC_VER + return static_cast( + std::chrono::duration_cast(end_ - start_) + .count()); +#else long duration = (end_.tv_sec - start_.tv_sec) * 1000 + (end_.tv_usec - start_.tv_usec) / 1000; return duration; +#endif } long get_total_query_count() { return total_query_count_; @@ -114,8 +132,13 @@ class BenchResult { long total_process_time_by_us_; long min_time_by_us_; long max_time_by_us_; +#ifdef _MSC_VER + std::chrono::steady_clock::time_point start_; + std::chrono::steady_clock::time_point end_; +#else struct timeval start_; struct timeval end_; +#endif ailego::SpinMutex lock_; std::map process_time_map_; // }; diff --git a/tools/core/local_builder.cc b/tools/core/local_builder.cc index 9d502a1e5..82bce65c9 100644 --- a/tools/core/local_builder.cc +++ b/tools/core/local_builder.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include diff --git a/tools/core/local_builder_original.cc b/tools/core/local_builder_original.cc index f6a44014d..d17e0bc7c 100644 --- a/tools/core/local_builder_original.cc +++ b/tools/core/local_builder_original.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include From 3255327cf38422121a9d3cf785b92011ffa86d72 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:42:56 +0800 Subject: [PATCH 15/51] func overload --- src/include/zvec/db/schema.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/include/zvec/db/schema.h b/src/include/zvec/db/schema.h index 09885c16a..bce2a1fd4 100644 --- a/src/include/zvec/db/schema.h +++ b/src/include/zvec/db/schema.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once +#include #include #include #include @@ -46,6 +47,9 @@ class FieldSchema { nullable_(nullable), dimension_(0), index_params_(index_params ? index_params->clone() : nullptr) {} + FieldSchema(const std::string &name, DataType type, bool nullable, + std::nullptr_t) + : FieldSchema(name, type, nullable, IndexParams::Ptr(nullptr)) {} FieldSchema(const std::string &name, DataType type, uint32_t dimension, bool nullable, const IndexParams::Ptr &index_params = nullptr) : name_(name), From 228dfae960faca045f6be7b318c19c8f68bfae92 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:52:19 +0800 Subject: [PATCH 16/51] vla --- tests/ailego/parallel/multi_thread_list_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ailego/parallel/multi_thread_list_test.cc b/tests/ailego/parallel/multi_thread_list_test.cc index 32bf955b7..b772bec05 100644 --- a/tests/ailego/parallel/multi_thread_list_test.cc +++ b/tests/ailego/parallel/multi_thread_list_test.cc @@ -121,8 +121,8 @@ TEST(MultiThreadListTest, ConsumeStopResume) { ailego::ThreadPool producer_pool; ailego::ThreadPool consumer_pool; - uint32_t num_of_consumer = 100; - uint32_t num_of_producer = 100; + constexpr uint32_t num_of_consumer = 100; + constexpr uint32_t num_of_producer = 100; std::vector consumer_results(2 * num_of_consumer); std::fill(consumer_results.begin(), consumer_results.end(), 0); From 402f2a104de08fb2c461d139500f77ae452f2a85 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:53:29 +0800 Subject: [PATCH 17/51] TODO: should just support std::uniform_int_distribution instead of change to int16 (maybe) --- ...ips_euclidean_distance_matrix_int4_test.cc | 36 +++++++++---------- ...ips_euclidean_distance_matrix_int8_test.cc | 12 +++---- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc b/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc index 7a536d282..f5e157c2c 100644 --- a/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc +++ b/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc @@ -115,13 +115,13 @@ TEST(DistanceMatrix, GeneralRepeatedQuadraticInjection) { const uint32_t dim = (std::uniform_int_distribution(2, 128))(gen) * 2; const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim / 2); std::vector vec2(dim / 2); for (size_t d = 0; d < dim / 2; ++d) { - vec1[d] = dist(gen); - vec2[d] = dist(gen); + vec1[d] = static_cast(dist(gen)); + vec2[d] = static_cast(dist(gen)); } ASSERT_NEAR( ConvertAndComputeByMips(vec1.data(), vec2.data(), dim, m_val, e2), @@ -212,12 +212,12 @@ void TestSquaredEuclideanMatrixRepeatedQuadraticInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } float squared_l2_norm = 0.0f; for (size_t i = 0; i < matrix_size; i += dimension) { @@ -421,12 +421,12 @@ void MipsRepeatedQuadraticInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < block_size; ++i) { @@ -596,13 +596,13 @@ TEST(DistanceMatrix, GeneralSphericalInjection) { const uint32_t dim = (std::uniform_int_distribution(2, 128))(gen) * 2; const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim / 2); std::vector vec2(dim / 2); for (size_t d = 0; d < dim / 2; ++d) { - vec1[d] = dist(gen); - vec2[d] = dist(gen); + vec1[d] = static_cast(dist(gen)); + vec2[d] = static_cast(dist(gen)); } ASSERT_NEAR(ConvertAndComputeByMips(vec1.data(), vec2.data(), dim, e2), MipsSquaredEuclidean(vec1.data(), vec2.data(), dim, e2), @@ -687,12 +687,12 @@ void TestSquaredEuclideanMatrixSphericalInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } float squared_l2_norm = 0.0f; for (size_t i = 0; i < matrix_size; i += dimension) { @@ -895,12 +895,12 @@ void MipsSphericalInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < block_size; ++i) { diff --git a/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc b/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc index 8a5f7c5bc..36f49a480 100644 --- a/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc +++ b/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc @@ -100,7 +100,7 @@ TEST(DistanceMatrix, GeneralRepeatedQuadraticInjection) { const float epsilon = 1e-6; const uint32_t dim = std::uniform_int_distribution(2, 128)(gen); const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim); std::vector vec2(dim); @@ -193,7 +193,7 @@ void TestSquaredEuclideanMatrixRepeatedQuadraticInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } @@ -402,7 +402,7 @@ void MipsRepeatedQuadraticInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } @@ -561,7 +561,7 @@ TEST(DistanceMatrix, GeneralSphericalInjection) { const float epsilon = 1e-6; const uint32_t dim = std::uniform_int_distribution(2, 128)(gen); const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim); std::vector vec2(dim); @@ -649,7 +649,7 @@ void TestSquaredEuclideanMatrixSphericalInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } @@ -853,7 +853,7 @@ void MipsSphericalInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } From fb8511fc3cb6252306afe4ed361c1c8d84047aa4 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:53:45 +0800 Subject: [PATCH 18/51] uint -> unsigned int --- tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc index 38c0c8756..03b117e8c 100644 --- a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc +++ b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc @@ -262,7 +262,7 @@ TEST(OptKmeansCluster, BinaryGeneral) { for (const auto &it : centroids) { const auto &vec = it.vector(); - uint mask = 0x1; + unsigned int mask = 0x1; std::cout << it.follows() << " (" << it.score() << ") { " << !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", " << !!(vec[0] & (mask << 2)) << ", ... , " @@ -278,7 +278,7 @@ TEST(OptKmeansCluster, BinaryGeneral) { for (const auto &it : centroids) { const auto &vec = it.vector(); - uint mask = 0x1; + unsigned int mask = 0x1; std::cout << it.follows() << " (" << it.score() << ") { " << !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", " << !!(vec[0] & (mask << 2)) << ", ... , " @@ -294,7 +294,7 @@ TEST(OptKmeansCluster, BinaryGeneral) { for (const auto &it : centroids) { const auto &vec = it.vector(); - uint mask = 0x1; + unsigned int mask = 0x1; std::cout << it.follows() << " (" << it.score() << ") { " << !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", " << !!(vec[0] & (mask << 2)) << ", ... , " From 828e4570dd6d67c5b6172455bfd1d6a430ba1de7 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:54:27 +0800 Subject: [PATCH 19/51] TODO: unknown reason --- src/ailego/math/inner_product_matrix.h | 2 +- tests/db/index/column/vector_column_indexer_test.cc | 6 +++--- tests/db/index/storage/mem_store_test.cc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ailego/math/inner_product_matrix.h b/src/ailego/math/inner_product_matrix.h index e1e871834..147e99726 100644 --- a/src/ailego/math/inner_product_matrix.h +++ b/src/ailego/math/inner_product_matrix.h @@ -2772,7 +2772,7 @@ struct MinusInnerProductSparseMatrix { const void *sparse_value, std::string &buffer); - static inline float ComputeInnerProductSparseInSegment( + static float ComputeInnerProductSparseInSegment( uint32_t m_sparse_count, const uint16_t *m_sparse_index, const ValueType *m_sparse_value, uint32_t q_sparse_count, const uint16_t *q_sparse_index, const ValueType *q_sparse_value); diff --git a/tests/db/index/column/vector_column_indexer_test.cc b/tests/db/index/column/vector_column_indexer_test.cc index 483efcdee..ff027a0da 100644 --- a/tests/db/index/column/vector_column_indexer_test.cc +++ b/tests/db/index/column/vector_column_indexer_test.cc @@ -1577,7 +1577,7 @@ TEST(VectorColumnIndexerTest, CosineGeneral) { const int kDim = 20; const int kCount = 20; // can't set too large, or the qunatization error // will be too large due to float's precision - const int kTopk = 10; + const uint32_t kTopk = 10; char cmd_buf[100]; snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); @@ -1725,7 +1725,7 @@ TEST(VectorColumnIndexerTest, CosineGeneral) { TEST(VectorColumnIndexerTest, Score) { const std::string index_file_path = "test_indexer.index"; - const int kTopk = 10; + const uint32_t kTopk = 10; constexpr idx_t kDocId1 = 2345; constexpr idx_t kDocId2 = 5432; auto vector1 = std::vector{3.0f, 4.0f, 5.0f}; @@ -2573,7 +2573,7 @@ TEST(VectorColumnIndexerTest, Refiner) { const int kDim = 20; const int kCount = 20; // can't set too large, or the qunatization error // will be too large due to float's precision - const int kTopk = 10; + const uint32_t kTopk = 10; auto del_index_file_func = [&](const std::string &file_name) { auto cmd_buf = "rm -f " + file_name; diff --git a/tests/db/index/storage/mem_store_test.cc b/tests/db/index/storage/mem_store_test.cc index 27c019144..3b98440a5 100644 --- a/tests/db/index/storage/mem_store_test.cc +++ b/tests/db/index/storage/mem_store_test.cc @@ -807,7 +807,7 @@ TEST_F(MemStoreTest, ThreadSafety) { std::vector> futures; for (int t = 0; t < num_threads; ++t) { - futures.push_back(std::async(std::launch::async, [this, t]() { + futures.push_back(std::async(std::launch::async, [this, t, inserts_per_thread]() { for (int i = 0; i < inserts_per_thread; ++i) { uint64_t doc_id = t * inserts_per_thread + i; store_->insert(CreateDoc(doc_id)); From 83a64230d08b3cc1f2b6a7c5986c3e90cab385a1 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:55:01 +0800 Subject: [PATCH 20/51] TODO: skip tests MSVC #define private public incompatibility (tests/db/index/CMakeLists.txt): Excluded 4 tests on MSVC (mmap_store_test, mem_store_test, inverted_indexer_util_test, segment_test) because MSVC encodes access specifiers in mangled names, making the #define private public hack fundamentally incompatible. --- tests/db/index/CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/db/index/CMakeLists.txt b/tests/db/index/CMakeLists.txt index 22a2932e1..555b92cf4 100644 --- a/tests/db/index/CMakeLists.txt +++ b/tests/db/index/CMakeLists.txt @@ -18,11 +18,25 @@ if(APPLE) endif() file(GLOB_RECURSE ALL_TEST_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *_test.cc) + +if(MSVC) + set(_MSVC_EXCLUDE_TESTS + "storage/mmap_store_test.cc" + "storage/mem_store_test.cc" + "column/inverted_column/inverted_indexer_util_test.cc" + "segment/segment_test.cc" + ) + foreach(_excl ${_MSVC_EXCLUDE_TESTS}) + list(FILTER ALL_TEST_SRCS EXCLUDE REGEX "${_excl}") + endforeach() +endif() + foreach(CC_SRCS ${ALL_TEST_SRCS}) get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE) cc_gmock( NAME ${CC_TARGET} STRICT LIBS zvec_db + zvec_ailego zvec_proto core_metric_static core_utility_static From c26de94ba7cf4bcbd4b12b2e82d970114182f007 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:56:24 +0800 Subject: [PATCH 21/51] TODO: may revert --- cmake/bazel.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/bazel.cmake b/cmake/bazel.cmake index f1effc6da..54da6c487 100644 --- a/cmake/bazel.cmake +++ b/cmake/bazel.cmake @@ -659,7 +659,7 @@ function(_target_link_libraries _NAME) endif() else() # Microsoft Visual C++ - list(APPEND LINK_LIBS /WHOLEARCHIVE:$) + list(APPEND MSVC_WHOLEARCHIVE_OPTS /WHOLEARCHIVE:$) get_target_property(OTHER_LINK_LIBS ${LIB} INTERFACE_LINK_LIBRARIES) if(OTHER_LINK_LIBS) foreach(OTHER_LIB ${OTHER_LINK_LIBS}) @@ -678,6 +678,9 @@ function(_target_link_libraries _NAME) endforeach() target_link_libraries(${_NAME} ${LINK_LIBS}) + if(MSVC_WHOLEARCHIVE_OPTS) + target_link_options(${_NAME} PRIVATE ${MSVC_WHOLEARCHIVE_OPTS}) + endif() if(LIBS_DEPS) add_dependencies(${_NAME} ${LIBS_DEPS}) target_include_directories(${_NAME} PRIVATE "${LIBS_INCS}") From 60da06efb86277274384d8362671507a33bad7ac Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:57:17 +0800 Subject: [PATCH 22/51] TODO: rocksdb can't find lz4, could be optimized --- thirdparty/rocksdb/CMakeLists.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/thirdparty/rocksdb/CMakeLists.txt b/thirdparty/rocksdb/CMakeLists.txt index 9d756bba8..ab1d6ec66 100644 --- a/thirdparty/rocksdb/CMakeLists.txt +++ b/thirdparty/rocksdb/CMakeLists.txt @@ -13,17 +13,37 @@ set(WITH_CORE_TOOLS OFF CACHE BOOL "build with ldb and sst_dump" FORCE) set(WITH_TOOLS OFF CACHE BOOL "build with tools" FORCE) set(WITH_LZ4 ON CACHE BOOL "build with lz4" FORCE) set(USE_RTTI ON CACHE BOOL "build with RTTI" FORCE) +set(ROCKSDB_SKIP_THIRDPARTY ON CACHE BOOL "skip thirdparty.inc" FORCE) set(FAIL_ON_WARNINGS OFF CACHE BOOL "build with no Werror" FORCE) set(PORTABLE ON CACHE BOOL "build a portable lib" FORCE) +set(lz4_INCLUDE_DIRS "${lz4_INCLUDE_DIR}" CACHE PATH "lz4 include directory" FORCE) +set(lz4_LIBRARIES "${lz4_LIBRARY}" CACHE FILEPATH "lz4 library" FORCE) +set(lz4_FOUND TRUE CACHE BOOL "lz4 found" FORCE) + +if(NOT TARGET lz4::lz4) + add_library(lz4::lz4 UNKNOWN IMPORTED GLOBAL) + set_target_properties(lz4::lz4 PROPERTIES + IMPORTED_LOCATION "${lz4_LIBRARY}" + IMPORTED_LOCATION_DEBUG "${lz4_LIBRARY}" + IMPORTED_LOCATION_RELEASE "${lz4_LIBRARY}" + IMPORTED_LOCATION_MINSIZEREL "${lz4_LIBRARY}" + IMPORTED_LOCATION_RELWITHDEBINFO "${lz4_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}" + ) + add_dependencies(lz4::lz4 Lz4.BUILD) +endif() + set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(rocksdb-8.1.1) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) + unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) get_target_property(rocksdb_SOURCE_DIR rocksdb SOURCE_DIR) set(ROCKSDB_INCLUDE_DIR ${rocksdb_SOURCE_DIR}/include) target_include_directories(rocksdb PUBLIC $) +target_include_directories(rocksdb PRIVATE ${lz4_INCLUDE_DIR}) add_dependencies(rocksdb Lz4.BUILD) From 3a267a044108cada86051ee68d42686509d0512f Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 15:58:20 +0800 Subject: [PATCH 23/51] third party Cmakelist: naming, compiling options --- thirdparty/antlr/CMakeLists.txt | 8 +++- thirdparty/arrow/CMakeLists.txt | 13 ++++- thirdparty/glog/CMakeLists.txt | 17 +++++++ thirdparty/googletest/CMakeLists.txt | 2 + thirdparty/lz4/CMakeLists.txt | 72 ++++++++++++++++++++-------- 5 files changed, 89 insertions(+), 23 deletions(-) diff --git a/thirdparty/antlr/CMakeLists.txt b/thirdparty/antlr/CMakeLists.txt index fe620ef02..dd3becdeb 100644 --- a/thirdparty/antlr/CMakeLists.txt +++ b/thirdparty/antlr/CMakeLists.txt @@ -2,13 +2,19 @@ set(ANTLR_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/antlr4) set(ANTLR_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/antlr4.patch) apply_patch_once("antlr4_fix" "${ANTLR_SRC_DIR}" "${ANTLR_PATCH}") +set(WITH_STATIC_CRT OFF CACHE BOOL "Use dynamic CRT for antlr4" FORCE) add_subdirectory(antlr4/runtime/Cpp/) add_library(antlr4 UNKNOWN IMPORTED GLOBAL) +if(MSVC) + set(_antlr4_lib_name "antlr4-runtime-static.lib") +else() + set(_antlr4_lib_name "libantlr4-runtime.a") +endif() set_target_properties( antlr4 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/runtime/src/" - IMPORTED_LOCATION "${EXTERNAL_LIB_DIR}/libantlr4-runtime.a" + IMPORTED_LOCATION "${EXTERNAL_LIB_DIR}/${_antlr4_lib_name}" ) add_dependencies(antlr4 antlr4_static) diff --git a/thirdparty/arrow/CMakeLists.txt b/thirdparty/arrow/CMakeLists.txt index a8129821a..b72f85c97 100644 --- a/thirdparty/arrow/CMakeLists.txt +++ b/thirdparty/arrow/CMakeLists.txt @@ -1,4 +1,4 @@ -set(ARROW_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0) +set(ARROW_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0) if(ANDROID) set(ARROW_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/arrow.android.patch) apply_patch_once("arrow_android_fix" "${ARROW_SRC_DIR}" "${ARROW_PATCH}") @@ -63,12 +63,16 @@ if(ANDROID) LOG_INSTALL ON ) else() + set(ARROW_EXTRA_CMAKE_ARGS "") + if(MSVC) + set(ARROW_EXTRA_CMAKE_ARGS -DCMAKE_C_FLAGS=/FS -DCMAKE_CXX_FLAGS=/FS) + endif() ExternalProject_Add( ARROW.BUILD PREFIX arrow SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 DOWNLOAD_COMMAND "" BUILD_IN_SOURCE false - CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} -DCMAKE_BUILD_TYPE=$ -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${ARROW_EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=$ -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" BUILD_COMMAND "${CMAKE_COMMAND}" --build . -j ${NPROC} --config $ INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local --config $ BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET} @@ -103,6 +107,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_ARROW}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends" + INTERFACE_COMPILE_DEFINITIONS "ARROW_STATIC" ) add_dependencies(Arrow::arrow_static ARROW.BUILD) @@ -112,6 +117,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_PARQUET}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static" + INTERFACE_COMPILE_DEFINITIONS "PARQUET_STATIC" ) add_dependencies(Arrow::parquet_static ARROW.BUILD) @@ -121,6 +127,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_COMPUTE}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static" + INTERFACE_COMPILE_DEFINITIONS "ARROW_COMPUTE_STATIC" ) add_dependencies(Arrow::arrow_compute ARROW.BUILD) @@ -130,6 +137,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_ACERO}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static;Arrow::arrow_compute" + INTERFACE_COMPILE_DEFINITIONS "ARROW_ACERO_STATIC" ) add_dependencies(Arrow::arrow_acero ARROW.BUILD) @@ -139,5 +147,6 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_ARROW_DATASET}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static;Arrow::arrow_compute;Arrow::arrow_acero" + INTERFACE_COMPILE_DEFINITIONS "ARROW_DS_STATIC" ) add_dependencies(Arrow::arrow_dataset ARROW.BUILD) diff --git a/thirdparty/glog/CMakeLists.txt b/thirdparty/glog/CMakeLists.txt index 611f18efc..e460aa681 100644 --- a/thirdparty/glog/CMakeLists.txt +++ b/thirdparty/glog/CMakeLists.txt @@ -3,7 +3,9 @@ set(WITH_GFLAGS OFF CACHE BOOL "Disable find_package(gflags) in glog" FORCE) set(WITH_UNWIND OFF CACHE BOOL "Disable find_package(unwind) in glog" FORCE) set(HAVE_LIB_GFLAGS TRUE CACHE BOOL "") +if(NOT MSVC) add_compile_options(-Wno-deprecated-declarations) +endif() set(GLOG_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/glog-0.5.0) if (ANDROID) @@ -14,6 +16,15 @@ else() apply_patch_once("glog_fix" "${GLOG_SRC_DIR}" "${GLOG_PATCH}") endif() +if(MSVC) + set(INTTYPES_FORMAT VC7) +# # Pre-populate glog type checks that may fail under MSVC in subdirectory builds. +# # MSVC has both uint16_t (via ) and unsigned __int16 (built-in). +# set(HAVE___UINT16 "2" CACHE INTERNAL "sizeof unsigned __int16" FORCE) +# set(HAVE_UINT16_T "2" CACHE INTERNAL "sizeof uint16_t" FORCE) +# set(HAVE_U_INT16_T "" CACHE INTERNAL "sizeof u_int16_t" FORCE) +endif() + set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(glog-0.5.0) @@ -22,6 +33,12 @@ unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) add_dependencies(glog gflags) +if(MSVC) + # Modern MSVC (1900+/VS2015+) provides snprintf in the CRT. + # Suppress glog's own declaration which conflicts with the SDK header. + target_compile_definitions(glog PRIVATE HAVE_SNPRINTF) +endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") target_compile_options(glog PRIVATE -Wno-sign-compare) endif() diff --git a/thirdparty/googletest/CMakeLists.txt b/thirdparty/googletest/CMakeLists.txt index e72a0c134..151302dc3 100644 --- a/thirdparty/googletest/CMakeLists.txt +++ b/thirdparty/googletest/CMakeLists.txt @@ -1,4 +1,6 @@ +if(NOT MSVC) add_compile_options(-Wno-deprecated-copy) +endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_compile_options(-Wno-maybe-uninitialized) endif() diff --git a/thirdparty/lz4/CMakeLists.txt b/thirdparty/lz4/CMakeLists.txt index 7db62d4f5..e66bdd101 100644 --- a/thirdparty/lz4/CMakeLists.txt +++ b/thirdparty/lz4/CMakeLists.txt @@ -3,8 +3,17 @@ set(lz4_LIBRARY_DIR "${EXTERNAL_BINARY_DIR}/usr/local/lib/") file(MAKE_DIRECTORY ${lz4_INCLUDE_DIR}) file(MAKE_DIRECTORY ${lz4_LIBRARY_DIR}) +if(MSVC) + set(LZ4_LIB_NAME lz4.lib) +else() + set(LZ4_LIB_NAME liblz4.a) +endif() + include(ExternalProject) +include(ProcessorCount) +ProcessorCount(NPROC) + set(_lz4_env "") if(ANDROID) string(REGEX REPLACE "^android-([0-9]+)$" "\\1" ANDROID_API_LEVEL "${ANDROID_PLATFORM}") @@ -45,32 +54,55 @@ else() list(APPEND _lz4_env "CFLAGS=-fPIC") endif() -ExternalProject_Add( - Lz4.BUILD - PREFIX lz4 - URL "${CMAKE_CURRENT_SOURCE_DIR}/lz4-1.9.4" - CONFIGURE_COMMAND "" - BUILD_COMMAND env ${_lz4_env} BUILD_SHARED=no make -j - INSTALL_COMMAND make DESTDIR=${EXTERNAL_BINARY_DIR} BUILD_SHARED=no install - BUILD_IN_SOURCE ON - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON - LOG_INSTALL ON - BUILD_BYPRODUCTS ${lz4_LIBRARY_DIR}/liblz4.a -) - +if(MSVC) + ExternalProject_Add( + Lz4.BUILD + PREFIX lz4 + URL "${CMAKE_CURRENT_SOURCE_DIR}/lz4-1.9.4" + SOURCE_SUBDIR build/cmake + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${EXTERNAL_BINARY_DIR}/usr/local + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF + -DBUILD_STATIC_LIBS=ON + -DLZ4_BUILD_CLI=OFF + -DLZ4_BUILD_LEGACY_LZ4C=OFF + BUILD_COMMAND "${CMAKE_COMMAND}" --build . --config $ -j + INSTALL_COMMAND "${CMAKE_COMMAND}" --install . --config $ + BUILD_IN_SOURCE OFF + LOG_DOWNLOAD ON + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON + BUILD_BYPRODUCTS ${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME} + ) +else() + ExternalProject_Add( + Lz4.BUILD + PREFIX lz4 + URL "${CMAKE_CURRENT_SOURCE_DIR}/lz4-1.9.4" + CONFIGURE_COMMAND "" + BUILD_COMMAND env ${_lz4_env} BUILD_SHARED=no make -j + INSTALL_COMMAND make DESTDIR=${EXTERNAL_BINARY_DIR} BUILD_SHARED=no install + BUILD_IN_SOURCE ON + LOG_DOWNLOAD ON + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON + BUILD_BYPRODUCTS ${lz4_LIBRARY_DIR}/liblz4.a + ) +endif() add_library(lz4 STATIC IMPORTED GLOBAL) set_target_properties( - lz4 PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}" - IMPORTED_LOCATION "${lz4_LIBRARY_DIR}/liblz4.a" + lz4 PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}" + IMPORTED_LOCATION "${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME}" ) add_dependencies(lz4 Lz4.BUILD) set(lz4_FOUND TRUE PARENT_SCOPE) -set(lz4_LIBRARY ${lz4_LIBRARY_DIR}/liblz4.a PARENT_SCOPE) -set(lz4_LIBRARIES ${lz4_LIBRARY_DIR}/liblz4.a PARENT_SCOPE) +set(lz4_LIBRARY ${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME} PARENT_SCOPE) +set(lz4_LIBRARIES ${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME} PARENT_SCOPE) set(lz4_INCLUDE_DIR "${EXTERNAL_BINARY_DIR}/usr/local/include" PARENT_SCOPE) set(lz4_INCLUDE_DIRS "${EXTERNAL_BINARY_DIR}/usr/local/include" PARENT_SCOPE) set(lz4_VERSION 1.9.4 PARENT_SCOPE) From c575663980f9d93eeb301314fea9d680f4afcfa8 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 16:00:42 +0800 Subject: [PATCH 24/51] TODO: designated init & c++ 20, may revert --- CMakeLists.txt | 10 +++++++++- .../index/column/vector_column/engine_helper.hpp | 14 ++++++++------ src/db/index/segment/segment.cc | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52a597548..6f4e65ca6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,15 @@ cmake_minimum_required(VERSION 3.13) cmake_policy(SET CMP0077 NEW) project(zvec) -set(CC_CXX_STANDARD 17) +#TODO(windows): maybe removed, +# to handle : +#zvec\tests\db\index\column\vector_column_indexer_test.cc(127,43): error C7555: use of designated initializers requires at least '/std:c++20' +# right now +if(MSVC) + set(CC_CXX_STANDARD 20) +else() + set(CC_CXX_STANDARD 17) +endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type") diff --git a/src/db/index/column/vector_column/engine_helper.hpp b/src/db/index/column/vector_column/engine_helper.hpp index de1cfc6c0..74359d79e 100644 --- a/src/db/index/column/vector_column/engine_helper.hpp +++ b/src/db/index/column/vector_column/engine_helper.hpp @@ -109,12 +109,14 @@ class ProximaEngineHelper { engine_query_param->is_linear = db_query_params.query_params->is_linear(); } if (db_query_params.refiner_param) { - engine_query_param->refiner_param = - std::make_shared( - core_interface::RefinerParam{ - .scale_factor_ = db_query_params.refiner_param->scale_factor_, - .reference_index = - db_query_params.refiner_param->reference_indexer->index}); + { + core_interface::RefinerParam rp; + rp.scale_factor_ = db_query_params.refiner_param->scale_factor_; + rp.reference_index = + db_query_params.refiner_param->reference_indexer->index; + engine_query_param->refiner_param = + std::make_shared(rp); + } } return engine_query_param; diff --git a/src/db/index/segment/segment.cc b/src/db/index/segment/segment.cc index 517215a3e..eae26602b 100644 --- a/src/db/index/segment/segment.cc +++ b/src/db/index/segment/segment.cc @@ -608,7 +608,7 @@ Status SegmentImpl::InsertVector(VectorColumnIndexer::Ptr &indexer, (void *)sparse_value.data()}; } else { vector_data.vector = - vector_column_params::DenseVector{.data = value.value().data()}; + vector_column_params::DenseVector{value.value().data()}; } auto &mem_block_meta = segment_meta_->writing_forward_block().value(); From 539dc208cef8bb5df3bf5c024d61269cd141a531 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 16:01:30 +0800 Subject: [PATCH 25/51] fix compiler options --- CMakeLists.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f4e65ca6..b513fd512 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,8 +11,16 @@ else() set(CC_CXX_STANDARD 17) endif() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type") +if(MSVC) + set(INTTYPES_FORMAT VC7) + add_compile_options(/FS) #handle .pdb + add_compile_options(/EHsc) # def c++ exception behavior + add_compile_options(/Zc:preprocessor) + add_compile_options(/we4716) # -Werror=return-type +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type") +endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-as-needed") From c92a51c8557edc2734d2bd354170d14c654e969a Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 16:01:57 +0800 Subject: [PATCH 26/51] TODO: ignore some warning --- CMakeLists.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index b513fd512..3ec39ee3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,25 @@ if(MSVC) add_compile_options(/EHsc) # def c++ exception behavior add_compile_options(/Zc:preprocessor) add_compile_options(/we4716) # -Werror=return-type + + + ###### reduce output length to make vibe coding work better :), which should be removed or solved later + add_compile_options(/wd4267 /wd4244) + add_compile_options(/wd4146) # unary minus operator applied to unsigned type #usage: uint32_t seg_id_{-1U}; + add_compile_options(/wd4310) # warning C4310: cast truncates constant value + add_compile_options(/wd4530) # warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc + add_compile_options(/wd4251) #protobuf + add_compile_options(/wd4245 /wd4334 /wd4702 /wd4305) + + # //warning C4334: '<<': result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?) + # keeps |= (1 << k); + add_compile_options(/wd4334) + ###### + + #TODO(windows): to be verified + add_definitions(-DARROW_STATIC -DPARQUET_STATIC -DARROW_ACERO_STATIC -DARROW_DS_STATIC -DARROW_COMPUTE_STATIC) + + else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type") From de9724b58700ec838342097a7995477bd40bac41 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Wed, 11 Mar 2026 16:15:47 +0800 Subject: [PATCH 27/51] TODO: to verify sparsehash's patch --- thirdparty/sparsehash/CMakeLists.txt | 5 +++++ thirdparty/sparsehash/sparsehash.windows.patch | 13 +++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 thirdparty/sparsehash/sparsehash.windows.patch diff --git a/thirdparty/sparsehash/CMakeLists.txt b/thirdparty/sparsehash/CMakeLists.txt index 9743c31aa..c389505fc 100644 --- a/thirdparty/sparsehash/CMakeLists.txt +++ b/thirdparty/sparsehash/CMakeLists.txt @@ -5,6 +5,11 @@ if(NOT EXISTS "${SPARSE_DST}") file(COPY "${SPARSE_SRC}" DESTINATION "${DESTINATION_DIR}") endif() +if(MSVC) + set(SPARSEHASH_WINDOWS_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/sparsehash.windows.patch) + apply_patch_once("sparsehash.windows.patch" "${DESTINATION_DIR}" "${SPARSEHASH_WINDOWS_PATCH}") +endif() + add_library(sparsehash INTERFACE) target_include_directories( sparsehash INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/sparsehash-2.0.4/src/" diff --git a/thirdparty/sparsehash/sparsehash.windows.patch b/thirdparty/sparsehash/sparsehash.windows.patch new file mode 100644 index 000000000..99aebf69d --- /dev/null +++ b/thirdparty/sparsehash/sparsehash.windows.patch @@ -0,0 +1,13 @@ +diff --git a/src/sparsehash/internal/hashtable-common.h b/src/sparsehash/internal/hashtable-common.h +index bac2b88..59c752c 100644 +--- a/src/sparsehash/internal/hashtable-common.h ++++ b/src/sparsehash/internal/hashtable-common.h +@@ -51,7 +51,7 @@ _START_GOOGLE_NAMESPACE_ + + template struct SparsehashCompileAssert { }; + #define SPARSEHASH_COMPILE_ASSERT(expr, msg) \ +- __attribute__((unused)) typedef SparsehashCompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] ++ static_assert(bool(expr), #msg) + + namespace sparsehash_internal { + From 3dab143fd1e1575756406399bbe70053a15fbce5 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:26:13 +0800 Subject: [PATCH 28/51] msvc debug will validate the boundary --- src/core/algorithm/cluster/kmeans_cluster.cc | 6 ++++-- src/core/algorithm/cluster/opt_kmeans_cluster.cc | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/core/algorithm/cluster/kmeans_cluster.cc b/src/core/algorithm/cluster/kmeans_cluster.cc index 507159b86..e73a803a6 100644 --- a/src/core/algorithm/cluster/kmeans_cluster.cc +++ b/src/core/algorithm/cluster/kmeans_cluster.cc @@ -891,8 +891,10 @@ void KmeansCluster::update_features_thread(size_t column, for (size_t i = column, j = 0; i < shard_cluster_features_.size(); i += cluster_count) { const std::vector &it = shard_cluster_features_[i]; - std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); - j += it.size(); + if (!it.empty()) { + std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); + j += it.size(); + } } } diff --git a/src/core/algorithm/cluster/opt_kmeans_cluster.cc b/src/core/algorithm/cluster/opt_kmeans_cluster.cc index 49ae8f5f3..aa2269f4d 100644 --- a/src/core/algorithm/cluster/opt_kmeans_cluster.cc +++ b/src/core/algorithm/cluster/opt_kmeans_cluster.cc @@ -327,8 +327,10 @@ void OptKmeansAlgorithm::update_features_thread( for (size_t i = column, j = 0; i < shard_cluster_features_.size(); i += cluster_count) { const std::vector &it = shard_cluster_features_[i]; - std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); - j += it.size(); + if (!it.empty()) { + std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); + j += it.size(); + } } } From 52170b985e08c596cdce2f7760e6b5cd199ac308 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:28:08 +0800 Subject: [PATCH 29/51] fix -1UL: On MSVC, unsigned long is 32-bit, so -1UL is 0xFFFFFFFF not 0xFFFFFFFFFFFFFFFF. --- src/core/algorithm/hnsw/hnsw_streamer_entity.cc | 6 ++++-- src/core/algorithm/hnsw/hnsw_streamer_entity.h | 6 +++--- .../algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc | 8 ++++---- .../algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h | 6 +++--- src/db/common/constants.h | 2 +- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc index 71e2e477b..6562f1263 100644 --- a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc @@ -485,7 +485,9 @@ int HnswStreamerEntity::check_hnsw_index(const HNSWHeader *hd) const { int HnswStreamerEntity::add_vector(level_t level, key_t key, const void *vec, node_id_t *id) { Chunk::Pointer node_chunk; - size_t chunk_offset = -1UL; + // On MSVC, unsigned long is 32-bit, so -1UL is 0xFFFFFFFF not 0xFFFFFFFFFFFFFFFF. + size_t chunk_offset = static_cast(-1); + std::lock_guard lock(mutex_); // duplicate check @@ -558,7 +560,7 @@ int HnswStreamerEntity::add_vector(level_t level, key_t key, const void *vec, int HnswStreamerEntity::add_vector_with_id(level_t level, node_id_t id, const void *vec) { Chunk::Pointer node_chunk; - size_t chunk_offset = -1UL; + size_t chunk_offset = static_cast(-1); key_t key = id; std::lock_guard lock(mutex_); diff --git a/src/core/algorithm/hnsw/hnsw_streamer_entity.h b/src/core/algorithm/hnsw/hnsw_streamer_entity.h index 1a01b1416..9e3a95cfd 100644 --- a/src/core/algorithm/hnsw/hnsw_streamer_entity.h +++ b/src/core/algorithm/hnsw/hnsw_streamer_entity.h @@ -331,10 +331,10 @@ class HnswStreamerEntity : public HnswEntity { return 0; } Chunk::Pointer chunk; - uint64_t chunk_offset = -1UL; + uint64_t chunk_offset = UINT64_MAX; size_t neighbors_size = get_total_upper_neighbors_size(level); - uint64_t chunk_index = upper_neighbor_chunks_.size() - 1UL; - if (chunk_index == -1UL || + uint64_t chunk_index = upper_neighbor_chunks_.size() - 1ULL; + if (chunk_index == UINT64_MAX || (upper_neighbor_chunks_[chunk_index]->padding_size() < neighbors_size)) { // no space left and need to alloc chunk_index++; diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc index 6b429acb9..811347f61 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc @@ -607,8 +607,8 @@ int HnswSparseStreamerEntity::add_vector(level_t level, key_t key, SparseChunk::Pointer node_chunk; SparseChunk::Pointer sparse_node_chunk; - size_t chunk_offset = -1UL; - size_t sparse_chunk_offset = -1UL; + size_t chunk_offset = static_cast(-1); + size_t sparse_chunk_offset = static_cast(-1); std::lock_guard lock(mutex_); // duplicate check @@ -744,8 +744,8 @@ int HnswSparseStreamerEntity::add_vector_with_id(level_t level, node_id_t id, key_t key = id; SparseChunk::Pointer node_chunk; SparseChunk::Pointer sparse_node_chunk; - size_t chunk_offset = -1UL; - size_t sparse_chunk_offset = -1UL; + size_t chunk_offset = static_cast(-1); + size_t sparse_chunk_offset = static_cast(-1); // allocat sparse chunk uint32_t sparse_vector_len = sparse_vec.size(); diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h index 84087afc3..67a7d89ad 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h @@ -360,10 +360,10 @@ class HnswSparseStreamerEntity : public HnswSparseEntity { return 0; } SparseChunk::Pointer chunk; - uint64_t chunk_offset = -1UL; + uint64_t chunk_offset = UINT64_MAX; size_t neighbors_size = get_total_upper_neighbors_size(level); - uint64_t chunk_index = upper_neighbor_chunks_.size() - 1UL; - if (chunk_index == -1UL || + uint64_t chunk_index = upper_neighbor_chunks_.size() - 1ULL; + if (chunk_index == UINT64_MAX || (upper_neighbor_chunks_[chunk_index]->padding_size() < neighbors_size)) { // no space left and need to alloc chunk_index++; diff --git a/src/db/common/constants.h b/src/db/common/constants.h index 39b13f445..339664228 100644 --- a/src/db/common/constants.h +++ b/src/db/common/constants.h @@ -24,7 +24,7 @@ const float DEFAULT_MEMORY_LIMIT_RATIO = 0.8f; const uint32_t MIN_MEMORY_LIMIT_BYTES = 100 * 1024 * 1024; -const uint64_t INVALID_DOC_ID = -1UL; +const uint64_t INVALID_DOC_ID = UINT64_MAX; const std::string LOCAL_ROW_ID = "_zvec_row_id_"; From 47df026b7b5ac34993d886ed63799834e2e03ef2 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:28:39 +0800 Subject: [PATCH 30/51] fix flexible structure --- src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h index 1128ae06e..65e918372 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h @@ -161,7 +161,7 @@ struct HNSWSparseHeader { struct NeighborsHeader { uint32_t neighbor_cnt; - node_id_t neighbors[1]; + node_id_t neighbors[]; }; struct Neighbors { From 1f87aa7fa4135c0536fc5a47abb0d47779915193 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:29:11 +0800 Subject: [PATCH 31/51] TODO: verify the fix of mmap alignment --- src/ailego/io/file.cc | 58 ++++++++++++++++++++++++++++-------- tests/ailego/io/file_test.cc | 2 +- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/ailego/io/file.cc b/src/ailego/io/file.cc index 79e58a1c1..cae474385 100644 --- a/src/ailego/io/file.cc +++ b/src/ailego/io/file.cc @@ -639,14 +639,28 @@ ssize_t File::offset(void) const { } void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { - LARGE_INTEGER file_size; - file_size.QuadPart = len; +// Root cause: Windows MapViewOfFile requires the file offset to be aligned to +// the allocation granularity (64 KB), but segment offsets were only +// page-aligned (4 KB). Also, CreateFileMapping was using len instead of +// off + len as the max size. +// +// Fix: Align the view offset down to allocation granularity, adjust the map +// length, and return base + excess. MemoryUnmap recovers the base by rounding +// down to granularity. + + SYSTEM_INFO si; + GetSystemInfo(&si); + DWORD granularity = si.dwAllocationGranularity; + ssize_t aligned_off = (off / (ssize_t)granularity) * (ssize_t)granularity; + size_t excess = (size_t)(off - aligned_off); + + LARGE_INTEGER max_size; + max_size.QuadPart = off + len; - // Create map object HANDLE file_mapping = CreateFileMapping( handle, nullptr, ((opts & File::MMAP_READONLY) ? PAGE_READONLY : PAGE_READWRITE), - file_size.HighPart, file_size.LowPart, nullptr); + max_size.HighPart, max_size.LowPart, nullptr); ailego_null_if_false(file_mapping != nullptr); DWORD desired_access = FILE_MAP_READ; @@ -656,14 +670,18 @@ void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { if (!(opts & File::MMAP_SHARED)) { desired_access |= FILE_MAP_COPY; } - file_size.QuadPart = off; - // Map the whole file to memory and close handle - void *addr = MapViewOfFile(file_mapping, desired_access, file_size.HighPart, - file_size.LowPart, 0); + LARGE_INTEGER view_offset; + view_offset.QuadPart = aligned_off; + size_t view_len = len + excess; + + void *base = MapViewOfFile(file_mapping, desired_access, + view_offset.HighPart, view_offset.LowPart, + view_len); CloseHandle(file_mapping); - ailego_null_if_false(addr); + ailego_null_if_false(base); + void *addr = (char *)base + excess; if (opts & File::MMAP_LOCKED) { VirtualLock(addr, len); } @@ -673,8 +691,17 @@ void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { return addr; } -void *File::MemoryMap(size_t, int) { - return nullptr; +void *File::MemoryMap(size_t len, int opts) { + void *addr = + VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + ailego_null_if_false(addr); + if (opts & File::MMAP_LOCKED) { + VirtualLock(addr, len); + } + if (opts & File::MMAP_WARMUP) { + File::MemoryWarmup(addr, len); + } + return addr; } void *File::MemoryRemap(void *, size_t, void *, size_t) { @@ -683,7 +710,14 @@ void *File::MemoryRemap(void *, size_t, void *, size_t) { void File::MemoryUnmap(void *addr, size_t /*len*/) { ailego_return_if_false(addr); - UnmapViewOfFile(addr); + MEMORY_BASIC_INFORMATION mbi; + if (VirtualQuery(addr, &mbi, sizeof(mbi))) { + if (mbi.Type == MEM_MAPPED) { + UnmapViewOfFile(mbi.AllocationBase); + } else { + VirtualFree(mbi.AllocationBase, 0, MEM_RELEASE); + } + } } bool File::MemoryFlush(void *addr, size_t /*len*/) { diff --git a/tests/ailego/io/file_test.cc b/tests/ailego/io/file_test.cc index b71873616..84b916ce5 100644 --- a/tests/ailego/io/file_test.cc +++ b/tests/ailego/io/file_test.cc @@ -233,7 +233,7 @@ TEST(File, MemoryMap) { const char *file_path = "file_map_testing.tmp"; size_t file_size = 2u * 1024u * 1024u + 12u * 1024; size_t map_offset = MemoryHelper::PageSize() * 16; - size_t map_size = file_size - MemoryHelper::PageSize(); + size_t map_size = file_size - map_offset; File::Delete(file_path); EXPECT_FALSE(File::IsRegular(file_path)); From d9abe332f12993111e16ca1f6dcadb2bc53cfdf6 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:30:56 +0800 Subject: [PATCH 32/51] fix ctest failure by storage.close --- tests/core/algorithm/flat/flat_streamer_buffer_test.cpp | 4 +++- tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp | 1 + tests/core/algorithm/flat/flat_streamer_test.cc | 3 ++- .../algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc | 2 ++ tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp | 1 + 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp index e9988692a..eabaeb3bd 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp @@ -79,7 +79,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); - + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); @@ -201,6 +201,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); ElapsedTime elapsed_time; IndexStreamer::Pointer read_streamer = @@ -295,6 +296,7 @@ TEST_F(FlatStreamerTest, TestBufferStorage) { } streamer->flush(0UL); streamer.reset(); + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp index c919e9fee..6648c2bae 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp @@ -79,6 +79,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); diff --git a/tests/core/algorithm/flat/flat_streamer_test.cc b/tests/core/algorithm/flat/flat_streamer_test.cc index 1ddbd2d9b..473bfb6b2 100644 --- a/tests/core/algorithm/flat/flat_streamer_test.cc +++ b/tests/core/algorithm/flat/flat_streamer_test.cc @@ -1148,7 +1148,8 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) { } write_streamer->flush(0UL); write_streamer->close(); - write_streamer.reset(); // + write_streamer.reset(); + write_storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc index 11c84e8a5..a3385afad 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc @@ -137,6 +137,8 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + write_storage->close(); + write_storage.reset(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatSparseStreamer"); diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp index bd96789a4..5131d43a8 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp @@ -204,6 +204,7 @@ TEST_F(HnswStreamerTest, TestHnswSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); ElapsedTime elapsed_time; IndexStreamer::Pointer read_streamer = From 689bb0468cfb6797b447151baf5b77b7405b20a8 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:33:32 +0800 Subject: [PATCH 33/51] fix mutex by atomic: TestDumpIndexAndAdd test uses mutex.unlock() from a non-owning thread, which is undefined behavior caught by MSVC's debug assertions --- .../core/algorithm/hnsw/hnsw_streamer_test.cc | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc index 577ab258d..9fa26f477 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc @@ -1651,10 +1651,12 @@ TEST_F(HnswStreamerTest, TestDumpIndexAndAdd) { IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); ASSERT_NE(nullptr, ctx); int code = 0; - std::mutex mutex; - auto addVector = [&](int a, int b) { + std::atomic async_started{false}; + auto addVector = [&](int a, int b, bool signal_start) { int success = 0; - mutex.unlock(); + if (signal_start) { + async_started.store(true, std::memory_order_release); + } for (int i = a; i < b; i++) { for (size_t j = 0; j < dim; ++j) { vec[j] = i; @@ -1671,18 +1673,17 @@ TEST_F(HnswStreamerTest, TestDumpIndexAndAdd) { } std::cout << "addVector: " << success << " success" << std::endl; }; - mutex.lock(); - addVector(0, 2000); - mutex.lock(); - auto t2 = std::async(std::launch::async, addVector, 2000, 3000); + addVector(0, 2000, false); + auto t2 = std::async(std::launch::async, addVector, 2000, 3000, true); auto path1 = dir_ + "/dumpIndex1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); - mutex.lock(); // sync: wait addVector start and release lock + while (!async_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } auto test_dumper = std::make_shared(); ASSERT_EQ(0, streamer->dump(test_dumper)); - mutex.unlock(); ASSERT_EQ(0, streamer->dump(dumper1)); ASSERT_EQ(0, dumper1->close()); t2.get(); From 9b7321f7f03246b0e5b27b39b10a60d857ba5e16 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:34:09 +0800 Subject: [PATCH 34/51] storage close --- .../algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp index 8c7aadb24..aebbaf9d8 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp @@ -142,6 +142,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + write_storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("HnswSparseStreamer"); @@ -270,6 +271,7 @@ TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + write_storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("HnswSparseStreamer"); From d4d4a0be0f184205b5748c2edf71dade93f81bf4 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:37:44 +0800 Subject: [PATCH 35/51] TODO: unify the file cleanup of UTs --- tests/ailego/buffer/buffer_manager_test.cc | 5 +- .../core/algorithm/flat/flat_builder_test.cc | 5 +- .../flat/flat_streamer_buffer_test.cpp | 20 +++-- .../flat/flat_streamer_buffer_time_test.cpp | 20 +++-- .../core/algorithm/flat/flat_streamer_test.cc | 37 +++++---- .../flat_sparse/flat_sparse_builder_test.cc | 5 +- .../flat_sparse/flat_sparse_searcher_test.cc | 13 ++-- .../flat_sparse_streamer_buffer_test.cc | 20 +++-- .../flat_sparse/flat_sparse_streamer_test.cc | 38 ++++++---- .../core/algorithm/hnsw/hnsw_builder_test.cc | 5 +- .../algorithm/hnsw/hnsw_searcher_test.cpp | 7 +- .../hnsw/hnsw_streamer_buffer_test.cpp | 21 +++-- .../core/algorithm/hnsw/hnsw_streamer_test.cc | 61 ++++++++------- .../hnsw_sparse/hnsw_sparse_builder_test.cc | 10 ++- .../hnsw_sparse/hnsw_sparse_searcher_test.cpp | 7 +- .../hnsw_sparse_streamer_buffer_test.cpp | 20 +++-- .../hnsw_sparse/hnsw_sparse_streamer_test.cc | 76 ++++++++++--------- tests/core/interface/index_interface_test.cc | 58 +++++++------- tests/db/index/CMakeLists.txt | 6 ++ ...erted_column_indexer_array_numbers_test.cc | 9 +-- .../inverted_column_indexer_bool_test.cc | 9 +-- ...rted_column_indexer_cyclic_numbers_test.cc | 9 +-- ..._column_indexer_sequential_numbers_test.cc | 9 +-- .../inverted_column_indexer_string_test.cc | 9 +-- .../inverted_indexer_util_test.cc | 9 +-- tests/db/index/storage/wal_file_test.cc | 8 +- 26 files changed, 272 insertions(+), 224 deletions(-) diff --git a/tests/ailego/buffer/buffer_manager_test.cc b/tests/ailego/buffer/buffer_manager_test.cc index 637c3ee9f..a9f4ae4c7 100644 --- a/tests/ailego/buffer/buffer_manager_test.cc +++ b/tests/ailego/buffer/buffer_manager_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -35,9 +36,7 @@ class BufferManagerTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); if (!File::MakePath(working_dir)) { LOG_ERROR("Failed to create working directory."); diff --git a/tests/core/algorithm/flat/flat_builder_test.cc b/tests/core/algorithm/flat/flat_builder_test.cc index ac3c3b2d7..41696b72c 100644 --- a/tests/core/algorithm/flat/flat_builder_test.cc +++ b/tests/core/algorithm/flat/flat_builder_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -54,9 +55,7 @@ void FlatBuilderTest::SetUp(void) { //! self-check column-major and row-major search. void FlatBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::ailego::FileHelper::RemovePath(dir_.c_str()); } void build_process(IndexBuilder::Pointer &builder, diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp index eabaeb3bd..6dfa9eb10 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,6 +8,7 @@ #include #include #include +#include "zvec/ailego/utility/file_helper.h" using namespace zvec::core; using namespace zvec::ailego; @@ -30,7 +32,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("streamer_test/"); +std::string FlatStreamerTest::dir_("flat_streamer_buffer_test_dir"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -38,15 +40,19 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void FlatStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(FlatStreamerTest, TestLinearSearch) { diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp index 6648c2bae..69dbe970b 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,6 +8,7 @@ #include #include #include +#include "zvec/ailego/utility/file_helper.h" using namespace zvec::core; using namespace zvec::ailego; @@ -30,7 +32,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("streamer_test/"); +std::string FlatStreamerTest::dir_("flat_streamer_buffer_time_test_dir"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -38,15 +40,19 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void FlatStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(FlatStreamerTest, TestLinearSearchMMap) { diff --git a/tests/core/algorithm/flat/flat_streamer_test.cc b/tests/core/algorithm/flat/flat_streamer_test.cc index 473bfb6b2..7e46b4d95 100644 --- a/tests/core/algorithm/flat/flat_streamer_test.cc +++ b/tests/core/algorithm/flat/flat_streamer_test.cc @@ -24,6 +24,7 @@ #include #include #include "algorithm/flat/flat_utility.h" +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -47,7 +48,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("streamer_test/"); +std::string FlatStreamerTest::dir_("flat_streamer_test_dir"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -55,15 +56,19 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void FlatStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(FlatStreamerTest, TestAddVector) { @@ -362,9 +367,9 @@ TEST_F(FlatStreamerTest, TestOpenClose) { ASSERT_NE(nullptr, storage2); Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "TestOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "/TestOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "TestOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "/TestOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -520,7 +525,7 @@ TEST_F(FlatStreamerTest, TestMultiThread) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -633,7 +638,7 @@ TEST_F(FlatStreamerTest, TestConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -739,7 +744,7 @@ TEST_F(FlatStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessFilter", true)); ASSERT_EQ(0, streamer->open(storage)); @@ -814,7 +819,7 @@ TEST_F(FlatStreamerTest, TestMaxIndexSize) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessMaxIndexSize", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessMaxIndexSize", true)); ASSERT_EQ(0, streamer->open(storage)); size_t vsz0 = 0; @@ -863,7 +868,7 @@ TEST_F(FlatStreamerTest, TestCleanUp) { ASSERT_NE(nullptr, storage1); Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "TessKnnCluenUp1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "/TessKnnCluenUp1", true)); Params params; constexpr size_t static dim1 = 32; IndexMeta meta1(IndexMeta::DataType::DT_FP32, dim1); @@ -880,7 +885,7 @@ TEST_F(FlatStreamerTest, TestCleanUp) { auto storage2 = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage2); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "TessKnnCluenUp2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "/TessKnnCluenUp2", true)); constexpr size_t static dim2 = 64; IndexMeta meta2(IndexMeta::DataType::DT_FP32, dim2); meta2.set_metric("SquaredEuclidean", 0, Params()); @@ -903,7 +908,7 @@ TEST_F(FlatStreamerTest, TestBloomFilter) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestBloomFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestBloomFilter", true)); Params params; ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc index af770255f..f175c5d2f 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include "zvec/ailego/utility/file_helper.h" using namespace zvec::core; using namespace zvec::ailego; @@ -46,9 +47,7 @@ void FlatSparseBuilderTest::SetUp(void) { } void FlatSparseBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + zvec::ailego::FileHelper::RemovePath(_dir.c_str()); } TEST_F(FlatSparseBuilderTest, TestGeneral) { diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc index f0553dd89..41f587281 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc @@ -22,6 +22,7 @@ #include #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" +#include "zvec/ailego/utility/file_helper.h" using namespace zvec::core; using namespace zvec::ailego; @@ -47,7 +48,7 @@ class FlatSparseSearcherTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatSparseSearcherTest::dir_("searcher_test/"); +std::string FlatSparseSearcherTest::dir_("searcher_test"); std::shared_ptr FlatSparseSearcherTest::index_meta_ptr_; void FlatSparseSearcherTest::generate_sparse_data( @@ -87,15 +88,11 @@ void FlatSparseSearcherTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::ailego::FileHelper::RemovePath(dir_.c_str()); } void FlatSparseSearcherTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::ailego::FileHelper::RemovePath(dir_.c_str()); } TEST_F(FlatSparseSearcherTest, TestGeneral) { @@ -662,7 +659,7 @@ TEST_F(FlatSparseSearcherTest, TestMultiThread) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc index a3385afad..915238265 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include "zvec/ailego/utility/file_helper.h" using namespace std; using namespace testing; @@ -48,7 +50,7 @@ class FlatSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string FlatSparseStreamerTest::dir_("FlatSparseStreamerTest/"); +std::string FlatSparseStreamerTest::dir_("FlatSparseStreamerTest"); shared_ptr FlatSparseStreamerTest::index_meta_ptr_; void FlatSparseStreamerTest::generate_sparse_data( @@ -86,15 +88,19 @@ void FlatSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void FlatSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(FlatSparseStreamerTest, TestGeneral) { diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc index c6f2e99e7..07205d7ed 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -21,6 +22,7 @@ #include #include #include +#include "zvec/ailego/utility/file_helper.h" #include #include #include @@ -49,7 +51,7 @@ class FlatSparseStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatSparseStreamerTest::dir_("streamer_test/"); +std::string FlatSparseStreamerTest::dir_("flat_sparse_streamer_test_dir"); std::shared_ptr FlatSparseStreamerTest::index_meta_ptr_; void FlatSparseStreamerTest::generate_sparse_data( @@ -89,15 +91,19 @@ void FlatSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void FlatSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(FlatSparseStreamerTest, TestGeneral) { @@ -473,9 +479,9 @@ TEST_F(FlatSparseStreamerTest, TestOpenAndClose) { ASSERT_NE(nullptr, storage2); Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "TessOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "/TessOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "TessOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "/TessOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -662,7 +668,7 @@ TEST_F(FlatSparseStreamerTest, TestMultiThread) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -810,7 +816,7 @@ TEST_F(FlatSparseStreamerTest, TestConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -940,7 +946,7 @@ TEST_F(FlatSparseStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestFilter", true)); ASSERT_EQ(0, streamer->open(storage)); size_t cnt = 100UL; @@ -1026,7 +1032,7 @@ TEST_F(FlatSparseStreamerTest, TestProvider) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestProvider.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestProvider.index", true)); Params params; ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1385,9 +1391,9 @@ TEST_F(FlatSparseStreamerTest, TestSharedContext) { streamer->open(storage); return streamer; }; - auto streamer1 = create_streamer(dir_ + "TestSharedContext.index1"); - auto streamer2 = create_streamer(dir_ + "TestSharedContext.index2"); - auto streamer3 = create_streamer(dir_ + "TestSharedContext.index3"); + auto streamer1 = create_streamer(dir_ + "/TestSharedContext.index1"); + auto streamer2 = create_streamer(dir_ + "/TestSharedContext.index2"); + auto streamer3 = create_streamer(dir_ + "/TestSharedContext.index3"); srand(Realtime::MilliSeconds()); IndexQueryMeta qmeta(IndexMeta::DT_FP32); diff --git a/tests/core/algorithm/hnsw/hnsw_builder_test.cc b/tests/core/algorithm/hnsw/hnsw_builder_test.cc index 402ae9720..e50952ae3 100644 --- a/tests/core/algorithm/hnsw/hnsw_builder_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_builder_test.cc @@ -19,6 +19,7 @@ #include #include #include "zvec/core/framework/index_framework.h" +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -52,9 +53,7 @@ void HnswBuilderTest::SetUp(void) { } void HnswBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + zvec::ailego::FileHelper::RemovePath(_dir.c_str()); } TEST_F(HnswBuilderTest, TestGeneral) { diff --git a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp index d3a7004fa..aca6169ff 100644 --- a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp @@ -24,6 +24,7 @@ #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" #include "hnsw_params.h" +#include "zvec/ailego/utility/file_helper.h" using namespace std; using namespace testing; @@ -48,7 +49,7 @@ class HnswSearcherTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswSearcherTest::_dir("HnswSearcherTest/"); +std::string HnswSearcherTest::_dir("HnswSearcherTest"); shared_ptr HnswSearcherTest::_index_meta_ptr; void HnswSearcherTest::SetUp(void) { @@ -58,9 +59,7 @@ void HnswSearcherTest::SetUp(void) { } void HnswSearcherTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + zvec::ailego::FileHelper::RemovePath(_dir.c_str()); } TEST_F(HnswSearcherTest, TestRnnSearch) { diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp index 5131d43a8..51244d9a6 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include "zvec/ailego/utility/file_helper.h" using namespace zvec::core; using namespace zvec::ailego; @@ -31,7 +33,7 @@ class HnswStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string HnswStreamerTest::dir_("streamer_test/"); +std::string HnswStreamerTest::dir_("hnsw_streamer_buffer_test_dir"); std::shared_ptr HnswStreamerTest::index_meta_ptr_; void HnswStreamerTest::SetUp(void) { @@ -39,15 +41,19 @@ void HnswStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void HnswStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(HnswStreamerTest, TestHnswSearch) { @@ -82,6 +88,7 @@ TEST_F(HnswStreamerTest, TestHnswSearch) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("HnswStreamer"); diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc index 9fa26f477..6235502e4 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "hnsw_streamer.h" +#include #include #include #ifndef _MSC_VER @@ -27,11 +28,13 @@ inline void usleep(unsigned int microseconds) { std::this_thread::sleep_for(std::chrono::microseconds(microseconds)); } #endif +#include #include #include #include #include #include +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -56,7 +59,7 @@ class HnswStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswStreamerTest::dir_("streamer_test/"); +std::string HnswStreamerTest::dir_("hnsw_streamer_test_dir"); shared_ptr HnswStreamerTest::index_meta_ptr_; void HnswStreamerTest::SetUp(void) { @@ -64,15 +67,19 @@ void HnswStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void HnswStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(HnswStreamerTest, TestAddVector) { @@ -573,9 +580,9 @@ TEST_F(HnswStreamerTest, TestOpenClose) { ASSERT_NE(nullptr, storage2); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "TessOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "/TessOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "TessOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "/TessOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -806,7 +813,7 @@ TEST_F(HnswStreamerTest, TestKnnMultiThread) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -932,7 +939,7 @@ TEST_F(HnswStreamerTest, TestKnnConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -1043,7 +1050,7 @@ TEST_F(HnswStreamerTest, TestBfThreshold) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessBfThreshold", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessBfThreshold", true)); ASSERT_EQ(0, streamer->open(storage)); NumericalVector vec(dim); @@ -1120,7 +1127,7 @@ TEST_F(HnswStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessFilter", true)); ASSERT_EQ(0, streamer->open(storage)); @@ -1199,7 +1206,7 @@ TEST_F(HnswStreamerTest, TestMaxIndexSize) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessMaxIndexSize", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessMaxIndexSize", true)); ASSERT_EQ(0, streamer->open(storage)); size_t vsz0 = 0; @@ -1243,7 +1250,7 @@ TEST_F(HnswStreamerTest, TestKnnCleanUp) { ASSERT_NE(nullptr, storage1); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "TessKnnCluenUp1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "/TessKnnCluenUp1", true)); ailego::Params params; constexpr size_t static dim1 = 32; IndexMeta meta1(IndexMeta::DataType::DT_FP32, dim1); @@ -1260,7 +1267,7 @@ TEST_F(HnswStreamerTest, TestKnnCleanUp) { auto storage2 = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage2); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "TessKnnCluenUp2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "/TessKnnCluenUp2", true)); constexpr size_t static dim2 = 64; IndexMeta meta2(IndexMeta::DataType::DT_FP32, dim2); meta2.set_metric("SquaredEuclidean", 0, ailego::Params()); @@ -1283,7 +1290,7 @@ TEST_F(HnswStreamerTest, TestIndexSizeQuota) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestIndexSizeQuota", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestIndexSizeQuota", true)); ailego::Params params; constexpr size_t static dim = 512; IndexMeta meta(IndexMeta::DataType::DT_FP32, dim); @@ -1320,7 +1327,7 @@ TEST_F(HnswStreamerTest, TestBloomFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestBloomFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestBloomFilter", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 16); @@ -1359,7 +1366,7 @@ TEST_F(HnswStreamerTest, TestStreamerParams) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestStreamerParams", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerParams", true)); ailego::Params params; params.set("proxima.hnsw.streamer.docs_hard_limit", 5); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); @@ -1388,7 +1395,7 @@ TEST_F(HnswStreamerTest, TestCheckCrc) ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - std::string path = dir_ + "TestCheckCrc"; + std::string path = dir_ + "/TestCheckCrc"; ASSERT_EQ(0, storage->open(path, true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); @@ -1566,7 +1573,7 @@ TEST_F(HnswStreamerTest, TestCheckDuplicateAndGetVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestCheckDuplicateAndGetVec", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestCheckDuplicateAndGetVec", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 5); @@ -1639,7 +1646,7 @@ TEST_F(HnswStreamerTest, TestDumpIndexAndAdd) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestDumpIndexAndAdd", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestDumpIndexAndAdd", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 5); @@ -1738,7 +1745,7 @@ TEST_F(HnswStreamerTest, TestProvider) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestGetVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestGetVector", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 5); @@ -1863,9 +1870,9 @@ TEST_F(HnswStreamerTest, TestSharedContext) { streamer->open(storage); return streamer; }; - auto streamer1 = create_streamer(dir_ + "TestSharedContext.index1"); - auto streamer2 = create_streamer(dir_ + "TestSharedContext.index2"); - auto streamer3 = create_streamer(dir_ + "TestSharedContext.index3"); + auto streamer1 = create_streamer(dir_ + "/TestSharedContext.index1"); + auto streamer2 = create_streamer(dir_ + "/TestSharedContext.index2"); + auto streamer3 = create_streamer(dir_ + "/TestSharedContext.index3"); srand(ailego::Realtime::MilliSeconds()); IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); @@ -1948,7 +1955,7 @@ TEST_F(HnswStreamerTest, TestMipsEuclideanMetric) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestMipsSquaredEuclidean", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestMipsSquaredEuclidean", true)); const size_t COUNT = 10000; IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); { diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc index 0f46dab99..6082fa0bc 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "hnsw_sparse_builder.h" +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include "zvec/core/framework/index_framework.h" +#include "zvec/ailego/utility/file_helper.h" #include "hnsw_sparse_params.h" #if defined(__GNUC__) || defined(__GNUG__) @@ -52,9 +54,11 @@ void HnswSparseBuilderTest::SetUp(void) { } void HnswSparseBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(_dir.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + _dir + " 2>NUL").c_str()); +#endif + } } TEST_F(HnswSparseBuilderTest, TestGeneral) { diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp index dcbff13e2..d2f0503db 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp @@ -21,6 +21,7 @@ #include #include #include "zvec/core/framework/index_framework.h" +#include "zvec/ailego/utility/file_helper.h" #include "hnsw_sparse_params.h" #if defined(__GNUC__) || defined(__GNUG__) @@ -50,7 +51,7 @@ class HnswSparseSearcherTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswSparseSearcherTest::dir_("HnswSparseSearcherTest/"); +std::string HnswSparseSearcherTest::dir_("HnswSparseSearcherTest"); shared_ptr HnswSparseSearcherTest::_index_meta_ptr; void HnswSparseSearcherTest::generate_sparse_data( @@ -90,9 +91,7 @@ void HnswSparseSearcherTest::SetUp(void) { } void HnswSparseSearcherTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::ailego::FileHelper::RemovePath(dir_.c_str()); } TEST_F(HnswSparseSearcherTest, TestGeneral) { diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp index aebbaf9d8..7c32520fe 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -21,6 +22,7 @@ #include #include #include "hnsw_sparse_streamer.h" +#include "zvec/ailego/utility/file_helper.h" using namespace std; using namespace testing; @@ -47,7 +49,7 @@ class HnswSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswSparseStreamerTest::dir_("HnswSparseStreamerTest/"); +std::string HnswSparseStreamerTest::dir_("hnsw_sparse_streamer_buffer_test_dir"); shared_ptr HnswSparseStreamerTest::index_meta_ptr_; void HnswSparseStreamerTest::generate_sparse_data( @@ -85,15 +87,19 @@ void HnswSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void HnswSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(HnswSparseStreamerTest, TestGeneral) { diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc index c183f9192..6b16db31d 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "hnsw_sparse_streamer.h" +#include +#include #include #include #ifndef _MSC_VER @@ -33,6 +35,7 @@ inline void usleep(unsigned int microseconds) { #include #include #include +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -61,7 +64,7 @@ class HnswSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswSparseStreamerTest::dir_("HnswSparseStreamerTest/"); +std::string HnswSparseStreamerTest::dir_("HnswSparseStreamerTest"); shared_ptr HnswSparseStreamerTest::index_meta_ptr_; void HnswSparseStreamerTest::generate_sparse_data( @@ -99,15 +102,19 @@ void HnswSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } void HnswSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); +#endif + } } TEST_F(HnswSparseStreamerTest, TestGeneral) { @@ -491,9 +498,9 @@ TEST_F(HnswSparseStreamerTest, TestOpenClose) { ASSERT_NE(nullptr, storage2); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "TessOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "/TessOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "TessOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "/TessOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -770,7 +777,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnMultiThread) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -921,7 +928,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessKnnConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessKnnConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -1053,7 +1060,7 @@ TEST_F(HnswSparseStreamerTest, TestBfThreshold) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TessBfThreshold", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TessBfThreshold", true)); ASSERT_EQ(0, streamer->open(storage)); size_t cnt = 10000; @@ -1141,7 +1148,7 @@ TEST_F(HnswSparseStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestFilter", true)); ASSERT_EQ(0, streamer->open(storage)); size_t cnt = 100UL; @@ -1232,7 +1239,7 @@ TEST_F(HnswSparseStreamerTest, TestMaxIndexSize) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestMaxIndexSize", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestMaxIndexSize", true)); ASSERT_EQ(0, streamer->open(storage)); size_t vsz0 = 0; @@ -1290,7 +1297,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnCleanUp) { ASSERT_NE(nullptr, storage1); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "TestKnnCluenUp1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "/TestKnnCluenUp1", true)); ailego::Params params; constexpr size_t static sparse_dim_count1 = 32; @@ -1318,7 +1325,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnCleanUp) { auto storage2 = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage2); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "TestKnnCluenUp2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "/TestKnnCluenUp2", true)); constexpr size_t static sparse_dim_count2 = 64; IndexMeta meta2(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP32); @@ -1355,7 +1362,7 @@ TEST_F(HnswSparseStreamerTest, TestIndexSizeQuota) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestIndexSizeQuota", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestIndexSizeQuota", true)); ailego::Params params; IndexMeta meta(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP32); @@ -1401,7 +1408,7 @@ TEST_F(HnswSparseStreamerTest, TestBloomFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestBloomFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestBloomFilter", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 16); @@ -1449,7 +1456,7 @@ TEST_F(HnswSparseStreamerTest, TestStreamerParams) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestStreamerParams", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerParams", true)); ailego::Params params; params.set("proxima.hnsw.sparse_streamer.docs_hard_limit", 5); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); @@ -1630,7 +1637,7 @@ TEST_F(HnswSparseStreamerTest, TestCheckDuplicateAndGetVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestCheckDuplicateAndGetVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestCheckDuplicateAndGetVector", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 5); @@ -1725,7 +1732,7 @@ TEST_F(HnswSparseStreamerTest, TestDumpIndexAndAdd) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestDumpIndexAndAdd", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestDumpIndexAndAdd", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 5); @@ -1737,10 +1744,12 @@ TEST_F(HnswSparseStreamerTest, TestDumpIndexAndAdd) { IndexMeta::DataType::DT_FP32); ASSERT_NE(nullptr, ctx); int code = 0; - std::mutex mutex; + std::atomic async_started{false}; - auto addVector = [&](int a, int b) { - mutex.unlock(); + auto addVector = [&](int a, int b, bool signal_start) { + if (signal_start) { + async_started.store(true, std::memory_order_release); + } for (int i = a; i < b; i++) { NumericalVector sparse_indices(sparse_dim_count); NumericalVector sparse_velues(sparse_dim_count); @@ -1761,18 +1770,17 @@ TEST_F(HnswSparseStreamerTest, TestDumpIndexAndAdd) { } }; - mutex.lock(); - addVector(0, 2000); - mutex.lock(); - auto t2 = std::async(std::launch::async, addVector, 2000, 3000); + addVector(0, 2000, false); + auto t2 = std::async(std::launch::async, addVector, 2000, 3000, true); auto path1 = dir_ + "/dumpIndex1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); - mutex.lock(); // sync: wait addVector start and release lock + while (!async_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } auto test_dumper = std::make_shared(); ASSERT_EQ(0, streamer->dump(test_dumper)); - mutex.unlock(); ASSERT_EQ(0, streamer->dump(dumper1)); ASSERT_EQ(0, dumper1->close()); t2.get(); @@ -1838,7 +1846,7 @@ TEST_F(HnswSparseStreamerTest, TestProvider) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "TestProvider.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "/TestProvider.index", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 5); @@ -1997,9 +2005,9 @@ TEST_F(HnswSparseStreamerTest, TestSharedContext) { streamer->open(storage); return streamer; }; - auto streamer1 = create_streamer(dir_ + "TestSharedContext.index1"); - auto streamer2 = create_streamer(dir_ + "TestSharedContext.index2"); - auto streamer3 = create_streamer(dir_ + "TestSharedContext.index3"); + auto streamer1 = create_streamer(dir_ + "/TestSharedContext.index1"); + auto streamer2 = create_streamer(dir_ + "/TestSharedContext.index2"); + auto streamer3 = create_streamer(dir_ + "/TestSharedContext.index3"); srand(ailego::Realtime::MilliSeconds()); IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE, diff --git a/tests/core/interface/index_interface_test.cc b/tests/core/interface/index_interface_test.cc index b38eea559..97c607b32 100644 --- a/tests/core/interface/index_interface_test.cc +++ b/tests/core/interface/index_interface_test.cc @@ -30,15 +30,26 @@ using namespace zvec::core_interface; +static void remove_files(const std::string &pattern) { + if (pattern.find('*') != std::string::npos || + pattern.find('?') != std::string::npos) { +#ifdef _WIN32 + system(("del /f /q " + pattern + " 2>NUL").c_str()); +#else + system(("rm -rf " + pattern).c_str()); +#endif + } else { + zvec::ailego::FileHelper::RemovePath(pattern.c_str()); + } +} + TEST(IndexInterface, General) { constexpr uint32_t kDimension = 64; const std::string index_name{"test.index"}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_name.c_str()); auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { - system(cmd_buf); + remove_files(index_name); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -86,7 +97,7 @@ TEST(IndexInterface, General) { ASSERT_FLOAT_EQ(1.0f, fetched_vector[1]); ASSERT_FLOAT_EQ(2.0f, fetched_vector[2]); index->Close(); - system(cmd_buf); + remove_files(index_name); }; @@ -155,13 +166,11 @@ TEST(IndexInterface, BufferGeneral) { zvec::ailego::BufferManager::Instance().init(100 * 1024 * 1024, 1); constexpr uint32_t kDimension = 64; const std::string index_name{"test.index"}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s*", index_name.c_str()); auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { std::string real_index_name = index_name; - system(cmd_buf); + remove_files(index_name + "*"); auto write_index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, write_index); @@ -214,7 +223,7 @@ TEST(IndexInterface, BufferGeneral) { ASSERT_FLOAT_EQ(1.0f, fetched_vector[1]); ASSERT_FLOAT_EQ(2.0f, fetched_vector[2]); read_index->Close(); - system(cmd_buf); + remove_files(index_name + "*"); }; @@ -266,12 +275,10 @@ TEST(IndexInterface, BufferGeneral) { TEST(IndexInterface, SparseGeneral) { constexpr uint32_t kSparseCount = 3; const std::string index_name{"test.index"}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_name.c_str()); auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { - system(cmd_buf); + remove_files(index_name); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -337,7 +344,7 @@ TEST(IndexInterface, SparseGeneral) { ASSERT_EQ(i, fetched_values[i]); } index->Close(); - system(cmd_buf); + remove_files(index_name); }; @@ -386,8 +393,7 @@ TEST(IndexInterface, Merge) { const std::string index_name{"test.index"}; auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + remove_files(file_name); }; auto create_index_func = @@ -762,7 +768,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + remove_files("test.index"); } // Test invalid vector data type for sparse operations @@ -785,7 +791,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + remove_files("test.index"); } // Test fetch non-existent document @@ -806,7 +812,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + remove_files("test.index"); } // Test search with invalid vector data @@ -839,7 +845,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + remove_files("test.index"); } // Test merge with invalid write concurrency @@ -883,7 +889,9 @@ TEST(IndexInterface, Failure) { index1->Close(); index2->Close(); index3->Close(); - system("rm -f test1.index test2.index test3.index"); + remove_files("test1.index"); + remove_files("test2.index"); + remove_files("test3.index"); } } @@ -1013,9 +1021,7 @@ TEST(IndexInterface, Score) { auto sparse_indices = std::vector{0, 1, 2}; auto query_vector = std::vector{1.0f, 2.0f, 3.0f}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + remove_files(index_file_path); auto check_score = [&](const SearchResult &result, MetricType metric_type) { ASSERT_EQ(result.doc_list_.size(), 2); @@ -1086,7 +1092,7 @@ TEST(IndexInterface, Score) { auto dense_func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer query_param, MetricType metric_type) { - system(cmd_buf); + remove_files(index_file_path); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -1108,13 +1114,13 @@ TEST(IndexInterface, Score) { check_score(result, metric_type); index->Close(); - system(cmd_buf); + remove_files(index_file_path); }; auto sparse_func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer query_param, MetricType metric_type) { - system(cmd_buf); + remove_files(index_file_path); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -1142,7 +1148,7 @@ TEST(IndexInterface, Score) { check_score(result, metric_type); index->Close(); - system(cmd_buf); + remove_files(index_file_path); }; constexpr uint32_t kDimension = 3; diff --git a/tests/db/index/CMakeLists.txt b/tests/db/index/CMakeLists.txt index 555b92cf4..9c28c70cb 100644 --- a/tests/db/index/CMakeLists.txt +++ b/tests/db/index/CMakeLists.txt @@ -25,6 +25,12 @@ if(MSVC) "storage/mem_store_test.cc" "column/inverted_column/inverted_indexer_util_test.cc" "segment/segment_test.cc" +# "buffer/buffer_manager_test.cc" +# "flat/flat_streamer_test.cc" +# "flat_sparse/flat_sparse_streamer_test.cc" +# "hnsw/hnsw_streamer_test.cc" +# "hnsw_sparse/hnsw_sparse_builder_test.cc" +# "hnsw_sparse/hnsw_sparse_streamer_test.cc" ) foreach(_excl ${_MSVC_EXCLUDE_TESTS}) list(FILTER ALL_TEST_SRCS EXCLUDE REGEX "${_excl}") diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc index 58e0a0a43..aba343137 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc @@ -15,6 +15,7 @@ #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -229,9 +230,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -242,9 +241,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc index 2c78f05b0..d78fcc7f3 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc @@ -15,6 +15,7 @@ #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -254,9 +255,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -267,9 +266,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc index bcda36898..5562e6495 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc @@ -16,6 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -368,9 +369,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -381,9 +380,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc index 3cd1ae775..e8db1fb1d 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc @@ -16,6 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -479,9 +480,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -492,9 +491,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc index 8732ed24c..8980a66f3 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc @@ -16,6 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "zvec/ailego/utility/file_helper.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -292,9 +293,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -305,9 +304,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc b/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc index 99045eacc..0d25a649d 100644 --- a/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc +++ b/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc @@ -14,6 +14,7 @@ #include +#include "zvec/ailego/utility/file_helper.h" #define private public #define protected public #include "db/index/column/inverted_column/inverted_indexer.h" @@ -38,9 +39,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -51,9 +50,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/storage/wal_file_test.cc b/tests/db/index/storage/wal_file_test.cc index 531f5d9fb..7687346e4 100644 --- a/tests/db/index/storage/wal_file_test.cc +++ b/tests/db/index/storage/wal_file_test.cc @@ -52,9 +52,11 @@ using SegmentID = uint32_t; class WalFileTest : public testing::Test { protected: void SetUp() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf ./data.wal.*"); - system(cmd_buf); +#ifdef _WIN32 + system("del /f /q .\\data.wal.*"); +#else + system("rm -rf ./data.wal.*"); +#endif } void TearDown() {} From 9133a76b6e75ded2389357f2bfc2e6e9abde2f37 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 14:38:49 +0800 Subject: [PATCH 36/51] TODO: currently replace dll by static to support indexFactory register, may use other way --- cmake/bazel.cmake | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/cmake/bazel.cmake b/cmake/bazel.cmake index 54da6c487..bc2764f6c 100644 --- a/cmake/bazel.cmake +++ b/cmake/bazel.cmake @@ -608,6 +608,11 @@ function(_target_link_libraries _NAME) get_target_property(ALWAYS_LINK ${LIB} ALWAYS_LINK) if(ALWAYS_LINK) list(APPEND LOCAL_RESULT ${LIB}) + elseif(MSVC AND TARGET ${LIB}_static) + get_target_property(_SIBLING_AL ${LIB}_static ALWAYS_LINK) + if(_SIBLING_AL) + list(APPEND LOCAL_RESULT ${LIB}_static) + endif() endif() get_target_property(DEP_LIBS ${LIB} INTERFACE_LINK_LIBRARIES) @@ -638,7 +643,31 @@ function(_target_link_libraries _NAME) endforeach() list(REMOVE_DUPLICATES ALL_LIBS_TO_PROCESS) - + + # On MSVC, each DLL has its own copy of template statics (e.g. Factory + # singletons), so registrations inside a DLL are invisible to the exe. + # Substitute SHARED libs with their ALWAYS_LINK _static counterparts and + # use /WHOLEARCHIVE so all registration code lives in the same module. + if(MSVC) + set(_SUBSTITUTED_LIBS "") + foreach(LIB ${ALL_LIBS_TO_PROCESS}) + if(TARGET ${LIB} AND TARGET ${LIB}_static) + get_target_property(_LIB_TYPE ${LIB} TYPE) + get_target_property(_STATIC_AL ${LIB}_static ALWAYS_LINK) + if("${_LIB_TYPE}" STREQUAL "SHARED_LIBRARY" AND _STATIC_AL) + list(APPEND _SUBSTITUTED_LIBS ${LIB}_static) + list(APPEND ALL_ALWAYS_LINK_LIBS ${LIB}_static) + continue() + endif() + endif() + list(APPEND _SUBSTITUTED_LIBS ${LIB}) + endforeach() + set(ALL_LIBS_TO_PROCESS ${_SUBSTITUTED_LIBS}) + if(ALL_ALWAYS_LINK_LIBS) + list(REMOVE_DUPLICATES ALL_ALWAYS_LINK_LIBS) + endif() + endif() + foreach(LIB ${ALL_LIBS_TO_PROCESS}) if(NOT TARGET ${LIB}) list(APPEND LINK_LIBS ${LIB}) From 938e5a55726665e5b263de6f1ea695d4fb3575ee Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 19:22:48 +0800 Subject: [PATCH 37/51] ignore more --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ec39ee3c..848d611a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,8 @@ if(MSVC) add_compile_options(/wd4310) # warning C4310: cast truncates constant value add_compile_options(/wd4530) # warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc add_compile_options(/wd4251) #protobuf - add_compile_options(/wd4245 /wd4334 /wd4702 /wd4305) + add_compile_options(/wd4245 /wd4334 /wd4702 /wd4305 /wd4099) + add_compile_options(/wd4200) # flexible structure # //warning C4334: '<<': result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?) # keeps |= (1 << k); From 6fb2707e1ac092e7777436c27b24a6725334948a Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 19:23:11 +0800 Subject: [PATCH 38/51] fix antlr4 release --- thirdparty/antlr/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/thirdparty/antlr/CMakeLists.txt b/thirdparty/antlr/CMakeLists.txt index dd3becdeb..f137628f6 100644 --- a/thirdparty/antlr/CMakeLists.txt +++ b/thirdparty/antlr/CMakeLists.txt @@ -16,6 +16,9 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/runtime/src/" IMPORTED_LOCATION "${EXTERNAL_LIB_DIR}/${_antlr4_lib_name}" ) +if(WIN32) + set_property(TARGET antlr4 PROPERTY INTERFACE_COMPILE_DEFINITIONS "ANTLR4CPP_STATIC") +endif() add_dependencies(antlr4 antlr4_static) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") From 0e427a2865eb3d6fc77abd8edce5fcab65a1fcb2 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 19:39:22 +0800 Subject: [PATCH 39/51] fix: gtest link warning gtest-all.obj : warning LNK4197: export '??_7TestSuite@testing@@6B@' specified multiple times; using first specification [C:\Users\Administrator\Desktop\workspace\zvec\build_vs2022\thirdparty\googletest\googletest-1.10.0\googletest\gtest.vcxproj] gtest-all.obj : warning LNK4197: export '??_7AssertionException@testing@@6B@' specified multiple times; using first specification [C:\Users\Administrator\Desktop\workspace\zvec\build_vs2022\thirdparty\googletest\googletest-1.10.0\googletest\gtest.vcxproj] gtest-all.obj : warning LNK4197: export '??_7UnitTest@testing@@6B@' specified multiple times; using first specification [C:\Users\Administrator\Desktop\workspace\zvec\build_vs2022\thirdparty\googletest\googletest-1.10.0\googletest\gtest.vcxproj] gtest-all.obj : warning LNK4197: export '??_7ScopedFakeTestPartResultReporter@testing@@6B@' specified multiple times; using first specification [C:\Users\Administrator\Desktop\workspace\zvec\build_vs2022\thirdparty\googletest\googletest-1.10.0\googletest\gtest.vcxproj] gtest-all.obj : warning LNK4197: export '??_7UnitTestImpl@ --- thirdparty/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index a32eac5ef..fb4415973 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -12,7 +12,12 @@ set(EXTERNAL_LIB_DIR ${CMAKE_BINARY_DIR}/external/usr/local/lib) file(MAKE_DIRECTORY ${EXTERNAL_INC_DIR}) file(MAKE_DIRECTORY ${EXTERNAL_LIB_DIR}) +set(_SAVED_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) +set(BUILD_SHARED_LIBS OFF) +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) add_subdirectory(googletest googletest EXCLUDE_FROM_ALL) +set(BUILD_SHARED_LIBS ${_SAVED_BUILD_SHARED_LIBS}) +unset(_SAVED_BUILD_SHARED_LIBS) add_subdirectory(gflags gflags EXCLUDE_FROM_ALL) add_subdirectory(glog glog EXCLUDE_FROM_ALL) add_subdirectory(sparsehash sparsehash EXCLUDE_FROM_ALL) From 1948e3901bcdd2bb4f0b190c86c09cc32809e406 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Fri, 13 Mar 2026 19:39:52 +0800 Subject: [PATCH 40/51] revert c++20 --- CMakeLists.txt | 10 +- .../column/vector_column_indexer_test.cc | 170 ++++++++++-------- tests/db/index/segment/segment_test.cc | 47 +++-- 3 files changed, 116 insertions(+), 111 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 848d611a7..c7d4f4f90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,15 +1,7 @@ cmake_minimum_required(VERSION 3.13) cmake_policy(SET CMP0077 NEW) project(zvec) -#TODO(windows): maybe removed, -# to handle : -#zvec\tests\db\index\column\vector_column_indexer_test.cc(127,43): error C7555: use of designated initializers requires at least '/std:c++20' -# right now -if(MSVC) - set(CC_CXX_STANDARD 20) -else() - set(CC_CXX_STANDARD 17) -endif() +set(CC_CXX_STANDARD 17) if(MSVC) set(INTTYPES_FORMAT VC7) diff --git a/tests/db/index/column/vector_column_indexer_test.cc b/tests/db/index/column/vector_column_indexer_test.cc index ff027a0da..792ebf6e7 100644 --- a/tests/db/index/column/vector_column_indexer_test.cc +++ b/tests/db/index/column/vector_column_indexer_test.cc @@ -121,15 +121,14 @@ TEST(VectorColumnIndexerTest, General) { ASSERT_NEAR(dense_vector[3], 0, 0.1); // 4. search - // https://stackoverflow.com/questions/69009389/how-to-get-away-with-using-designated-initializers-in-c17-or-why-is-it-seemi auto query_vector = std::vector{1.0f, 2.0f, 3.0f, 0}; auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto indexer_query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .query_params = query_params}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams indexer_query_params; + indexer_query_params.topk = 10; + indexer_query_params.filter = nullptr; + indexer_query_params.fetch_vector = true; + indexer_query_params.query_params = query_params; auto results = indexer->Search(query, indexer_query_params); ASSERT_TRUE(results.has_value()); @@ -307,12 +306,12 @@ TEST(VectorColumnIndexerTest, DenseDataTypeFP16) { buffer.data()); auto query_vector = buffer; auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto indexer_query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .query_params = query_params}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams indexer_query_params; + indexer_query_params.topk = 10; + indexer_query_params.filter = nullptr; + indexer_query_params.fetch_vector = true; + indexer_query_params.query_params = query_params; auto results = indexer->Search(query, indexer_query_params); ASSERT_TRUE(results.has_value()); @@ -445,15 +444,14 @@ TEST(VectorColumnIndexerTest, DenseDataTypeINT8) { } // 4. search - // https://stackoverflow.com/questions/69009389/how-to-get-away-with-using-designated-initializers-in-c17-or-why-is-it-seemi auto query_vector = std::vector{1, 2, 3, 0}; auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto indexer_query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .query_params = query_params}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams indexer_query_params; + indexer_query_params.topk = 10; + indexer_query_params.filter = nullptr; + indexer_query_params.fetch_vector = true; + indexer_query_params.query_params = query_params; auto results = indexer->Search(query, indexer_query_params); ASSERT_TRUE(results.has_value()); @@ -560,8 +558,10 @@ TEST(VectorColumnIndexerTest, SparseGeneral) { auto query = vector_column_params::VectorData{vector_column_params::SparseVector{ kSparseCount, indices.data(), values.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -674,8 +674,10 @@ TEST(VectorColumnIndexerTest, SparseDataTypeFP16) { auto query = vector_column_params::VectorData{vector_column_params::SparseVector{ kSparseCount, indices.data(), values.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -853,8 +855,10 @@ TEST(VectorColumnIndexerTest, Merge) { // search with fetch vector auto query = vector_column_params::VectorData{ vector_column_params::DenseVector{vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer2->Search(query, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = @@ -1305,14 +1309,15 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{1}; + auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; auto query = vector_column_params::VectorData{vector_column_params::DenseVector{ - .data = std::vector{1.0f, 2.0f, 3.0f}.data()}}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .bf_pks = {bf_pks}}; + query_vec.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.bf_pks = {bf_pks}; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1334,14 +1339,15 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{1, 2}; + auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; auto query = vector_column_params::VectorData{vector_column_params::DenseVector{ - .data = std::vector{1.0f, 2.0f, 3.0f}.data()}}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .bf_pks = {bf_pks}}; + query_vec.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.bf_pks = {bf_pks}; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1363,14 +1369,15 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{2}; + auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; auto query = vector_column_params::VectorData{vector_column_params::DenseVector{ - .data = std::vector{1.0f, 2.0f, 3.0f}.data()}}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .bf_pks = {bf_pks}}; + query_vec.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.bf_pks = {bf_pks}; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1623,11 +1630,11 @@ TEST(VectorColumnIndexerTest, CosineGeneral) { vector_column_params::DenseVector{buffer.data.data()}}; auto _t = std::make_shared(100); _t->set_is_linear(true); - auto query_params = - vector_column_params::QueryParams{.topk = kTopk, - .filter = nullptr, - .fetch_vector = true, - .query_params = _t}; + vector_column_params::QueryParams query_params; + query_params.topk = kTopk; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.query_params = _t; auto results = indexer->Search(data, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = @@ -1833,9 +1840,11 @@ TEST(VectorColumnIndexerTest, Score) { .ok()); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = kTopk, .filter = nullptr, .fetch_vector = true}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = kTopk; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1884,8 +1893,10 @@ TEST(VectorColumnIndexerTest, Score) { vector_column_params::VectorData{vector_column_params::SparseVector{ 3, reinterpret_cast(sparse_indices.data()), query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1978,9 +1989,11 @@ TEST(VectorColumnIndexerTest, Failure) { // Test Search on unopened indexer auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = false}; + vector_column_params::DenseVector{vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; auto search_result = indexer->Search(query, query_params); ASSERT_FALSE(search_result.has_value()); ASSERT_EQ(search_result.error().message(), "Index not opened"); @@ -2115,14 +2128,14 @@ TEST(VectorColumnIndexerTest, Failure) { // Test search with bf_pks size > 1 auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = vector.data()}}; + vector_column_params::DenseVector{vector.data()}}; auto bf_pks1 = std::vector{1, 2}; auto bf_pks2 = std::vector{3, 4}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = false, - .bf_pks = {bf_pks1, bf_pks2}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; + query_params.bf_pks = {bf_pks1, bf_pks2}; auto search_result = indexer->Search(query, query_params); ASSERT_FALSE(search_result.has_value()); @@ -2305,9 +2318,11 @@ TEST(VectorColumnIndexerTest, Failure) { // // // Test search with unsupported index type // auto query = vector_column_params::VectorData{ - // vector_column_params::DenseVector{.data = vector.data()}}; - // auto query_params = vector_column_params::QueryParams{ - // .topk = 10, .filter = nullptr, .fetch_vector = false}; + // vector_column_params::DenseVector{vector.data()}}; + // vector_column_params::QueryParams query_params; + // query_params.topk = 10; + // query_params.filter = nullptr; + // query_params.fetch_vector = false; // // auto search_result = indexer->Search(query, query_params); // ASSERT_FALSE(search_result.has_value()); @@ -2465,8 +2480,10 @@ TEST(VectorColumnIndexerTest, CosineMerge) { // search with fetch vector auto query = vector_column_params::VectorData{ vector_column_params::DenseVector{vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer2->Search(query, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = @@ -2624,15 +2641,14 @@ TEST(VectorColumnIndexerTest, Refiner) { auto data = vector_column_params::VectorData{ vector_column_params::DenseVector{buffer.data.data()}}; ; - auto query_params = vector_column_params::QueryParams{ - .topk = kTopk, - .filter = nullptr, - .fetch_vector = true, - .query_params = std::make_shared(100), - .refiner_param = std::make_shared( - vector_column_params::RefinerParam{ - .scale_factor_ = 10, - .reference_indexer = reference_indexer})}; + vector_column_params::QueryParams query_params; + query_params.topk = kTopk; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.query_params = std::make_shared(100); + query_params.refiner_param = + std::make_shared( + vector_column_params::RefinerParam{10, reference_indexer}); auto results = indexer->Search(data, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = diff --git a/tests/db/index/segment/segment_test.cc b/tests/db/index/segment/segment_test.cc index 6ca6fffe4..c15502adc 100644 --- a/tests/db/index/segment/segment_test.cc +++ b/tests/db/index/segment/segment_test.cc @@ -1078,12 +1078,12 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexer) { auto dense_fp32_field = schema->get_field("dense_fp32"); auto query_vector = new_doc.get>("dense_fp32").value(); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .dimension = dense_fp32_field->dimension(), - .topk = 10, - .filter = nullptr, - .fetch_vector = false}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.dimension = dense_fp32_field->dimension(); + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; auto results = combined_indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1144,13 +1144,13 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexerWithQuantVectorIndex) { auto dense_fp32_field = schema->get_field("dense_fp32"); auto query_vector = new_doc.get>("dense_fp32").value(); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .dimension = dense_fp32_field->dimension(), - .topk = 10, - .filter = nullptr, - .fetch_vector = false, - .query_params = std::make_shared(IndexType::HNSW)}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.dimension = dense_fp32_field->dimension(); + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; + query_params.query_params = std::make_shared(IndexType::HNSW); query_params.query_params->set_is_using_refiner(true); auto results = combined_indexer->Search(query, query_params); @@ -1195,18 +1195,15 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexerQueryWithPks) { auto dense_fp32_field = schema->get_field("dense_fp32"); auto query_vector = verify_doc.get>("dense_fp32").value(); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .data_type = dense_fp32_field->data_type(), - .dimension = dense_fp32_field->dimension(), - .topk = 10, - .filter = nullptr, - .fetch_vector = false, - .query_params = std::make_shared(IndexType::HNSW), - .group_by = nullptr, - .bf_pks = bf_pks, - .refiner_param = nullptr, - .extra_params = {}}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.data_type = dense_fp32_field->data_type(); + query_params.dimension = dense_fp32_field->dimension(); + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; + query_params.query_params = std::make_shared(IndexType::HNSW); + query_params.bf_pks = bf_pks; auto results = combined_indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); From 7a5cfb638d9a84f4b84c2010ab0e6477cafd0a33 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 15:04:14 +0800 Subject: [PATCH 41/51] platform related code refactor --- src/ailego/utility/string_helper.cc | 5 +--- .../algorithm/hnsw/hnsw_streamer_entity.cc | 3 +-- src/core/utility/visit_filter.h | 6 +---- src/db/index/common/doc.cc | 23 ++++--------------- src/include/zvec/ailego/internal/platform.h | 20 +++++++++++++++- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/ailego/utility/string_helper.cc b/src/ailego/utility/string_helper.cc index 6c2015782..74d3affd6 100644 --- a/src/ailego/utility/string_helper.cc +++ b/src/ailego/utility/string_helper.cc @@ -14,6 +14,7 @@ #include #include +#include #include namespace zvec { @@ -64,10 +65,6 @@ std::string StringHelper::CopyTrim(std::string str) { return str; } -#if defined(_MSC_VER) -#define strncasecmp _strnicmp -#endif - bool StringHelper::CompareIgnoreCase(const std::string &a, const std::string &b) { if (a.size() != b.size()) { diff --git a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc index 6562f1263..9c85a8bbc 100644 --- a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc @@ -89,7 +89,7 @@ int HnswStreamerEntity::update_neighbors( } auto loc = get_neighbor_chunk_loc(level, id); - size_t size = reinterpret_cast(&hd->neighbors[i]) - &buffer[0]; + size_t size = reinterpret_cast(&hd->neighbors[i]) - buffer.data(); size_t ret = loc.first->write(loc.second, hd, size); if (ailego_unlikely(ret != size)) { LOG_ERROR("Write neighbor header failed, ret=%zu", ret); @@ -488,7 +488,6 @@ int HnswStreamerEntity::add_vector(level_t level, key_t key, const void *vec, // On MSVC, unsigned long is 32-bit, so -1UL is 0xFFFFFFFF not 0xFFFFFFFFFFFFFFFF. size_t chunk_offset = static_cast(-1); - std::lock_guard lock(mutex_); // duplicate check if (ailego_unlikely(filter_same_key_ && get_id(key) != kInvalidNodeId)) { diff --git a/src/core/utility/visit_filter.h b/src/core/utility/visit_filter.h index 4f46cadd4..959a0b450 100644 --- a/src/core/utility/visit_filter.h +++ b/src/core/utility/visit_filter.h @@ -22,17 +22,13 @@ #include #include #include +#include #include #include namespace zvec { namespace core { -// id_t is a POSIX type from ; provide a definition for Windows -#if defined(_MSC_VER) -typedef unsigned int id_t; -#endif - struct VisitFilterHeader { VisitFilterHeader() : maxDocCnt(0), maxScanNum(0) {} uint64_t maxDocCnt; diff --git a/src/db/index/common/doc.cc b/src/db/index/common/doc.cc index 8fd1c6266..48c4a5058 100644 --- a/src/db/index/common/doc.cc +++ b/src/db/index/common/doc.cc @@ -16,9 +16,7 @@ #include #include #include -#ifdef _MSC_VER -#include -#endif +#include #include #include "db/common/constants.h" #include "db/index/common/type_helper.h" @@ -119,29 +117,16 @@ template T byte_swap(T value) { if constexpr (std::is_same_v) { uint16_t val = *reinterpret_cast(&value); -#ifdef _MSC_VER - val = _byteswap_ushort(val); -#else - val = __builtin_bswap16(val); -#endif + val = ailego_bswap16(val); return *reinterpret_cast(&val); } else if constexpr (sizeof(T) == 1) { return value; } else if constexpr (sizeof(T) == 2) { return (value << 8) | ((value >> 8) & 0xFF); } else if constexpr (sizeof(T) == 4) { -#ifdef _MSC_VER - return static_cast(_byteswap_ulong(static_cast(value))); -#else - return __builtin_bswap32(value); -#endif + return static_cast(ailego_bswap32(static_cast(value))); } else if constexpr (sizeof(T) == 8) { -#ifdef _MSC_VER - return static_cast( - _byteswap_uint64(static_cast(value))); -#else - return __builtin_bswap64(value); -#endif + return static_cast(ailego_bswap64(static_cast(value))); } else { T result = 0; for (size_t i = 0; i < sizeof(T); ++i) { diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 1bdde31f1..832d377d1 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -44,7 +44,7 @@ #endif #endif -//! Add 'ssize_t' for MSVC (before extern "C" so it's available in C++) +//! MSVC platform compatibility (before extern "C" so available in C++) #if defined(_MSC_VER) #if !defined(S_ISDIR) @@ -56,8 +56,26 @@ typedef intptr_t ssize_t; #endif +#if !defined(_ID_T_DEFINED) +#define _ID_T_DEFINED +typedef unsigned int id_t; #endif +#include +#define ailego_bswap16(x) _byteswap_ushort(x) +#define ailego_bswap32(x) _byteswap_ulong(x) +#define ailego_bswap64(x) _byteswap_uint64(x) + +#define strncasecmp _strnicmp + +#else // !_MSC_VER + +#define ailego_bswap16(x) __builtin_bswap16(x) +#define ailego_bswap32(x) __builtin_bswap32(x) +#define ailego_bswap64(x) __builtin_bswap64(x) + +#endif // _MSC_VER + #if defined(__cplusplus) extern "C" { From ed5779356ce6827085db0c3eff15100e4e43fa97 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 15:13:44 +0800 Subject: [PATCH 42/51] buffer manager cleanup to avoid ctest failure --- src/ailego/buffer/buffer_manager.cc | 13 +++++++++---- src/include/zvec/ailego/buffer/buffer_manager.h | 2 ++ tests/ailego/buffer/buffer_manager_test.cc | 5 ++++- tests/core/interface/index_interface_test.cc | 1 + 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/ailego/buffer/buffer_manager.cc b/src/ailego/buffer/buffer_manager.cc index e42e58d9f..be3b6755d 100644 --- a/src/ailego/buffer/buffer_manager.cc +++ b/src/ailego/buffer/buffer_manager.cc @@ -103,7 +103,7 @@ struct BufferManager::BufferContext { ~BufferContext() { if (vector) { - free(vector); + ailego_aligned_free(vector); } } @@ -264,7 +264,7 @@ bool BufferManager::BufferContext::read_vector() { uint32_t offset = id.vector().offset; if (file.read(offset, vector, len) != len) { LOG_ERROR("Failed to read file[%s]", file_name.c_str()); - free(vector); + ailego_aligned_free(vector); vector = nullptr; return false; } @@ -390,7 +390,7 @@ class BufferManager::BufferPool { if (victim->id.type == BufferID::TYPE::PARQUET) { victim->arrow_refs.clear(); } else { - free(victim->vector); + ailego_aligned_free(victim->vector); victim->vector = nullptr; } victim->state = BufferContext::State::IDLE; @@ -585,10 +585,15 @@ uint64_t BufferManager::total_size_in_bytes() const { } -BufferManager::~BufferManager() { +void BufferManager::cleanup() { for (auto pool : pools_) { delete pool; } + pools_.clear(); +} + +BufferManager::~BufferManager() { + cleanup(); } diff --git a/src/include/zvec/ailego/buffer/buffer_manager.h b/src/include/zvec/ailego/buffer/buffer_manager.h index 8aebf759d..1e3496196 100644 --- a/src/include/zvec/ailego/buffer/buffer_manager.h +++ b/src/include/zvec/ailego/buffer/buffer_manager.h @@ -179,6 +179,8 @@ class BufferManager : public Singleton { uint64_t total_size_in_bytes() const; + void cleanup(); + ~BufferManager(); private: diff --git a/tests/ailego/buffer/buffer_manager_test.cc b/tests/ailego/buffer/buffer_manager_test.cc index a9f4ae4c7..ada241da8 100644 --- a/tests/ailego/buffer/buffer_manager_test.cc +++ b/tests/ailego/buffer/buffer_manager_test.cc @@ -58,7 +58,10 @@ class BufferManagerTest : public testing::Test { BufferManager::Instance().init(4 * 1024 * 1024, 1); } - static void TearDownTestCase() {} + static void TearDownTestCase() { + BufferManager::Instance().cleanup(); + zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + } /***** Global initialization and cleanup - End *****/ ; }; diff --git a/tests/core/interface/index_interface_test.cc b/tests/core/interface/index_interface_test.cc index 97c607b32..66cfae723 100644 --- a/tests/core/interface/index_interface_test.cc +++ b/tests/core/interface/index_interface_test.cc @@ -269,6 +269,7 @@ TEST(IndexInterface, BufferGeneral) { .with_fetch_vector(true) .with_ef_search(20) .build()); + zvec::ailego::BufferManager::Instance().cleanup(); } From 5c612126e91f474dda38bce3fd6211a8219ff0db Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 15:14:01 +0800 Subject: [PATCH 43/51] refactor: file remove/test setup --- CMakeLists.txt | 1 + tests/ailego/buffer/buffer_manager_test.cc | 6 +- .../core/algorithm/flat/flat_builder_test.cc | 34 ++--- .../flat/flat_streamer_buffer_test.cpp | 29 ++-- .../flat/flat_streamer_buffer_time_test.cpp | 21 +-- .../core/algorithm/flat/flat_streamer_test.cc | 58 ++++---- .../flat_sparse/flat_sparse_builder_test.cc | 12 +- .../flat_sparse/flat_sparse_searcher_test.cc | 32 ++--- .../flat_sparse_streamer_buffer_test.cc | 21 +-- .../flat_sparse/flat_sparse_streamer_test.cc | 59 ++++---- .../core/algorithm/hnsw/hnsw_builder_test.cc | 18 +-- .../algorithm/hnsw/hnsw_searcher_test.cpp | 80 +++++------ .../hnsw/hnsw_streamer_buffer_test.cpp | 25 ++-- .../core/algorithm/hnsw/hnsw_streamer_test.cc | 128 ++++++++---------- .../hnsw_sparse/hnsw_sparse_builder_test.cc | 21 ++- .../hnsw_sparse/hnsw_sparse_searcher_test.cpp | 48 +++---- .../hnsw_sparse_streamer_buffer_test.cpp | 25 ++-- .../hnsw_sparse/hnsw_sparse_streamer_test.cc | 90 +++++------- tests/core/interface/index_interface_test.cc | 52 +++---- ...erted_column_indexer_array_numbers_test.cc | 6 +- .../inverted_column_indexer_bool_test.cc | 6 +- ...rted_column_indexer_cyclic_numbers_test.cc | 6 +- ..._column_indexer_sequential_numbers_test.cc | 6 +- .../inverted_column_indexer_string_test.cc | 6 +- .../inverted_indexer_util_test.cc | 6 +- .../column/vector_column_indexer_test.cc | 73 ++++------ tests/db/index/storage/wal_file_test.cc | 17 +-- 27 files changed, 369 insertions(+), 517 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c7d4f4f90..3f25d14cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,6 +49,7 @@ include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) include_directories(${PROJECT_ROOT_DIR}/src/include) include_directories(${PROJECT_ROOT_DIR}/src) +include_directories(${PROJECT_ROOT_DIR}) option(BUILD_PYTHON_BINDINGS "Build Python bindings using pybind11" OFF) message(STATUS "BUILD_PYTHON_BINDINGS:${BUILD_PYTHON_BINDINGS}") diff --git a/tests/ailego/buffer/buffer_manager_test.cc b/tests/ailego/buffer/buffer_manager_test.cc index ada241da8..90c9f4e93 100644 --- a/tests/ailego/buffer/buffer_manager_test.cc +++ b/tests/ailego/buffer/buffer_manager_test.cc @@ -17,7 +17,7 @@ #include #include #include -#include +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -36,7 +36,7 @@ class BufferManagerTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); if (!File::MakePath(working_dir)) { LOG_ERROR("Failed to create working directory."); @@ -60,7 +60,7 @@ class BufferManagerTest : public testing::Test { static void TearDownTestCase() { BufferManager::Instance().cleanup(); - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ ; diff --git a/tests/core/algorithm/flat/flat_builder_test.cc b/tests/core/algorithm/flat/flat_builder_test.cc index 41696b72c..1a447dfbc 100644 --- a/tests/core/algorithm/flat/flat_builder_test.cc +++ b/tests/core/algorithm/flat/flat_builder_test.cc @@ -17,7 +17,7 @@ #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -44,7 +44,7 @@ class FlatBuilderTest : public testing::Test { static IndexMeta meta_; }; -std::string FlatBuilderTest ::dir_("flat_builder_test"); +std::string FlatBuilderTest ::dir_("flat_builder_test/"); IndexMeta FlatBuilderTest::meta_; void FlatBuilderTest::SetUp(void) { @@ -55,7 +55,7 @@ void FlatBuilderTest::SetUp(void) { //! self-check column-major and row-major search. void FlatBuilderTest::TearDown(void) { - zvec::ailego::FileHelper::RemovePath(dir_.c_str()); + zvec::test_util::RemoveTestPath(dir_); } void build_process(IndexBuilder::Pointer &builder, @@ -67,7 +67,7 @@ void build_process(IndexBuilder::Pointer &builder, auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - std::string path = FlatBuilderTest::dir_ + "/TestGeneral"; + std::string path = FlatBuilderTest::dir_ + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -127,7 +127,7 @@ TEST_F(FlatBuilderTest, TestBinaryInvalidColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; } TEST_F(FlatBuilderTest, TestBuildWithRowMajor) { @@ -137,7 +137,7 @@ TEST_F(FlatBuilderTest, TestBuildWithRowMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -165,7 +165,7 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithRowMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -194,7 +194,7 @@ TEST_F(FlatBuilderTest, TestBinaryBuildWithRowMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); @@ -222,7 +222,7 @@ TEST_F(FlatBuilderTest, TestBuildWithColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -251,7 +251,7 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); size_t doc_cnt = 128UL; @@ -279,7 +279,7 @@ TEST_F(FlatBuilderTest, TestBinaryBuildWithColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); @@ -306,7 +306,7 @@ TEST_F(FlatBuilderTest, TestWithRowMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -331,7 +331,7 @@ TEST_F(FlatBuilderTest, TestInt8WithRowMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -357,7 +357,7 @@ TEST_F(FlatBuilderTest, TestBinaryWithRowMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); @@ -382,7 +382,7 @@ TEST_F(FlatBuilderTest, TestWithColumnMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -408,7 +408,7 @@ TEST_F(FlatBuilderTest, TestInt8WithColumnMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); size_t doc_cnt = 128UL; @@ -433,7 +433,7 @@ TEST_F(FlatBuilderTest, TestBinaryWithColumnMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp index 6dfa9eb10..616dfe256 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -8,7 +7,7 @@ #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -32,7 +31,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("flat_streamer_buffer_test_dir"); +std::string FlatStreamerTest::dir_("flat_streamer_buffer_test_dir/"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -40,19 +39,11 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void FlatStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatStreamerTest, TestLinearSearch) { @@ -67,7 +58,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearch", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -93,7 +84,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); @@ -189,7 +180,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchMMap", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchMMap", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -216,7 +207,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchMMap", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchMMap", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); @@ -285,7 +276,7 @@ TEST_F(FlatStreamerTest, TestBufferStorage) { ASSERT_NE(nullptr, storage); Params stg_params; EXPECT_EQ(0, storage->init(stg_params)); - EXPECT_EQ(0, storage->open(dir_ + "/Test/LinearSearch", true)); + EXPECT_EQ(0, storage->open(dir_ + "Test/LinearSearch", true)); EXPECT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -311,7 +302,7 @@ TEST_F(FlatStreamerTest, TestBufferStorage) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); EXPECT_EQ(0, read_storage->init(stg_params)); - EXPECT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearch", false)); + EXPECT_EQ(0, read_storage->open(dir_ + "Test/LinearSearch", false)); EXPECT_EQ(0, read_streamer->open(read_storage)); auto read_ctx = read_streamer->create_context(); auto provider = read_streamer->create_provider(); diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp index 69dbe970b..5265878d8 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -8,7 +7,7 @@ #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -32,7 +31,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("flat_streamer_buffer_time_test_dir"); +std::string FlatStreamerTest::dir_("flat_streamer_buffer_time_test_dir/"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -40,19 +39,11 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void FlatStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatStreamerTest, TestLinearSearchMMap) { @@ -67,7 +58,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchMMap", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchMMap", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -93,7 +84,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchMMap", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchMMap", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 30; ElapsedTime elapsed_time; diff --git a/tests/core/algorithm/flat/flat_streamer_test.cc b/tests/core/algorithm/flat/flat_streamer_test.cc index 7e46b4d95..cd8c6ff13 100644 --- a/tests/core/algorithm/flat/flat_streamer_test.cc +++ b/tests/core/algorithm/flat/flat_streamer_test.cc @@ -24,7 +24,7 @@ #include #include #include "algorithm/flat/flat_utility.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -48,7 +48,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("flat_streamer_test_dir"); +std::string FlatStreamerTest::dir_("flat_streamer_test_dir/"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -56,19 +56,11 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void FlatStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatStreamerTest, TestAddVector) { @@ -82,7 +74,7 @@ TEST_F(FlatStreamerTest, TestAddVector) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/AddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -117,7 +109,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/AddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -191,7 +183,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -224,7 +216,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearcherSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearcherSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearcherSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -241,7 +233,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearcherSearch) { streamer->add_impl(i, vec.data(), qmeta, ctx); } - std::string path1 = dir_ + "/TestAddAndSearcherSearchDump"; + std::string path1 = dir_ + "TestAddAndSearcherSearchDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_EQ(0, dumper->init(Params())); ASSERT_EQ(0, dumper->create(path1)); @@ -282,7 +274,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchRandomData) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearchRandomData", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearchRandomData", true)); ASSERT_EQ(0, streamer->init(meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -367,9 +359,9 @@ TEST_F(FlatStreamerTest, TestOpenClose) { ASSERT_NE(nullptr, storage2); Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "/TestOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "TestOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "/TestOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "TestOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -456,7 +448,7 @@ TEST_F(FlatStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -496,7 +488,7 @@ TEST_F(FlatStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -525,7 +517,7 @@ TEST_F(FlatStreamerTest, TestMultiThread) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -638,7 +630,7 @@ TEST_F(FlatStreamerTest, TestConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -744,7 +736,7 @@ TEST_F(FlatStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessFilter", true)); ASSERT_EQ(0, streamer->open(storage)); @@ -819,7 +811,7 @@ TEST_F(FlatStreamerTest, TestMaxIndexSize) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessMaxIndexSize", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessMaxIndexSize", true)); ASSERT_EQ(0, streamer->open(storage)); size_t vsz0 = 0; @@ -868,7 +860,7 @@ TEST_F(FlatStreamerTest, TestCleanUp) { ASSERT_NE(nullptr, storage1); Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "/TessKnnCluenUp1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "TessKnnCluenUp1", true)); Params params; constexpr size_t static dim1 = 32; IndexMeta meta1(IndexMeta::DataType::DT_FP32, dim1); @@ -885,7 +877,7 @@ TEST_F(FlatStreamerTest, TestCleanUp) { auto storage2 = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage2); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "/TessKnnCluenUp2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "TessKnnCluenUp2", true)); constexpr size_t static dim2 = 64; IndexMeta meta2(IndexMeta::DataType::DT_FP32, dim2); meta2.set_metric("SquaredEuclidean", 0, Params()); @@ -908,7 +900,7 @@ TEST_F(FlatStreamerTest, TestBloomFilter) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestBloomFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestBloomFilter", true)); Params params; ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -941,7 +933,7 @@ TEST_F(FlatStreamerTest, TestGroup) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -1049,7 +1041,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -1130,7 +1122,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) { Params write_stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(write_stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, write_params)); ASSERT_EQ(0, write_streamer->open(write_storage)); auto ctx = write_streamer->create_context(); @@ -1163,7 +1155,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) { Params read_stg_params; auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, read_storage->init(read_stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, read_storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, read_params)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc index f175c5d2f..d50fc983b 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc @@ -17,7 +17,7 @@ #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -37,7 +37,7 @@ class FlatSparseBuilderTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string FlatSparseBuilderTest::_dir("FlatSparseBuilderTest"); +std::string FlatSparseBuilderTest::_dir("FlatSparseBuilderTest/"); shared_ptr FlatSparseBuilderTest::_index_meta_ptr; void FlatSparseBuilderTest::SetUp(void) { @@ -47,7 +47,7 @@ void FlatSparseBuilderTest::SetUp(void) { } void FlatSparseBuilderTest::TearDown(void) { - zvec::ailego::FileHelper::RemovePath(_dir.c_str()); + zvec::test_util::RemoveTestPath(_dir); } TEST_F(FlatSparseBuilderTest, TestGeneral) { @@ -83,7 +83,7 @@ TEST_F(FlatSparseBuilderTest, TestGeneral) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -187,7 +187,7 @@ TEST_F(FlatSparseBuilderTest, TestIndexThreads) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndexThreads"; + string path = _dir + "TestIndexThreads"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder1->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -244,7 +244,7 @@ TEST_F(FlatSparseBuilderTest, TestHalfFloatConverter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestHalFloatConverter"; + string path = _dir + "TestHalFloatConverter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc index 41f587281..6f95fa621 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc @@ -22,7 +22,7 @@ #include #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -48,7 +48,7 @@ class FlatSparseSearcherTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatSparseSearcherTest::dir_("searcher_test"); +std::string FlatSparseSearcherTest::dir_("searcher_test/"); std::shared_ptr FlatSparseSearcherTest::index_meta_ptr_; void FlatSparseSearcherTest::generate_sparse_data( @@ -88,11 +88,11 @@ void FlatSparseSearcherTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, Params()); - zvec::ailego::FileHelper::RemovePath(dir_.c_str()); + zvec::test_util::RemoveTestPath(dir_); } void FlatSparseSearcherTest::TearDown(void) { - zvec::ailego::FileHelper::RemovePath(dir_.c_str()); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatSparseSearcherTest, TestGeneral) { @@ -101,7 +101,7 @@ TEST_F(FlatSparseSearcherTest, TestGeneral) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -162,7 +162,7 @@ TEST_F(FlatSparseSearcherTest, TestGeneral) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -212,7 +212,7 @@ TEST_F(FlatSparseSearcherTest, TestStreamerDump) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -233,7 +233,7 @@ TEST_F(FlatSparseSearcherTest, TestStreamerDump) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestStreamerDump"; + auto path = dir_ + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -298,7 +298,7 @@ TEST_F(FlatSparseSearcherTest, TestLoadClose) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -359,7 +359,7 @@ TEST_F(FlatSparseSearcherTest, TestLoadClose) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -417,7 +417,7 @@ TEST_F(FlatSparseSearcherTest, TestSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -439,7 +439,7 @@ TEST_F(FlatSparseSearcherTest, TestSearch) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -511,7 +511,7 @@ TEST_F(FlatSparseSearcherTest, TestSearchPKeys) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -539,7 +539,7 @@ TEST_F(FlatSparseSearcherTest, TestSearchPKeys) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -659,7 +659,7 @@ TEST_F(FlatSparseSearcherTest, TestMultiThread) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -724,7 +724,7 @@ TEST_F(FlatSparseSearcherTest, TestMultiThread) { ASSERT_EQ(2999, max); // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc index 915238265..b6bc546e0 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -23,7 +22,7 @@ #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace std; using namespace testing; @@ -50,7 +49,7 @@ class FlatSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string FlatSparseStreamerTest::dir_("FlatSparseStreamerTest"); +std::string FlatSparseStreamerTest::dir_("FlatSparseStreamerTest/"); shared_ptr FlatSparseStreamerTest::index_meta_ptr_; void FlatSparseStreamerTest::generate_sparse_data( @@ -88,19 +87,11 @@ void FlatSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void FlatSparseStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatSparseStreamerTest, TestGeneral) { @@ -119,7 +110,7 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { ailego::Params stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/Test/FlatSparseSearch", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "Test/FlatSparseSearch", true)); ASSERT_EQ(0, write_streamer->init(index_meta, params)); ASSERT_EQ(0, write_streamer->open(write_storage)); @@ -152,7 +143,7 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/FlatSparseSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/FlatSparseSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc index 07205d7ed..ffa383749 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -22,7 +21,7 @@ #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #include #include #include @@ -51,7 +50,7 @@ class FlatSparseStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatSparseStreamerTest::dir_("flat_sparse_streamer_test_dir"); +std::string FlatSparseStreamerTest::dir_("flat_sparse_streamer_test_dir/"); std::shared_ptr FlatSparseStreamerTest::index_meta_ptr_; void FlatSparseStreamerTest::generate_sparse_data( @@ -91,19 +90,11 @@ void FlatSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void FlatSparseStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatSparseStreamerTest, TestGeneral) { @@ -112,7 +103,7 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -182,7 +173,7 @@ TEST_F(FlatSparseStreamerTest, TestLinearSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -256,7 +247,7 @@ TEST_F(FlatSparseStreamerTest, TestLinearSearchByKeys) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -381,7 +372,7 @@ TEST_F(FlatSparseStreamerTest, TestCreateIterator) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCreateIterator", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -479,9 +470,9 @@ TEST_F(FlatSparseStreamerTest, TestOpenAndClose) { ASSERT_NE(nullptr, storage2); Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "/TessOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "TessOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "/TessOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "TessOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -580,7 +571,7 @@ TEST_F(FlatSparseStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -630,7 +621,7 @@ TEST_F(FlatSparseStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -668,7 +659,7 @@ TEST_F(FlatSparseStreamerTest, TestMultiThread) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -816,7 +807,7 @@ TEST_F(FlatSparseStreamerTest, TestConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -946,7 +937,7 @@ TEST_F(FlatSparseStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestFilter", true)); ASSERT_EQ(0, streamer->open(storage)); size_t cnt = 100UL; @@ -1032,7 +1023,7 @@ TEST_F(FlatSparseStreamerTest, TestProvider) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestProvider.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestProvider.index", true)); Params params; ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1115,7 +1106,7 @@ TEST_F(FlatSparseStreamerTest, TestProvider) { } // dump - // auto path1 = dir_ + "/TestProvider"; + // auto path1 = dir_ + "TestProvider"; // auto dumper1 = IndexFactory::CreateDumper("FileDumper"); // ASSERT_NE(dumper1, nullptr); // ASSERT_EQ(0, dumper1->create(path1)); @@ -1218,7 +1209,7 @@ TEST_F(FlatSparseStreamerTest, TestParamsMaxDocCount) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -1299,7 +1290,7 @@ TEST_F(FlatSparseStreamerTest, TestParamsDataChunkSize) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -1391,9 +1382,9 @@ TEST_F(FlatSparseStreamerTest, TestSharedContext) { streamer->open(storage); return streamer; }; - auto streamer1 = create_streamer(dir_ + "/TestSharedContext.index1"); - auto streamer2 = create_streamer(dir_ + "/TestSharedContext.index2"); - auto streamer3 = create_streamer(dir_ + "/TestSharedContext.index3"); + auto streamer1 = create_streamer(dir_ + "TestSharedContext.index1"); + auto streamer2 = create_streamer(dir_ + "TestSharedContext.index2"); + auto streamer3 = create_streamer(dir_ + "TestSharedContext.index3"); srand(Realtime::MilliSeconds()); IndexQueryMeta qmeta(IndexMeta::DT_FP32); @@ -1491,7 +1482,7 @@ TEST_F(FlatSparseStreamerTest, TestGroupBy) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1566,7 +1557,7 @@ TEST_F(FlatSparseStreamerTest, TestGroupByNotEnoughNum) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1641,7 +1632,7 @@ TEST_F(FlatSparseStreamerTest, TestAddAndSearchWithID) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); diff --git a/tests/core/algorithm/hnsw/hnsw_builder_test.cc b/tests/core/algorithm/hnsw/hnsw_builder_test.cc index e50952ae3..3d65c9af8 100644 --- a/tests/core/algorithm/hnsw/hnsw_builder_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_builder_test.cc @@ -19,7 +19,7 @@ #include #include #include "zvec/core/framework/index_framework.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -43,7 +43,7 @@ class HnswBuilderTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswBuilderTest::_dir("hnswBuilderTest"); +std::string HnswBuilderTest::_dir("hnswBuilderTest/"); shared_ptr HnswBuilderTest::_index_meta_ptr; void HnswBuilderTest::SetUp(void) { @@ -53,7 +53,7 @@ void HnswBuilderTest::SetUp(void) { } void HnswBuilderTest::TearDown(void) { - zvec::ailego::FileHelper::RemovePath(_dir.c_str()); + zvec::test_util::RemoveTestPath(_dir); } TEST_F(HnswBuilderTest, TestGeneral) { @@ -82,7 +82,7 @@ TEST_F(HnswBuilderTest, TestGeneral) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -190,7 +190,7 @@ TEST_F(HnswBuilderTest, TestIndexThreads) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndexThreads"; + string path = _dir + "TestIndexThreads"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder1->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -242,7 +242,7 @@ TEST_F(HnswBuilderTest, TestCosine) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosine"; + string path = _dir + "TestCosine"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -326,7 +326,7 @@ TEST_F(HnswBuilderTest, TestCosineFp16Converter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosineFp16Converter"; + string path = _dir + "TestCosineFp16Converter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -408,7 +408,7 @@ TEST_F(HnswBuilderTest, TestCosineInt8Converter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosineInt8Converter"; + string path = _dir + "TestCosineInt8Converter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -490,7 +490,7 @@ TEST_F(HnswBuilderTest, TestCosineInt4Converter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosineInt4Converter"; + string path = _dir + "TestCosineInt4Converter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); diff --git a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp index aca6169ff..5b6fce16a 100644 --- a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp @@ -24,7 +24,7 @@ #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" #include "hnsw_params.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace std; using namespace testing; @@ -49,7 +49,7 @@ class HnswSearcherTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswSearcherTest::_dir("HnswSearcherTest"); +std::string HnswSearcherTest::_dir("HnswSearcherTest/"); shared_ptr HnswSearcherTest::_index_meta_ptr; void HnswSearcherTest::SetUp(void) { @@ -59,7 +59,7 @@ void HnswSearcherTest::SetUp(void) { } void HnswSearcherTest::TearDown(void) { - zvec::ailego::FileHelper::RemovePath(_dir.c_str()); + zvec::test_util::RemoveTestPath(_dir); } TEST_F(HnswSearcherTest, TestRnnSearch) { @@ -81,7 +81,7 @@ TEST_F(HnswSearcherTest, TestRnnSearch) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRnnSearch"; + string path = _dir + "TestRnnSearch"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -147,7 +147,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchInnerProduct) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRnnSearchInnerProduct"; + string path = _dir + "TestRnnSearchInnerProduct"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -229,7 +229,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchCosine) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRnnSearchCosine"; + string path = _dir + "TestRnnSearchCosine"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -300,7 +300,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchMipsSquaredEuclidean) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -344,7 +344,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchMipsSquaredEuclidean) { ASSERT_LT(-radius, results[topk - 1].score()); } - auto path = _dir + "/TestStreamerDump"; + auto path = _dir + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -415,7 +415,7 @@ TEST_F(HnswSearcherTest, TestGeneral) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -560,7 +560,7 @@ TEST_F(HnswSearcherTest, TestClearAndReload) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -658,7 +658,7 @@ TEST_F(HnswSearcherTest, TestFilter) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -758,7 +758,7 @@ TEST_F(HnswSearcherTest, TestStreamerDump) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(*_index_meta_ptr, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -773,7 +773,7 @@ TEST_F(HnswSearcherTest, TestStreamerDump) { } streamer->add_impl(i, vec.data(), qmeta, ctx); } - auto path = _dir + "/TestStreamerDump"; + auto path = _dir + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -886,9 +886,9 @@ TEST_F(HnswSearcherTest, TestSharedContext) { size_t docs1 = rand() % 500 + 100; size_t docs2 = rand() % 5000 + 100; size_t docs3 = rand() % 50000 + 100; - auto path1 = _dir + "/TestSharedContext.index1"; - auto path2 = _dir + "/TestSharedContext.index2"; - auto path3 = _dir + "/TestSharedContext.index3"; + auto path1 = _dir + "TestSharedContext.index1"; + auto path2 = _dir + "TestSharedContext.index2"; + auto path3 = _dir + "TestSharedContext.index3"; auto searcher1 = gen_index(0, docs1, path1); auto searcher2 = gen_index(1, docs2, path2); auto searcher3 = gen_index(2, docs3, path3); @@ -956,7 +956,7 @@ TEST_F(HnswSearcherTest, TestSharedContext) { IndexFactory::CreateStreamer("HnswStreamer"); auto storage = IndexFactory::CreateStorage("MMapFileStorage"); storage->init(ailego::Params()); - storage->open(_dir + "/TestSharedContext.index4", true); + storage->open(_dir + "TestSharedContext.index4", true); streamer->init(*_index_meta_ptr, ailego::Params()); streamer->open(storage); NumericalVector query(dim); @@ -1010,7 +1010,7 @@ TEST_F(HnswSearcherTest, TestProvider) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestProvider"; + string path = _dir + "TestProvider"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1076,7 +1076,7 @@ TEST_F(HnswSearcherTest, TestMipsEuclideanMetric) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestMipsEuclideanMetric"; + string path = _dir + "TestMipsEuclideanMetric"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1131,7 +1131,7 @@ TEST_F(HnswSearcherTest, TestRandomPaddingTopk) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRandomPadding"; + string path = _dir + "TestRandomPadding"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1199,7 +1199,7 @@ TEST_F(HnswSearcherTest, TestBruteForceSetupInContext) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1336,7 +1336,7 @@ TEST_F(HnswSearcherTest, TestCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestCosine.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1364,7 +1364,7 @@ TEST_F(HnswSearcherTest, TestCosine) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestCosine"; + auto path = _dir + "TestCosine"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1473,7 +1473,7 @@ TEST_F(HnswSearcherTest, TestFetchVector) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVector.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestFetchVector.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1491,7 +1491,7 @@ TEST_F(HnswSearcherTest, TestFetchVector) { streamer->add_impl(i, vec.data(), qmeta, ctx); } - auto path = _dir + "/TestFetchVector"; + auto path = _dir + "TestFetchVector"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1593,7 +1593,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosine.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1623,7 +1623,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosine) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestFetchVectorCosine"; + auto path = _dir + "TestFetchVectorCosine"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1751,7 +1751,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineHalfFloatConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ( - 0, storage->open(_dir + "/TestFetchVectorCosineHalfFloatConverter.index", + 0, storage->open(_dir + "TestFetchVectorCosineHalfFloatConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1786,7 +1786,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineHalfFloatConverter) { vecs.push_back(vec); } - auto path = _dir + "/TestFetchVectorCosineHalfFloatConverter"; + auto path = _dir + "TestFetchVectorCosineHalfFloatConverter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1915,7 +1915,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineFp16Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosineFp16Converter.index", + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosineFp16Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1949,7 +1949,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineFp16Converter) { vecs.push_back(vec); } - auto path = _dir + "/TestFetchVectorCosineFp16Converter"; + auto path = _dir + "TestFetchVectorCosineFp16Converter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -2070,7 +2070,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt8Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosineInt8Converter.index", + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosineInt8Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2101,7 +2101,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt8Converter) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestFetchVectorCosineInt8Converter"; + auto path = _dir + "TestFetchVectorCosineInt8Converter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -2228,7 +2228,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt4Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosineInt4Converter.index", + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosineInt4Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2259,7 +2259,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt4Converter) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestFetchVectorCosineInt4Converter"; + auto path = _dir + "TestFetchVectorCosineInt4Converter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -2374,7 +2374,7 @@ TEST_F(HnswSearcherTest, TestGroup) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGroup"; + string path = _dir + "TestGroup"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -2501,7 +2501,7 @@ TEST_F(HnswSearcherTest, TestGroupNotEnoughNum) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGroupNotEnoughNum"; + string path = _dir + "TestGroupNotEnoughNum"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -2590,7 +2590,7 @@ TEST_F(HnswSearcherTest, TestGroupInBruteforceSearch) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGroupInBruteforceSearch"; + string path = _dir + "TestGroupInBruteforceSearch"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -2695,7 +2695,7 @@ TEST_F(HnswSearcherTest, TestBinaryConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestBinaryConverter.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestBinaryConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2726,7 +2726,7 @@ TEST_F(HnswSearcherTest, TestBinaryConverter) { vecs.push_back(vec); } - auto path = _dir + "/TestBinaryConverter"; + auto path = _dir + "TestBinaryConverter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp index 51244d9a6..94e35c938 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -9,7 +8,7 @@ #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -33,7 +32,7 @@ class HnswStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string HnswStreamerTest::dir_("hnsw_streamer_buffer_test_dir"); +std::string HnswStreamerTest::dir_("hnsw_streamer_buffer_test_dir/"); std::shared_ptr HnswStreamerTest::index_meta_ptr_; void HnswStreamerTest::SetUp(void) { @@ -41,19 +40,11 @@ void HnswStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void HnswStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswStreamerTest, TestHnswSearch) { @@ -70,7 +61,7 @@ TEST_F(HnswStreamerTest, TestHnswSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/HnswSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/HnswSearch", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -96,7 +87,7 @@ TEST_F(HnswStreamerTest, TestHnswSearch) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); @@ -193,7 +184,7 @@ TEST_F(HnswStreamerTest, TestHnswSearchMMap) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/HnswSearchMMap", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/HnswSearchMMap", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -220,7 +211,7 @@ TEST_F(HnswStreamerTest, TestHnswSearchMMap) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSearchMMap", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSearchMMap", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc index 6235502e4..ee25ba44a 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc @@ -18,23 +18,13 @@ #ifndef _MSC_VER #include #include -#else -#include -#include -inline void sleep(unsigned int seconds) { - std::this_thread::sleep_for(std::chrono::seconds(seconds)); -} -inline void usleep(unsigned int microseconds) { - std::this_thread::sleep_for(std::chrono::microseconds(microseconds)); -} #endif -#include #include #include #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -59,7 +49,7 @@ class HnswStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswStreamerTest::dir_("hnsw_streamer_test_dir"); +std::string HnswStreamerTest::dir_("hnsw_streamer_test_dir/"); shared_ptr HnswStreamerTest::index_meta_ptr_; void HnswStreamerTest::SetUp(void) { @@ -67,19 +57,11 @@ void HnswStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, ailego::Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void HnswStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswStreamerTest, TestAddVector) { @@ -96,7 +78,7 @@ TEST_F(HnswStreamerTest, TestAddVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/AddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -128,7 +110,7 @@ TEST_F(HnswStreamerTest, TestLinearSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -198,7 +180,7 @@ TEST_F(HnswStreamerTest, TestLinearSearchByKeys) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -314,7 +296,7 @@ TEST_F(HnswStreamerTest, TestKnnSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -398,7 +380,7 @@ TEST_F(HnswStreamerTest, TestAddAndSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -489,7 +471,7 @@ TEST_F(HnswStreamerTest, TestKnnSearchRandomData) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearchRandomData", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearchRandomData", true)); ASSERT_EQ(0, streamer->init(meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -580,9 +562,9 @@ TEST_F(HnswStreamerTest, TestOpenClose) { ASSERT_NE(nullptr, storage2); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "/TessOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "TessOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "/TessOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "TessOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -663,7 +645,7 @@ TEST_F(HnswStreamerTest, TestCreateIterator) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCreateIterator", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -736,7 +718,7 @@ TEST_F(HnswStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -777,7 +759,7 @@ TEST_F(HnswStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -813,7 +795,7 @@ TEST_F(HnswStreamerTest, TestKnnMultiThread) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -939,7 +921,7 @@ TEST_F(HnswStreamerTest, TestKnnConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -1050,7 +1032,7 @@ TEST_F(HnswStreamerTest, TestBfThreshold) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessBfThreshold", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessBfThreshold", true)); ASSERT_EQ(0, streamer->open(storage)); NumericalVector vec(dim); @@ -1127,7 +1109,7 @@ TEST_F(HnswStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessFilter", true)); ASSERT_EQ(0, streamer->open(storage)); @@ -1206,7 +1188,7 @@ TEST_F(HnswStreamerTest, TestMaxIndexSize) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessMaxIndexSize", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessMaxIndexSize", true)); ASSERT_EQ(0, streamer->open(storage)); size_t vsz0 = 0; @@ -1250,7 +1232,7 @@ TEST_F(HnswStreamerTest, TestKnnCleanUp) { ASSERT_NE(nullptr, storage1); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "/TessKnnCluenUp1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "TessKnnCluenUp1", true)); ailego::Params params; constexpr size_t static dim1 = 32; IndexMeta meta1(IndexMeta::DataType::DT_FP32, dim1); @@ -1267,7 +1249,7 @@ TEST_F(HnswStreamerTest, TestKnnCleanUp) { auto storage2 = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage2); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "/TessKnnCluenUp2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "TessKnnCluenUp2", true)); constexpr size_t static dim2 = 64; IndexMeta meta2(IndexMeta::DataType::DT_FP32, dim2); meta2.set_metric("SquaredEuclidean", 0, ailego::Params()); @@ -1290,7 +1272,7 @@ TEST_F(HnswStreamerTest, TestIndexSizeQuota) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestIndexSizeQuota", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestIndexSizeQuota", true)); ailego::Params params; constexpr size_t static dim = 512; IndexMeta meta(IndexMeta::DataType::DT_FP32, dim); @@ -1327,7 +1309,7 @@ TEST_F(HnswStreamerTest, TestBloomFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestBloomFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestBloomFilter", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 16); @@ -1366,7 +1348,7 @@ TEST_F(HnswStreamerTest, TestStreamerParams) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerParams", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestStreamerParams", true)); ailego::Params params; params.set("proxima.hnsw.streamer.docs_hard_limit", 5); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); @@ -1395,7 +1377,7 @@ TEST_F(HnswStreamerTest, TestCheckCrc) ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - std::string path = dir_ + "/TestCheckCrc"; + std::string path = dir_ + "TestCheckCrc"; ASSERT_EQ(0, storage->open(path, true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); @@ -1453,7 +1435,7 @@ TEST_F(HnswStreamerTest, TestCheckStats) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - std::string path = dir_ + "/TestCheckStats.index"; + std::string path = dir_ + "TestCheckStats.index"; ASSERT_EQ(0, storage->open(path, true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 100); @@ -1549,7 +1531,7 @@ TEST_F(HnswStreamerTest, TestCheckStats) { ASSERT_EQ(createTime3, createTime1); ASSERT_GT(updateTime3, updateTime2); - auto dpath = dir_ + "/dumpIndex"; + auto dpath = dir_ + "dumpIndex"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(dpath)); @@ -1573,7 +1555,7 @@ TEST_F(HnswStreamerTest, TestCheckDuplicateAndGetVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCheckDuplicateAndGetVec", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCheckDuplicateAndGetVec", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 5); @@ -1646,7 +1628,7 @@ TEST_F(HnswStreamerTest, TestDumpIndexAndAdd) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestDumpIndexAndAdd", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestDumpIndexAndAdd", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 5); @@ -1682,7 +1664,7 @@ TEST_F(HnswStreamerTest, TestDumpIndexAndAdd) { }; addVector(0, 2000, false); auto t2 = std::async(std::launch::async, addVector, 2000, 3000, true); - auto path1 = dir_ + "/dumpIndex1"; + auto path1 = dir_ + "dumpIndex1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); @@ -1745,7 +1727,7 @@ TEST_F(HnswStreamerTest, TestProvider) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGetVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGetVector", true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_STREAMER_SCALING_FACTOR, 5); @@ -1788,7 +1770,7 @@ TEST_F(HnswStreamerTest, TestProvider) { streamer->add_impl(keys[i], vec.data(), qmeta, ctx); } - auto path1 = dir_ + "/TestGetVector1"; + auto path1 = dir_ + "TestGetVector1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); @@ -1870,9 +1852,9 @@ TEST_F(HnswStreamerTest, TestSharedContext) { streamer->open(storage); return streamer; }; - auto streamer1 = create_streamer(dir_ + "/TestSharedContext.index1"); - auto streamer2 = create_streamer(dir_ + "/TestSharedContext.index2"); - auto streamer3 = create_streamer(dir_ + "/TestSharedContext.index3"); + auto streamer1 = create_streamer(dir_ + "TestSharedContext.index1"); + auto streamer2 = create_streamer(dir_ + "TestSharedContext.index2"); + auto streamer3 = create_streamer(dir_ + "TestSharedContext.index3"); srand(ailego::Realtime::MilliSeconds()); IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); @@ -1955,7 +1937,7 @@ TEST_F(HnswStreamerTest, TestMipsEuclideanMetric) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestMipsSquaredEuclidean", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestMipsSquaredEuclidean", true)); const size_t COUNT = 10000; IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); { @@ -2023,7 +2005,7 @@ TEST_F(HnswStreamerTest, TestBruteForceSetupInContext) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ(0, - storage->open(dir_ + "/TestBruteForceSetupInContext.index", true)); + storage->open(dir_ + "TestBruteForceSetupInContext.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2156,7 +2138,7 @@ TEST_F(HnswStreamerTest, TestKnnSearchCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearchCosine.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearchCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2272,7 +2254,7 @@ TEST_F(HnswStreamerTest, TestFetchVector) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVector.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVector.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2369,7 +2351,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosine.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2498,7 +2480,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineHalfFloatConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ( - 0, storage->open(dir_ + "/TestFetchVectorCosineHalfFloatConverter.index", + 0, storage->open(dir_ + "TestFetchVectorCosineHalfFloatConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2638,7 +2620,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineFp16Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosineFp16Converter.index", + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosineFp16Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2772,7 +2754,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosineInt8Converter.index", + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosineInt8Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2903,7 +2885,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt4Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosineInt4Converter.index", + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosineInt4Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3018,7 +3000,7 @@ TEST_F(HnswStreamerTest, TestRnnSearch) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestRnnSearchInnerProduct.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestRnnSearchInnerProduct.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3081,7 +3063,7 @@ TEST_F(HnswStreamerTest, TestRnnSearchInnerProduct) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestRnnSearchInnerProduct.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestRnnSearchInnerProduct.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3158,7 +3140,7 @@ TEST_F(HnswStreamerTest, TestRnnSearchCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestRnnSearchCosine.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestRnnSearchCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3233,7 +3215,7 @@ TEST_F(HnswStreamerTest, TestGroup) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3348,7 +3330,7 @@ TEST_F(HnswStreamerTest, TestGroupNotEnoughNum) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroupNotEnoughNum.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroupNotEnoughNum.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3427,7 +3409,7 @@ TEST_F(HnswStreamerTest, TestGroupInBruteforceSearch) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ(0, - storage->open(dir_ + "/TestGroupInBruteforceSearch.index", true)); + storage->open(dir_ + "TestGroupInBruteforceSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3522,7 +3504,7 @@ TEST_F(HnswStreamerTest, TestBinaryConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestBinaryConverter.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestBinaryConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3587,7 +3569,7 @@ TEST_F(HnswStreamerTest, TestAddAndSearchWithID) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3722,7 +3704,7 @@ TEST_F(HnswStreamerTest, TestBasicRefiner) { ailego::Params base_stg_params; auto base_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, base_storage->init(base_stg_params)); - ASSERT_EQ(0, base_storage->open(dir_ + "/TestBasicRefinerBase.index", true)); + ASSERT_EQ(0, base_storage->open(dir_ + "TestBasicRefinerBase.index", true)); ASSERT_EQ(0, base_streamer->init(index_meta_binary, params)); ASSERT_EQ(0, base_streamer->open(base_storage)); @@ -3734,7 +3716,7 @@ TEST_F(HnswStreamerTest, TestBasicRefiner) { auto refine_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, refine_storage->init(refine_stg_params)); ASSERT_EQ(0, - refine_storage->open(dir_ + "/TestBasicRefinerRefine.index", true)); + refine_storage->open(dir_ + "TestBasicRefinerRefine.index", true)); ASSERT_EQ(0, refine_streamer->init(index_meta, params)); ASSERT_EQ(0, refine_streamer->open(refine_storage)); auto refine_ctx = refine_streamer->create_context(); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc index 6082fa0bc..ae0d73c83 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "hnsw_sparse_builder.h" -#include #include #include #include @@ -20,8 +19,8 @@ #include #include #include "zvec/core/framework/index_framework.h" -#include "zvec/ailego/utility/file_helper.h" #include "hnsw_sparse_params.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -44,7 +43,7 @@ class HnswSparseBuilderTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswSparseBuilderTest::_dir("HnswSparseBuilderTest"); +std::string HnswSparseBuilderTest::_dir("HnswSparseBuilderTest/"); shared_ptr HnswSparseBuilderTest::_index_meta_ptr; void HnswSparseBuilderTest::SetUp(void) { @@ -54,11 +53,7 @@ void HnswSparseBuilderTest::SetUp(void) { } void HnswSparseBuilderTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(_dir.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + _dir + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(_dir); } TEST_F(HnswSparseBuilderTest, TestGeneral) { @@ -96,7 +91,7 @@ TEST_F(HnswSparseBuilderTest, TestGeneral) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -234,7 +229,7 @@ TEST_F(HnswSparseBuilderTest, TestIndexThreads) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndexThreads"; + string path = _dir + "TestIndexThreads"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder1->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -293,7 +288,7 @@ TEST_F(HnswSparseBuilderTest, TestHalfFloatConverter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestHalFloatConverter"; + string path = _dir + "TestHalFloatConverter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -389,7 +384,7 @@ TEST_F(HnswSparseBuilderTest, TestIndptr) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndptr"; + string path = _dir + "TestIndptr"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -451,7 +446,7 @@ TEST_F(HnswSparseBuilderTest, TestIndptrFp16) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndptrFp16"; + string path = _dir + "TestIndptrFp16"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp index d2f0503db..cba21a8f0 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp @@ -21,7 +21,7 @@ #include #include #include "zvec/core/framework/index_framework.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #include "hnsw_sparse_params.h" #if defined(__GNUC__) || defined(__GNUG__) @@ -51,7 +51,7 @@ class HnswSparseSearcherTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswSparseSearcherTest::dir_("HnswSparseSearcherTest"); +std::string HnswSparseSearcherTest::dir_("HnswSparseSearcherTest/"); shared_ptr HnswSparseSearcherTest::_index_meta_ptr; void HnswSparseSearcherTest::generate_sparse_data( @@ -91,7 +91,7 @@ void HnswSparseSearcherTest::SetUp(void) { } void HnswSparseSearcherTest::TearDown(void) { - zvec::ailego::FileHelper::RemovePath(dir_.c_str()); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswSparseSearcherTest, TestGeneral) { @@ -115,7 +115,7 @@ TEST_F(HnswSparseSearcherTest, TestGeneral) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -138,7 +138,7 @@ TEST_F(HnswSparseSearcherTest, TestGeneral) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestGeneral"; + auto path = dir_ + "TestGeneral"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -248,7 +248,7 @@ TEST_F(HnswSparseSearcherTest, TestRnnSearch) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestRnnSearch"; + string path = dir_ + "TestRnnSearch"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -333,7 +333,7 @@ TEST_F(HnswSparseSearcherTest, TestClearAndReload) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestClearAndReload"; + string path = dir_ + "TestClearAndReload"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -468,7 +468,7 @@ TEST_F(HnswSparseSearcherTest, TestFilter) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestFilter"; + string path = dir_ + "TestFilter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -620,7 +620,7 @@ TEST_F(HnswSparseSearcherTest, TestBatchQuery) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestBatchQuery"; + string path = dir_ + "TestBatchQuery"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -755,7 +755,7 @@ TEST_F(HnswSparseSearcherTest, TestStreamerDump) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(*_index_meta_ptr, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -777,7 +777,7 @@ TEST_F(HnswSparseSearcherTest, TestStreamerDump) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestStreamerDump"; + auto path = dir_ + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -912,9 +912,9 @@ TEST_F(HnswSparseSearcherTest, TestSharedContext) { size_t docs1 = rand() % 500 + 100; size_t docs2 = rand() % 5000 + 100; size_t docs3 = rand() % 50000 + 100; - auto path1 = dir_ + "/TestSharedContext.index1"; - auto path2 = dir_ + "/TestSharedContext.index2"; - auto path3 = dir_ + "/TestSharedContext.index3"; + auto path1 = dir_ + "TestSharedContext.index1"; + auto path2 = dir_ + "TestSharedContext.index2"; + auto path3 = dir_ + "TestSharedContext.index3"; auto searcher1 = gen_index(0, docs1, path1); auto searcher2 = gen_index(1, docs2, path2); auto searcher3 = gen_index(2, docs3, path3); @@ -999,7 +999,7 @@ TEST_F(HnswSparseSearcherTest, TestSharedContext) { IndexFactory::CreateStreamer("HnswSparseStreamer"); auto storage = IndexFactory::CreateStorage("MMapFileStorage"); storage->init(ailego::Params()); - storage->open(dir_ + "/TestSharedContext.index4", true); + storage->open(dir_ + "TestSharedContext.index4", true); streamer->init(*_index_meta_ptr, ailego::Params()); streamer->open(storage); @@ -1074,7 +1074,7 @@ TEST_F(HnswSparseSearcherTest, TestProvider) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestProvider"; + string path = dir_ + "TestProvider"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1161,7 +1161,7 @@ TEST_F(HnswSparseSearcherTest, TestRandomPaddingTopk) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestRandomPadding"; + string path = dir_ + "TestRandomPadding"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1248,7 +1248,7 @@ TEST_F(HnswSparseSearcherTest, TestBruteForceSetupInContext) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestBruteForceSetupInContext"; + string path = dir_ + "TestBruteForceSetupInContext"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1394,7 +1394,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatConverter) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestHalfFloatConverter.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestHalfFloatConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1425,7 +1425,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatConverter) { new_vec.data(), new_meta, ctx)); } - auto path = dir_ + "/TestHalfFloatConverter"; + auto path = dir_ + "TestHalfFloatConverter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1531,7 +1531,7 @@ TEST_F(HnswSparseSearcherTest, TestQueryFilteringRatio) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestQueryFilteringRatio.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestQueryFilteringRatio.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1554,7 +1554,7 @@ TEST_F(HnswSparseSearcherTest, TestQueryFilteringRatio) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestQueryFilteringRatio"; + auto path = dir_ + "TestQueryFilteringRatio"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1671,7 +1671,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatRevert) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestHalfFloatRevert.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestHalfFloatRevert.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1729,7 +1729,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatRevert) { } } - auto path = dir_ + "/TestHalfFloatRevert"; + auto path = dir_ + "TestHalfFloatRevert"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp index 7c32520fe..70405d0fd 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include @@ -22,7 +21,7 @@ #include #include #include "hnsw_sparse_streamer.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" using namespace std; using namespace testing; @@ -49,7 +48,7 @@ class HnswSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswSparseStreamerTest::dir_("hnsw_sparse_streamer_buffer_test_dir"); +std::string HnswSparseStreamerTest::dir_("hnsw_sparse_streamer_buffer_test_dir/"); shared_ptr HnswSparseStreamerTest::index_meta_ptr_; void HnswSparseStreamerTest::generate_sparse_data( @@ -87,19 +86,11 @@ void HnswSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void HnswSparseStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswSparseStreamerTest, TestGeneral) { @@ -124,7 +115,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { ailego::Params stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/Test/HnswSparseSearch", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "Test/HnswSparseSearch", true)); ASSERT_EQ(0, write_streamer->init(index_meta, params)); ASSERT_EQ(0, write_streamer->open(write_storage)); @@ -156,7 +147,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSparseSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSparseSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); @@ -253,7 +244,7 @@ TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) { ailego::Params stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/Test/HnswSparseSearch", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "Test/HnswSparseSearch", true)); ASSERT_EQ(0, write_streamer->init(index_meta, params)); ASSERT_EQ(0, write_streamer->open(write_storage)); @@ -285,7 +276,7 @@ TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSparseSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSparseSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc index 6b16db31d..1cec7c4f2 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc @@ -13,21 +13,11 @@ // limitations under the License. #include "hnsw_sparse_streamer.h" #include -#include #include #include #ifndef _MSC_VER #include #include -#else -#include -#include -inline void sleep(unsigned int seconds) { - std::this_thread::sleep_for(std::chrono::seconds(seconds)); -} -inline void usleep(unsigned int microseconds) { - std::this_thread::sleep_for(std::chrono::microseconds(microseconds)); -} #endif #include #include @@ -35,7 +25,7 @@ inline void usleep(unsigned int microseconds) { #include #include #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -64,7 +54,7 @@ class HnswSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswSparseStreamerTest::dir_("HnswSparseStreamerTest"); +std::string HnswSparseStreamerTest::dir_("HnswSparseStreamerTest/"); shared_ptr HnswSparseStreamerTest::index_meta_ptr_; void HnswSparseStreamerTest::generate_sparse_data( @@ -102,19 +92,11 @@ void HnswSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } void HnswSparseStreamerTest::TearDown(void) { - if (!zvec::ailego::FileHelper::RemovePath(dir_.c_str())) { -#ifdef _WIN32 - system(("rmdir /s /q " + dir_ + " 2>NUL").c_str()); -#endif - } + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswSparseStreamerTest, TestGeneral) { @@ -138,7 +120,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -248,7 +230,7 @@ TEST_F(HnswSparseStreamerTest, TestAddVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -284,7 +266,7 @@ TEST_F(HnswSparseStreamerTest, TestLinearSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -364,7 +346,7 @@ TEST_F(HnswSparseStreamerTest, TestLinearSearchByKeys) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -498,9 +480,9 @@ TEST_F(HnswSparseStreamerTest, TestOpenClose) { ASSERT_NE(nullptr, storage2); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "/TessOpenAndClose1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "TessOpenAndClose1", true)); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "/TessOpenAndClose2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "TessOpenAndClose2", true)); ASSERT_EQ(0, streamer->init(meta, params)); auto checkIter = [](size_t base, size_t total, IndexStreamer::Pointer &streamer) { @@ -593,7 +575,7 @@ TEST_F(HnswSparseStreamerTest, TestCreateIterator) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCreateIterator", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -681,7 +663,7 @@ TEST_F(HnswSparseStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -732,7 +714,7 @@ TEST_F(HnswSparseStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -777,7 +759,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnMultiThread) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnMultiThread", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnMultiThread", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -928,7 +910,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnConcurrentAddAndSearch) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessKnnConcurrentAddAndSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessKnnConcurrentAddAndSearch", true)); ASSERT_EQ(0, streamer->open(storage)); auto addVector = [&streamer](int baseKey, size_t addCnt) { @@ -1060,7 +1042,7 @@ TEST_F(HnswSparseStreamerTest, TestBfThreshold) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TessBfThreshold", true)); + ASSERT_EQ(0, storage->open(dir_ + "TessBfThreshold", true)); ASSERT_EQ(0, streamer->open(storage)); size_t cnt = 10000; @@ -1148,7 +1130,7 @@ TEST_F(HnswSparseStreamerTest, TestFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestFilter", true)); ASSERT_EQ(0, streamer->open(storage)); size_t cnt = 100UL; @@ -1239,7 +1221,7 @@ TEST_F(HnswSparseStreamerTest, TestMaxIndexSize) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestMaxIndexSize", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestMaxIndexSize", true)); ASSERT_EQ(0, streamer->open(storage)); size_t vsz0 = 0; @@ -1297,7 +1279,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnCleanUp) { ASSERT_NE(nullptr, storage1); ailego::Params stg_params; ASSERT_EQ(0, storage1->init(stg_params)); - ASSERT_EQ(0, storage1->open(dir_ + "/TestKnnCluenUp1", true)); + ASSERT_EQ(0, storage1->open(dir_ + "TestKnnCluenUp1", true)); ailego::Params params; constexpr size_t static sparse_dim_count1 = 32; @@ -1325,7 +1307,7 @@ TEST_F(HnswSparseStreamerTest, TestKnnCleanUp) { auto storage2 = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage2); ASSERT_EQ(0, storage2->init(stg_params)); - ASSERT_EQ(0, storage2->open(dir_ + "/TestKnnCluenUp2", true)); + ASSERT_EQ(0, storage2->open(dir_ + "TestKnnCluenUp2", true)); constexpr size_t static sparse_dim_count2 = 64; IndexMeta meta2(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP32); @@ -1362,7 +1344,7 @@ TEST_F(HnswSparseStreamerTest, TestIndexSizeQuota) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestIndexSizeQuota", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestIndexSizeQuota", true)); ailego::Params params; IndexMeta meta(IndexMeta::MetaType::MT_SPARSE, IndexMeta::DataType::DT_FP32); @@ -1408,7 +1390,7 @@ TEST_F(HnswSparseStreamerTest, TestBloomFilter) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestBloomFilter", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestBloomFilter", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 16); @@ -1456,7 +1438,7 @@ TEST_F(HnswSparseStreamerTest, TestStreamerParams) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerParams", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestStreamerParams", true)); ailego::Params params; params.set("proxima.hnsw.sparse_streamer.docs_hard_limit", 5); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); @@ -1499,7 +1481,7 @@ TEST_F(HnswSparseStreamerTest, TestCheckStats) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - std::string path = dir_ + "/TestCheckStats.index"; + std::string path = dir_ + "TestCheckStats.index"; ASSERT_EQ(0, storage->open(path, true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 100); @@ -1613,7 +1595,7 @@ TEST_F(HnswSparseStreamerTest, TestCheckStats) { ASSERT_EQ(createTime3, createTime1); ASSERT_GT(updateTime3, updateTime2); - auto dpath = dir_ + "/dumpIndex"; + auto dpath = dir_ + "dumpIndex"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(dpath)); @@ -1637,7 +1619,7 @@ TEST_F(HnswSparseStreamerTest, TestCheckDuplicateAndGetVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCheckDuplicateAndGetVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCheckDuplicateAndGetVector", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 5); @@ -1732,7 +1714,7 @@ TEST_F(HnswSparseStreamerTest, TestDumpIndexAndAdd) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestDumpIndexAndAdd", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestDumpIndexAndAdd", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 5); @@ -1772,7 +1754,7 @@ TEST_F(HnswSparseStreamerTest, TestDumpIndexAndAdd) { addVector(0, 2000, false); auto t2 = std::async(std::launch::async, addVector, 2000, 3000, true); - auto path1 = dir_ + "/dumpIndex1"; + auto path1 = dir_ + "dumpIndex1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); @@ -1846,7 +1828,7 @@ TEST_F(HnswSparseStreamerTest, TestProvider) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestProvider.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestProvider.index", true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 10); params.set(PARAM_HNSW_SPARSE_STREAMER_SCALING_FACTOR, 5); @@ -1897,7 +1879,7 @@ TEST_F(HnswSparseStreamerTest, TestProvider) { sparse_velues.data(), qmeta, ctx)); } - auto path1 = dir_ + "/TestProvider"; + auto path1 = dir_ + "TestProvider"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); @@ -2005,9 +1987,9 @@ TEST_F(HnswSparseStreamerTest, TestSharedContext) { streamer->open(storage); return streamer; }; - auto streamer1 = create_streamer(dir_ + "/TestSharedContext.index1"); - auto streamer2 = create_streamer(dir_ + "/TestSharedContext.index2"); - auto streamer3 = create_streamer(dir_ + "/TestSharedContext.index3"); + auto streamer1 = create_streamer(dir_ + "TestSharedContext.index1"); + auto streamer2 = create_streamer(dir_ + "TestSharedContext.index2"); + auto streamer3 = create_streamer(dir_ + "TestSharedContext.index3"); srand(ailego::Realtime::MilliSeconds()); IndexQueryMeta qmeta(IndexMeta::MetaType::MT_SPARSE, @@ -2112,7 +2094,7 @@ TEST_F(HnswSparseStreamerTest, TestBruteForceSetupInContext) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ(0, - storage->open(dir_ + "/TestBruteForceSetupInContext.index", true)); + storage->open(dir_ + "TestBruteForceSetupInContext.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2252,7 +2234,7 @@ TEST_F(HnswSparseStreamerTest, TestQueryFilteringRatio) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestQueryFilteringRatio", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestQueryFilteringRatio", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2368,7 +2350,7 @@ TEST_F(HnswSparseStreamerTest, TestAddAndSearchWithID) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); diff --git a/tests/core/interface/index_interface_test.cc b/tests/core/interface/index_interface_test.cc index 66cfae723..b97405488 100644 --- a/tests/core/interface/index_interface_test.cc +++ b/tests/core/interface/index_interface_test.cc @@ -22,6 +22,7 @@ #include "zvec/core/interface/index_factory.h" #include "zvec/core/interface/index_param.h" #include "zvec/core/interface/index_param_builders.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -30,26 +31,13 @@ using namespace zvec::core_interface; -static void remove_files(const std::string &pattern) { - if (pattern.find('*') != std::string::npos || - pattern.find('?') != std::string::npos) { -#ifdef _WIN32 - system(("del /f /q " + pattern + " 2>NUL").c_str()); -#else - system(("rm -rf " + pattern).c_str()); -#endif - } else { - zvec::ailego::FileHelper::RemovePath(pattern.c_str()); - } -} - TEST(IndexInterface, General) { constexpr uint32_t kDimension = 64; const std::string index_name{"test.index"}; auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { - remove_files(index_name); + zvec::test_util::RemoveTestFiles(index_name); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -97,7 +85,7 @@ TEST(IndexInterface, General) { ASSERT_FLOAT_EQ(1.0f, fetched_vector[1]); ASSERT_FLOAT_EQ(2.0f, fetched_vector[2]); index->Close(); - remove_files(index_name); + zvec::test_util::RemoveTestFiles(index_name); }; @@ -170,7 +158,7 @@ TEST(IndexInterface, BufferGeneral) { auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { std::string real_index_name = index_name; - remove_files(index_name + "*"); + zvec::test_util::RemoveTestFiles(index_name + "*"); auto write_index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, write_index); @@ -223,7 +211,7 @@ TEST(IndexInterface, BufferGeneral) { ASSERT_FLOAT_EQ(1.0f, fetched_vector[1]); ASSERT_FLOAT_EQ(2.0f, fetched_vector[2]); read_index->Close(); - remove_files(index_name + "*"); + zvec::test_util::RemoveTestFiles(index_name + "*"); }; @@ -279,7 +267,7 @@ TEST(IndexInterface, SparseGeneral) { auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { - remove_files(index_name); + zvec::test_util::RemoveTestFiles(index_name); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -345,7 +333,7 @@ TEST(IndexInterface, SparseGeneral) { ASSERT_EQ(i, fetched_values[i]); } index->Close(); - remove_files(index_name); + zvec::test_util::RemoveTestFiles(index_name); }; @@ -394,7 +382,7 @@ TEST(IndexInterface, Merge) { const std::string index_name{"test.index"}; auto del_index_file_func = [&](const std::string file_name) { - remove_files(file_name); + zvec::test_util::RemoveTestFiles(file_name); }; auto create_index_func = @@ -769,7 +757,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - remove_files("test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test invalid vector data type for sparse operations @@ -792,7 +780,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - remove_files("test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test fetch non-existent document @@ -813,7 +801,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - remove_files("test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test search with invalid vector data @@ -846,7 +834,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - remove_files("test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test merge with invalid write concurrency @@ -890,9 +878,9 @@ TEST(IndexInterface, Failure) { index1->Close(); index2->Close(); index3->Close(); - remove_files("test1.index"); - remove_files("test2.index"); - remove_files("test3.index"); + zvec::test_util::RemoveTestFiles("test1.index"); + zvec::test_util::RemoveTestFiles("test2.index"); + zvec::test_util::RemoveTestFiles("test3.index"); } } @@ -1022,7 +1010,7 @@ TEST(IndexInterface, Score) { auto sparse_indices = std::vector{0, 1, 2}; auto query_vector = std::vector{1.0f, 2.0f, 3.0f}; - remove_files(index_file_path); + zvec::test_util::RemoveTestFiles(index_file_path); auto check_score = [&](const SearchResult &result, MetricType metric_type) { ASSERT_EQ(result.doc_list_.size(), 2); @@ -1093,7 +1081,7 @@ TEST(IndexInterface, Score) { auto dense_func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer query_param, MetricType metric_type) { - remove_files(index_file_path); + zvec::test_util::RemoveTestFiles(index_file_path); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -1115,13 +1103,13 @@ TEST(IndexInterface, Score) { check_score(result, metric_type); index->Close(); - remove_files(index_file_path); + zvec::test_util::RemoveTestFiles(index_file_path); }; auto sparse_func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer query_param, MetricType metric_type) { - remove_files(index_file_path); + zvec::test_util::RemoveTestFiles(index_file_path); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -1149,7 +1137,7 @@ TEST(IndexInterface, Score) { check_score(result, metric_type); index->Close(); - remove_files(index_file_path); + zvec::test_util::RemoveTestFiles(index_file_path); }; constexpr uint32_t kDimension = 3; diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc index aba343137..620b4ef6f 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc @@ -15,7 +15,7 @@ #include #include "db/index/column/inverted_column/inverted_indexer.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -230,7 +230,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -241,7 +241,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc index d78fcc7f3..953a45f7f 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc @@ -15,7 +15,7 @@ #include #include "db/index/column/inverted_column/inverted_indexer.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -255,7 +255,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -266,7 +266,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc index 5562e6495..07bc1e7da 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc @@ -16,7 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -369,7 +369,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -380,7 +380,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc index e8db1fb1d..8dfe7cda1 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc @@ -16,7 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -480,7 +480,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -491,7 +491,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc index 8980a66f3..b1da86595 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc @@ -16,7 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -293,7 +293,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -304,7 +304,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc b/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc index 0d25a649d..858af223d 100644 --- a/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc +++ b/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc @@ -14,7 +14,7 @@ #include -#include "zvec/ailego/utility/file_helper.h" +#include "tests/test_util.h" #define private public #define protected public #include "db/index/column/inverted_column/inverted_indexer.h" @@ -39,7 +39,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -50,7 +50,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - zvec::ailego::FileHelper::RemovePath(working_dir.c_str()); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/vector_column_indexer_test.cc b/tests/db/index/column/vector_column_indexer_test.cc index 792ebf6e7..208a41077 100644 --- a/tests/db/index/column/vector_column_indexer_test.cc +++ b/tests/db/index/column/vector_column_indexer_test.cc @@ -21,6 +21,7 @@ #include "zvec/ailego/utility/float_helper.h" #include "zvec/db/doc.h" #include "zvec/db/index_params.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -61,9 +62,7 @@ TEST(VectorColumnIndexerTest, General) { const std::string index_file_path = "test_indexer.index"; constexpr idx_t kDocId = 2345; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -172,7 +171,7 @@ TEST(VectorColumnIndexerTest, General) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP), @@ -214,9 +213,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeFP16) { constexpr idx_t kDocId = 2345; constexpr int dimension = 4; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -355,7 +352,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeFP16) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP), @@ -371,9 +368,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeINT8) { constexpr idx_t kDocId = 2345; constexpr int dimension = 4; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -495,7 +490,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeINT8) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP), @@ -511,9 +506,7 @@ TEST(VectorColumnIndexerTest, SparseGeneral) { const std::string index_file_path = "test_indexer.index"; constexpr idx_t kDocId = 2345; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // create indexer auto indexer = std::make_shared( @@ -607,7 +600,7 @@ TEST(VectorColumnIndexerTest, SparseGeneral) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP)); @@ -623,9 +616,7 @@ TEST(VectorColumnIndexerTest, SparseDataTypeFP16) { const std::string index_file_path = "test_indexer.index"; constexpr idx_t kDocId = 2345; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // create indexer auto indexer = std::make_shared( @@ -723,7 +714,7 @@ TEST(VectorColumnIndexerTest, SparseDataTypeFP16) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP)); @@ -734,9 +725,8 @@ TEST(VectorColumnIndexerTest, Merge) { constexpr uint32_t kDimension = 64; const std::string index_name{"test_indexer.index"}; - auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = @@ -1003,9 +993,8 @@ TEST(VectorColumnIndexerTest, SparseMerge) { constexpr uint32_t kUnitSize = sizeof(float); // VECTOR_FP32 const std::string index_name{"test_indexer.index"}; - auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = @@ -1281,9 +1270,7 @@ TEST(VectorColumnIndexerTest, BfPks) { auto func = [&](const IndexParams::Ptr index_params) { const std::string index_file_path = "test_indexer.index"; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -1399,7 +1386,7 @@ TEST(VectorColumnIndexerTest, BfPks) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::COSINE)); @@ -1586,12 +1573,10 @@ TEST(VectorColumnIndexerTest, CosineGeneral) { // will be too large due to float's precision const uint32_t kTopk = 10; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto func = [&](const IndexParams::Ptr index_params, DataType data_type) { - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto indexer = std::make_shared( index_file_path, FieldSchema("test", data_type, kDim, false, index_params)); @@ -1744,9 +1729,7 @@ TEST(VectorColumnIndexerTest, Score) { auto sparse_indices = std::vector{0, 1, 2}; auto query_vector = std::vector{1.0f, 2.0f, 3.0f}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto check_score = [&](VectorIndexResults *vector_results, @@ -1948,9 +1931,7 @@ TEST(VectorColumnIndexerTest, Failure) { constexpr idx_t kDocId = 1234; auto vector = std::vector{1.0f, 2.0f, 3.0f}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // Test case 1: Operations on unopened indexer { @@ -2331,16 +2312,15 @@ TEST(VectorColumnIndexerTest, Failure) { // indexer->Close(); // } - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); } TEST(VectorColumnIndexerTest, CosineMerge) { constexpr uint32_t kDimension = 64; const std::string index_name{"test_indexer.index"}; - auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = @@ -2592,9 +2572,8 @@ TEST(VectorColumnIndexerTest, Refiner) { // will be too large due to float's precision const uint32_t kTopk = 10; - auto del_index_file_func = [&](const std::string &file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = diff --git a/tests/db/index/storage/wal_file_test.cc b/tests/db/index/storage/wal_file_test.cc index 7687346e4..243e2a0eb 100644 --- a/tests/db/index/storage/wal_file_test.cc +++ b/tests/db/index/storage/wal_file_test.cc @@ -25,17 +25,8 @@ #include #include #include -#ifdef _MSC_VER -#include -inline int truncate(const char *path, long length) { - int fd = _open(path, _O_RDWR); - if (fd < 0) return -1; - int ret = _chsize(fd, length); - _close(fd); - return ret; -} -#endif #include +#include "tests/test_util.h" #include #include #include @@ -52,11 +43,7 @@ using SegmentID = uint32_t; class WalFileTest : public testing::Test { protected: void SetUp() { -#ifdef _WIN32 - system("del /f /q .\\data.wal.*"); -#else - system("rm -rf ./data.wal.*"); -#endif + zvec::test_util::RemoveTestFiles("./data.wal.*"); } void TearDown() {} From c6e5012241ebe79403da0e1ae2cbc0e40bb8f863 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 15:44:54 +0800 Subject: [PATCH 44/51] fix --- tests/test_util.h | 71 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/test_util.h diff --git a/tests/test_util.h b/tests/test_util.h new file mode 100644 index 000000000..e9b35eadd --- /dev/null +++ b/tests/test_util.h @@ -0,0 +1,71 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#ifdef _MSC_VER +#include +#include +#include +#include +#endif + +namespace zvec { +namespace test_util { + +inline void RemoveTestPath(const std::string &path) { + if (!ailego::FileHelper::RemovePath(path.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q \"" + path + "\" 2>NUL").c_str()); +#endif + } +} + +inline void RemoveTestFiles(const std::string &pattern) { + if (pattern.find('*') != std::string::npos || + pattern.find('?') != std::string::npos) { +#ifdef _WIN32 + system(("del /f /q " + pattern + " 2>NUL").c_str()); +#else + system(("rm -rf " + pattern).c_str()); +#endif + } else { + ailego::FileHelper::RemovePath(pattern.c_str()); + } +} + +} // namespace test_util +} // namespace zvec + +#ifdef _MSC_VER +inline void sleep(unsigned int seconds) { + std::this_thread::sleep_for(std::chrono::seconds(seconds)); +} + +inline void usleep(unsigned int microseconds) { + std::this_thread::sleep_for(std::chrono::microseconds(microseconds)); +} + +inline int truncate(const char *path, long length) { + int fd = _open(path, _O_RDWR); + if (fd < 0) return -1; + int ret = _chsize(fd, length); + _close(fd); + return ret; +} +#endif From a7e51900a8f35b29fc11fc348eec9d23505eb756 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 16:12:26 +0800 Subject: [PATCH 45/51] fix memory limit --- src/db/common/cgroup_util.cc | 27 ++++++++++++++++++++++++++- src/db/common/cgroup_util.h | 3 +++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/db/common/cgroup_util.cc b/src/db/common/cgroup_util.cc index 87896b04f..e358c2d20 100644 --- a/src/db/common/cgroup_util.cc +++ b/src/db/common/cgroup_util.cc @@ -75,6 +75,8 @@ uint64_t CgroupUtil::getUptime() { time_t csec = time(NULL); return csec - bsec; } +#elif defined(PLATFORM_WINDOWS) + return GetTickCount64() / 1000; #endif return 0; } @@ -97,6 +99,13 @@ void CgroupUtil::updateCpuCores() { if (cpu_cores_ <= 0) { cpu_cores_ = 1; } +#elif defined(PLATFORM_WINDOWS) + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + cpu_cores_ = static_cast(sysinfo.dwNumberOfProcessors); + if (cpu_cores_ <= 0) { + cpu_cores_ = 1; + } #endif } @@ -162,6 +171,14 @@ void CgroupUtil::updateMemoryLimit() { } else { memory_limit_ = 0; } +#elif defined(PLATFORM_WINDOWS) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + memory_limit_ = statex.ullTotalPhys; + } else { + memory_limit_ = 0; + } #endif } @@ -253,6 +270,13 @@ uint64_t CgroupUtil::getCurrentMemoryUsage() { return readMemoryUsageProc(); #elif defined(PLATFORM_MACOS) return getMacOSMemoryUsage(); +#elif defined(PLATFORM_WINDOWS) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + return statex.ullTotalPhys - statex.ullAvailPhys; + } + return 0; #else return 0; #endif @@ -371,8 +395,9 @@ double CgroupUtil::calculateCpuUsage() { return calculateLinuxCpuUsage(); #elif defined(PLATFORM_MACOS) return calculateMacOSCpuUsage(); -#endif +#else return 0.0; +#endif } #if defined(PLATFORM_LINUX) diff --git a/src/db/common/cgroup_util.h b/src/db/common/cgroup_util.h index 7df82aee0..759e2317c 100644 --- a/src/db/common/cgroup_util.h +++ b/src/db/common/cgroup_util.h @@ -32,6 +32,9 @@ #define PLATFORM_LINUX 1 #include #include +#elif defined(_WIN32) || defined(_WIN64) +#define PLATFORM_WINDOWS 1 +#include #endif namespace zvec { From e7bbbfef86b4cac10e726bf7a1af6eeb9e6f1838 Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 16:12:36 +0800 Subject: [PATCH 46/51] fix segment_helper_test --- src/db/index/segment/segment_helper.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/db/index/segment/segment_helper.cc b/src/db/index/segment/segment_helper.cc index 45b2dfb30..2c1038f2c 100644 --- a/src/db/index/segment/segment_helper.cc +++ b/src/db/index/segment/segment_helper.cc @@ -642,6 +642,9 @@ Status SegmentHelper::ReduceVectorIndex( s = vector_indexer->Flush(); CHECK_RETURN_STATUS(s); + s = vector_indexer->Close(); + CHECK_RETURN_STATUS(s); + BlockMeta new_block_meta; new_block_meta.set_id(vector_block_id); new_block_meta.set_type(BlockType::VECTOR_INDEX); @@ -687,6 +690,9 @@ Status SegmentHelper::ReduceVectorIndex( s = vector_indexer->Flush(); CHECK_RETURN_STATUS(s); + s = vector_indexer->Close(); + CHECK_RETURN_STATUS(s); + BlockMeta new_block_meta; new_block_meta.set_id(vector_block_id); new_block_meta.set_type(BlockType::VECTOR_INDEX); @@ -719,6 +725,9 @@ Status SegmentHelper::ReduceVectorIndex( s = vector_indexer_quantize->Flush(); CHECK_RETURN_STATUS(s); + s = vector_indexer_quantize->Close(); + CHECK_RETURN_STATUS(s); + new_block_meta.set_id(vector_quan_block_id); new_block_meta.set_type(BlockType::VECTOR_INDEX_QUANTIZE); new_block_meta.set_columns({field->name()}); From 5c89b6e1d1c682e9b487c9dfed2925ec0874d2ad Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 19:27:30 +0800 Subject: [PATCH 47/51] fix write_recovery_test --- tests/db/crash_recovery/data_generator.cc | 2 +- .../db/crash_recovery/write_recovery_test.cc | 186 ++++++++++++------ 2 files changed, 122 insertions(+), 66 deletions(-) diff --git a/tests/db/crash_recovery/data_generator.cc b/tests/db/crash_recovery/data_generator.cc index d0f5798fe..9baa548f9 100644 --- a/tests/db/crash_recovery/data_generator.cc +++ b/tests/db/crash_recovery/data_generator.cc @@ -13,7 +13,7 @@ // limitations under the License. -#include +#include #include #include #include diff --git a/tests/db/crash_recovery/write_recovery_test.cc b/tests/db/crash_recovery/write_recovery_test.cc index 1f53a5f41..e4eecb31e 100644 --- a/tests/db/crash_recovery/write_recovery_test.cc +++ b/tests/db/crash_recovery/write_recovery_test.cc @@ -20,8 +20,20 @@ #include #include #include +#include "tests/test_util.h" #include "utility.h" +#ifdef _WIN32 +#include +#include +typedef HANDLE pid_t; +#define SIGKILL 9 +// 模拟简单的 WIFEXITED 逻辑 +#define WIFEXITED(status) true +#define WEXITSTATUS(status) (status) +#define WIFSIGNALED(status) ((status) != 0) +#endif + namespace zvec { @@ -34,8 +46,29 @@ const zvec::CollectionOptions options_{false, true}; static std::string LocateDataGenerator() { namespace fs = std::filesystem; - const std::vector candidates{"./data_generator", - "./bin/data_generator"}; + std::cout << "Current path: " << fs::current_path() << std::endl; + + const std::string base_name = "data_generator"; + std::vector candidates; + + // Define potential search locations relative to the current working directory + const std::vector search_paths = {"./", "./bin/"}; +#ifdef _WIN32 + for (const auto &p : search_paths) { + candidates.push_back(p); + candidates.push_back(p + "Debug/"); + candidates.push_back(p + "Release/"); + } +#endif + + + for (auto &p : candidates) { + p += "/" + base_name; +#ifdef _WIN32 + p += ".exe"; // Append .exe suffix for Windows compatibility +#endif + } + for (const auto &p : candidates) { if (fs::exists(p)) { return fs::canonical(p).string(); @@ -45,93 +78,116 @@ static std::string LocateDataGenerator() { } -void RunGenerator(const std::string &start, const std::string &end, - const std::string &op, const std::string &version) { - pid_t pid = fork(); - ASSERT_GE(pid, 0); +/** + * Internal helper to execute the data generator process. + * If 'kill_after_seconds' is greater than 0, the process will be forcibly + * terminated after the specified duration to simulate a crash. + */ +void ExecuteProcess(const std::string &start, const std::string &end, + const std::string &op, const std::string &version, + int kill_after_seconds = -1) { + bool should_crash = kill_after_seconds >= 0; + +#ifdef _WIN32 + // 1. Build the command line string with quotes to handle paths with spaces + std::string cmd_str = data_generator_bin_ + " --path " + dir_path_ + + " --start " + start + " --end " + end + " --op " + op + + " --version " + version; + + STARTUPINFOA si = {sizeof(si)}; + PROCESS_INFORMATION pi; + + std::cout << cmd_str << std::endl; + + std::vector cmd_buf(cmd_str.begin(), cmd_str.end()); + cmd_buf.push_back('\0'); + if (!CreateProcessA(NULL, cmd_buf.data(), NULL, NULL, FALSE, 0, NULL, NULL, + &si, &pi)) { + FAIL() << "CreateProcess failed (" << GetLastError() << ")"; + } - if (pid == 0) { // Child process - char arg_path[] = "--path"; - char arg_start[] = "--start"; - char arg_end[] = "--end"; - char arg_op[] = "--op"; - char arg_version[] = "--version"; - char *args[] = {const_cast(data_generator_bin_.c_str()), - arg_path, - const_cast(dir_path_.c_str()), - arg_start, - const_cast(start.c_str()), - arg_end, - const_cast(end.c_str()), - arg_op, - const_cast(op.c_str()), - arg_version, - const_cast(version.c_str()), - nullptr}; - execvp(args[0], args); - perror("execvp failed"); - _exit(1); + if (should_crash) { + std::this_thread::sleep_for(std::chrono::seconds(kill_after_seconds)); + // Simulate a crash by killing the process + TerminateProcess(pi.hProcess, 1); } - int status; - waitpid(pid, &status, 0); - ASSERT_TRUE(WIFEXITED(status)) - << "Child process did not exit normally. Terminated by signal?"; - int exit_code = WEXITSTATUS(status); - ASSERT_EQ(exit_code, 0) << "data_generator failed with exit code: " - << exit_code; -} + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD exit_code; + GetExitCodeProcess(pi.hProcess, &exit_code); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); -void RunGeneratorAndCrash(const std::string &start, const std::string &end, - const std::string &op, const std::string &version, - int seconds) { + if (should_crash) { + // In crash tests, we expect the process to have been terminated + ASSERT_NE(exit_code, 0) << "Process was expected to crash/terminate."; + } else { + ASSERT_EQ(exit_code, 0) << "Process failed with exit code: " << exit_code; + } + +#else + // POSIX Implementation pid_t pid = fork(); ASSERT_GE(pid, 0); if (pid == 0) { // Child process - char arg_path[] = "--path"; - char arg_start[] = "--start"; - char arg_end[] = "--end"; - char arg_op[] = "--op"; - char arg_version[] = "--version"; - char *args[] = {const_cast(data_generator_bin_.c_str()), - arg_path, - const_cast(dir_path_.c_str()), - arg_start, - const_cast(start.c_str()), - arg_end, - const_cast(end.c_str()), - arg_op, - const_cast(op.c_str()), - arg_version, - const_cast(version.c_str()), - nullptr}; - execvp(args[0], args); + // Use a vector to manage arguments cleanly + std::vector args = {data_generator_bin_.c_str(), + "--path", + dir_path_.c_str(), + "--start", + start.c_str(), + "--end", + end.c_str(), + "--op", + op.c_str(), + "--version", + version.c_str(), + nullptr}; + execvp(args[0], const_cast(args.data())); perror("execvp failed"); _exit(1); } - std::this_thread::sleep_for(std::chrono::seconds(seconds)); - if (kill(pid, 0) == 0) { + int status; + if (should_crash) { + std::this_thread::sleep_for(std::chrono::seconds(kill_after_seconds)); kill(pid, SIGKILL); + waitpid(pid, &status, 0); + ASSERT_TRUE(WIFSIGNALED(status)) << "Process did not crash as expected."; + } else { + waitpid(pid, &status, 0); + ASSERT_TRUE(WIFEXITED(status)) << "Process exited abnormally."; + ASSERT_EQ(WEXITSTATUS(status), 0); } - int status; - waitpid(pid, &status, 0); - ASSERT_TRUE(WIFSIGNALED(status)) - << "Child process was not killed by a signal. It exited normally?"; +#endif +} + +// Public API 1: Normal execution +void RunGenerator(const std::string &start, const std::string &end, + const std::string &op, const std::string &version) { + ExecuteProcess(start, end, op, version); +} + +// Public API 2: Execution with simulated crash +void RunGeneratorAndCrash(const std::string &start, const std::string &end, + const std::string &op, const std::string &version, + int seconds) { + ExecuteProcess(start, end, op, version, seconds); } class CrashRecoveryTest : public ::testing::Test { protected: void SetUp() override { - system("rm -rf ./crash_test_db"); - ASSERT_NO_THROW(data_generator_bin_ = LocateDataGenerator()); + zvec::test_util::RemoveTestPath("./crash_test_db"); + // ASSERT_NO_THROW(data_generator_bin_ = LocateDataGenerator()); + data_generator_bin_ = LocateDataGenerator(); } void TearDown() override { - system("rm -rf ./crash_test_db"); + zvec::test_util::RemoveTestPath("./crash_test_db"); } }; From 62683bff83498ac0c3fa6bad3bd8324602f9490d Mon Sep 17 00:00:00 2001 From: JalinWang Date: Mon, 16 Mar 2026 19:28:31 +0800 Subject: [PATCH 48/51] fix __m64 --- src/ailego/math/distance_matrix_accum_fp16.i | 14 +++++++------- src/ailego/math/distance_matrix_fp16.i | 8 ++++---- .../math/inner_product_matrix_fp16_avx512.cc | 2 +- src/ailego/math/inner_product_matrix_fp32_sse.cc | 2 +- .../mips_euclidean_distance_matrix_fp32_sse.cc | 2 +- src/ailego/math/norm_matrix_fp16.i | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/ailego/math/distance_matrix_accum_fp16.i b/src/ailego/math/distance_matrix_accum_fp16.i index 43cc68d33..6f94b06b8 100644 --- a/src/ailego/math/distance_matrix_accum_fp16.i +++ b/src/ailego/math/distance_matrix_accum_fp16.i @@ -82,7 +82,7 @@ __m128 xmm_sum_0_0 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_0), \ _mm256_extractf128_ps(ymm_sum_0_0, 1)); \ if (qe >= qe_aligned + 2) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ __m128 xmm_q = _mm_cvtph_ps( \ _mm_shufflelo_epi16(_mm_broadcast_si32(q), _MM_SHUFFLE(1, 1, 0, 0))); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_q, xmm_sum_0_0) \ @@ -120,8 +120,8 @@ __m128 xmm_sum_0_1 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_1), \ _mm256_extractf128_ps(ymm_sum_0_1, 1)); \ if (qe >= qe_aligned + 4) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ - __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(q))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ + __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(q))); \ __m128 xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(2, 2, 0, 0)); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_p, xmm_sum_0_0) \ xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(3, 3, 1, 1)); \ @@ -164,7 +164,7 @@ __m128 xmm_sum_0_0 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_0), \ _mm256_extractf128_ps(ymm_sum_0_0, 1)); \ if (q != qe) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi16(*(const short *)(q))); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_q, xmm_sum_0_0) \ } \ @@ -198,7 +198,7 @@ if (q != qe) { \ __m128 xmm_q_0 = _mm_cvtph_ps(_mm_set1_epi16(*(const short *)(q + 0))); \ __m128 xmm_q_1 = _mm_cvtph_ps(_mm_set1_epi16(*(const short *)(q + 1))); \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ MATRIX_VAR_PROC(1, 2, 0, xmm_m, xmm_q, xmm_sum, ACCUM_FP32_STEP_SSE) \ } \ if (((uintptr_t)out & 0xf) == 0) { \ @@ -233,8 +233,8 @@ __m128 xmm_sum_0_3 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_3), \ _mm256_extractf128_ps(ymm_sum_0_3, 1)); \ if (q != qe) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ - __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(q))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ + __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(q))); \ __m128 xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(0, 0, 0, 0)); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_p, xmm_sum_0_0) \ xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(1, 1, 1, 1)); \ diff --git a/src/ailego/math/distance_matrix_fp16.i b/src/ailego/math/distance_matrix_fp16.i index 26ab75f08..11f05eb2f 100644 --- a/src/ailego/math/distance_matrix_fp16.i +++ b/src/ailego/math/distance_matrix_fp16.i @@ -67,9 +67,9 @@ } \ case 4: { \ __m256 ymm_lhs = _mm256_cvtph_ps( \ - _mm_set_epi64((__m64)(_MASK), *((const __m64 *)(lhs)))); \ + _mm_set_epi64x((long long)(_MASK), *(const long long *)(lhs))); \ __m256 ymm_rhs = _mm256_cvtph_ps( \ - _mm_set_epi64((__m64)(_MASK), *((const __m64 *)(rhs)))); \ + _mm_set_epi64x((long long)(_MASK), *(const long long *)(rhs))); \ _PROC(ymm_lhs, ymm_rhs, _RES##_0_0) \ break; \ } \ @@ -125,7 +125,7 @@ { \ __m256 ymm_m = _mm256_cvtph_ps(_LOAD((const __m128i *)(m))); \ __m256 ymm_q = _mm256_cvtph_ps(_mm_shufflehi_epi16( \ - _mm_shufflelo_epi16(_mm_set1_epi64(*(const __m64 *)(q)), \ + _mm_shufflelo_epi16(_mm_set1_epi64x(*(const long long *)(q)), \ _MM_SHUFFLE(1, 1, 0, 0)), \ _MM_SHUFFLE(3, 3, 2, 2))); \ _PROC(ymm_m, ymm_q, _RES##_0_0) \ @@ -155,7 +155,7 @@ //! Iterative process of computing distance (FP16, M=4, N=2) #define MATRIX_FP16_ITER_4X2_AVX(m, q, _RES, _LOAD, _PROC) \ { \ - __m128i xmm_qi = _mm_set1_epi64(*(const __m64 *)(q)); \ + __m128i xmm_qi = _mm_set1_epi64x(*(const long long *)(q)); \ __m256 ymm_m = _mm256_cvtph_ps(_LOAD((const __m128i *)(m))); \ __m256 ymm_q_0 = _mm256_cvtph_ps(_mm_shufflehi_epi16( \ _mm_shufflelo_epi16(xmm_qi, _MM_SHUFFLE(0, 0, 0, 0)), \ diff --git a/src/ailego/math/inner_product_matrix_fp16_avx512.cc b/src/ailego/math/inner_product_matrix_fp16_avx512.cc index 7e07952ee..0aa86af6a 100644 --- a/src/ailego/math/inner_product_matrix_fp16_avx512.cc +++ b/src/ailego/math/inner_product_matrix_fp16_avx512.cc @@ -735,7 +735,7 @@ float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count, _mm_loadu_ph(val_start_2 + k))); } - Float16 __attribute__((aligned(16))) tmp_res[8]; + alignas(16) Float16 tmp_res[8]; _mm_store_ph(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3] + tmp_res[4] + tmp_res[5] + tmp_res[6] + tmp_res[7]); diff --git a/src/ailego/math/inner_product_matrix_fp32_sse.cc b/src/ailego/math/inner_product_matrix_fp32_sse.cc index f90801eec..accd70ece 100644 --- a/src/ailego/math/inner_product_matrix_fp32_sse.cc +++ b/src/ailego/math/inner_product_matrix_fp32_sse.cc @@ -298,7 +298,7 @@ float InnerProductSparseInSegmentSSE(uint32_t m_sparse_count, _mm_loadu_ps(val_start_2 + k))); } - float __attribute__((aligned(16))) tmp_res[4]; + alignas(16) float tmp_res[4]; _mm_store_ps(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); } diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc index 357703db6..ff9f29ebc 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc @@ -328,7 +328,7 @@ float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count, _mm_loadu_ps(val_start_2 + k))); } - float __attribute__((aligned(16))) tmp_res[4]; + alignas(16) float tmp_res[4]; _mm_store_ps(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); } diff --git a/src/ailego/math/norm_matrix_fp16.i b/src/ailego/math/norm_matrix_fp16.i index fec074211..c1d3bd219 100644 --- a/src/ailego/math/norm_matrix_fp16.i +++ b/src/ailego/math/norm_matrix_fp16.i @@ -49,7 +49,7 @@ } \ case 4: { \ __m256 ymm_m = _mm256_cvtph_ps( \ - _mm_set_epi64((__m64)(0ull), *((const __m64 *)(m)))); \ + _mm_set_epi64x(0LL, *(const long long *)(m))); \ NORM_FP32_STEP_AVX(ymm_m, _RES##_0_0) \ break; \ } \ From cda7187ef8cd866e5c563ff8b2027bf5d4321c41 Mon Sep 17 00:00:00 2001 From: Jianning Wang Date: Mon, 16 Mar 2026 16:41:21 +0800 Subject: [PATCH 49/51] fix linux ut --- tests/core/algorithm/ivf/ivf_searcher_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/algorithm/ivf/ivf_searcher_test.cc b/tests/core/algorithm/ivf/ivf_searcher_test.cc index 9911e0e2e..282bde9d2 100644 --- a/tests/core/algorithm/ivf/ivf_searcher_test.cc +++ b/tests/core/algorithm/ivf/ivf_searcher_test.cc @@ -392,7 +392,7 @@ TEST_F(IVFSearcherTest, TestSimpleCosine) { { size_t topk = 33; context->set_topk(topk); - + std::string new_vec; IndexQueryMeta new_meta; ASSERT_EQ(0, reformer->convert(query.data(), qmeta, &new_vec, &new_meta)); @@ -3001,7 +3001,7 @@ TEST_F(IVFSearcherTest, TestProvider) { } else { std::iota(keys.begin(), keys.end(), 0U); std::transform(keys.begin(), keys.end(), keys.begin(), - [&](key_t k) { return step * k; }); + [&](uint64_t k) { return step * k; }); if (rand_order) { uint32_t seed = Realtime::Seconds(); std::shuffle(keys.begin(), keys.end(), std::default_random_engine(seed)); @@ -3070,7 +3070,7 @@ TEST_F(IVFSearcherTest, TestProviderInt8) { auto holder = make_shared>( dimension_); size_t doc_cnt = 5000UL; - std::vector keys(doc_cnt); + std::vector keys(doc_cnt); srand(Realtime::MilliSeconds()); bool rand_key = rand() % 2; bool rand_order = rand() % 2; @@ -3086,7 +3086,7 @@ TEST_F(IVFSearcherTest, TestProviderInt8) { } else { std::iota(keys.begin(), keys.end(), 0U); std::transform(keys.begin(), keys.end(), keys.begin(), - [&](key_t k) { return step * k; }); + [&](uint64_t k) { return step * k; }); if (rand_order) { uint32_t seed = Realtime::Seconds(); std::shuffle(keys.begin(), keys.end(), std::default_random_engine(seed)); From 26085af2e1749e6c10ca687d18da371d76687e62 Mon Sep 17 00:00:00 2001 From: Jianning Wang Date: Tue, 17 Mar 2026 10:28:47 +0800 Subject: [PATCH 50/51] fix write_recovery_test on linux --- tests/db/crash_recovery/write_recovery_test.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/db/crash_recovery/write_recovery_test.cc b/tests/db/crash_recovery/write_recovery_test.cc index e4eecb31e..d8b564c42 100644 --- a/tests/db/crash_recovery/write_recovery_test.cc +++ b/tests/db/crash_recovery/write_recovery_test.cc @@ -50,12 +50,14 @@ static std::string LocateDataGenerator() { const std::string base_name = "data_generator"; std::vector candidates; - // Define potential search locations relative to the current working directory const std::vector search_paths = {"./", "./bin/"}; -#ifdef _WIN32 + for (const auto &p : search_paths) { candidates.push_back(p); + } +#ifdef _WIN32 + for (const auto &p : search_paths) { candidates.push_back(p + "Debug/"); candidates.push_back(p + "Release/"); } @@ -63,7 +65,7 @@ static std::string LocateDataGenerator() { for (auto &p : candidates) { - p += "/" + base_name; + p += base_name; #ifdef _WIN32 p += ".exe"; // Append .exe suffix for Windows compatibility #endif From 6795a6c81bea3d06a0b4d23a1bd2624372a53389 Mon Sep 17 00:00:00 2001 From: Jianning Wang Date: Tue, 17 Mar 2026 11:26:41 +0800 Subject: [PATCH 51/51] format --- src/ailego/io/file.cc | 21 +++++++++---------- .../algorithm/hnsw/hnsw_streamer_entity.cc | 3 ++- src/include/zvec/ailego/internal/platform.h | 2 +- src/include/zvec/ailego/logger/logger.h | 6 ++++-- .../zvec/core/framework/index_logger.h | 3 ++- .../ailego/parallel/multi_thread_list_test.cc | 2 +- tests/ailego/utility/string_helper_test.cc | 2 +- .../cluster/opt_kmeans_cluster_test.cc | 2 +- .../flat_sparse/flat_sparse_searcher_test.cc | 2 +- .../flat_sparse/flat_sparse_streamer_test.cc | 2 +- .../core/algorithm/hnsw/hnsw_builder_test.cc | 2 +- .../algorithm/hnsw/hnsw_searcher_test.cpp | 2 +- .../core/algorithm/hnsw/hnsw_streamer_test.cc | 5 ++--- .../hnsw_sparse/hnsw_sparse_builder_test.cc | 2 +- .../hnsw_sparse/hnsw_sparse_searcher_test.cpp | 2 +- .../hnsw_sparse_streamer_buffer_test.cpp | 5 +++-- .../hnsw_sparse/hnsw_sparse_streamer_test.cc | 2 +- tests/core/interface/index_interface_test.cc | 2 +- .../metric/quantized_integer_metric_test.cc | 4 ++-- tests/db/crash_recovery/data_generator.cc | 2 +- .../column/vector_column_indexer_test.cc | 17 +++++++-------- tests/db/index/segment/segment_test.cc | 6 ++++-- tests/db/index/storage/mem_store_test.cc | 13 ++++++------ tests/db/index/storage/wal_file_test.cc | 2 +- tests/test_util.h | 2 +- 25 files changed, 58 insertions(+), 55 deletions(-) diff --git a/src/ailego/io/file.cc b/src/ailego/io/file.cc index cae474385..1a458cf3d 100644 --- a/src/ailego/io/file.cc +++ b/src/ailego/io/file.cc @@ -639,14 +639,14 @@ ssize_t File::offset(void) const { } void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { -// Root cause: Windows MapViewOfFile requires the file offset to be aligned to -// the allocation granularity (64 KB), but segment offsets were only -// page-aligned (4 KB). Also, CreateFileMapping was using len instead of -// off + len as the max size. -// -// Fix: Align the view offset down to allocation granularity, adjust the map -// length, and return base + excess. MemoryUnmap recovers the base by rounding -// down to granularity. + // Root cause: Windows MapViewOfFile requires the file offset to be aligned to + // the allocation granularity (64 KB), but segment offsets were only + // page-aligned (4 KB). Also, CreateFileMapping was using len instead of + // off + len as the max size. + // + // Fix: Align the view offset down to allocation granularity, adjust the map + // length, and return base + excess. MemoryUnmap recovers the base by rounding + // down to granularity. SYSTEM_INFO si; GetSystemInfo(&si); @@ -675,9 +675,8 @@ void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { view_offset.QuadPart = aligned_off; size_t view_len = len + excess; - void *base = MapViewOfFile(file_mapping, desired_access, - view_offset.HighPart, view_offset.LowPart, - view_len); + void *base = MapViewOfFile(file_mapping, desired_access, view_offset.HighPart, + view_offset.LowPart, view_len); CloseHandle(file_mapping); ailego_null_if_false(base); diff --git a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc index 9c85a8bbc..90a2a373e 100644 --- a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc @@ -485,7 +485,8 @@ int HnswStreamerEntity::check_hnsw_index(const HNSWHeader *hd) const { int HnswStreamerEntity::add_vector(level_t level, key_t key, const void *vec, node_id_t *id) { Chunk::Pointer node_chunk; - // On MSVC, unsigned long is 32-bit, so -1UL is 0xFFFFFFFF not 0xFFFFFFFFFFFFFFFF. + // On MSVC, unsigned long is 32-bit, so -1UL is 0xFFFFFFFF not + // 0xFFFFFFFFFFFFFFFF. size_t chunk_offset = static_cast(-1); std::lock_guard lock(mutex_); diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 1a521141f..ccd33971e 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -48,7 +48,7 @@ #if defined(_MSC_VER) #if !defined(S_ISDIR) -#define S_ISDIR(mode) (((mode)&S_IFMT) == S_IFDIR) +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) #endif #if !defined(_SSIZE_T_DEFINED) diff --git a/src/include/zvec/ailego/logger/logger.h b/src/include/zvec/ailego/logger/logger.h index 08aeae1af..a7c90a9be 100644 --- a/src/include/zvec/ailego/logger/logger.h +++ b/src/include/zvec/ailego/logger/logger.h @@ -21,7 +21,8 @@ // Define printf format attribute for GCC/Clang, empty for MSVC #if defined(__GNUC__) || defined(__clang__) -#define AILEGO_PRINTF_FORMAT(fmt_idx, arg_idx) __attribute__((format(printf, fmt_idx, arg_idx))) +#define AILEGO_PRINTF_FORMAT(fmt_idx, arg_idx) \ + __attribute__((format(printf, fmt_idx, arg_idx))) #else #define AILEGO_PRINTF_FORMAT(fmt_idx, arg_idx) #endif @@ -158,7 +159,8 @@ class LoggerBroker { //! Log Message AILEGO_PRINTF_FORMAT(4, 5) - static void Log(int level, const char *file, int line, const char *format, ...) { + static void Log(int level, const char *file, int line, const char *format, + ...) { if (IsLevelEnabled(level)) { va_list args; va_start(args, format); diff --git a/src/include/zvec/core/framework/index_logger.h b/src/include/zvec/core/framework/index_logger.h index 3a6144e9c..842c6e98f 100644 --- a/src/include/zvec/core/framework/index_logger.h +++ b/src/include/zvec/core/framework/index_logger.h @@ -144,7 +144,8 @@ class IndexLoggerBroker { //! Log Message AILEGO_PRINTF_FORMAT(4, 5) - static void Log(int level, const char *file, int line, const char *format, ...) { + static void Log(int level, const char *file, int line, const char *format, + ...) { if (IsLevelEnabled(level)) { va_list args; va_start(args, format); diff --git a/tests/ailego/parallel/multi_thread_list_test.cc b/tests/ailego/parallel/multi_thread_list_test.cc index b772bec05..17a896f1b 100644 --- a/tests/ailego/parallel/multi_thread_list_test.cc +++ b/tests/ailego/parallel/multi_thread_list_test.cc @@ -13,8 +13,8 @@ // limitations under the License. #include -#include #include +#include #include #define private public diff --git a/tests/ailego/utility/string_helper_test.cc b/tests/ailego/utility/string_helper_test.cc index dde0a8a74..705cd352c 100644 --- a/tests/ailego/utility/string_helper_test.cc +++ b/tests/ailego/utility/string_helper_test.cc @@ -13,9 +13,9 @@ // limitations under the License. #include -#include #include #include +#include #include using namespace zvec; diff --git a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc index 03b117e8c..9e0461960 100644 --- a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc +++ b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc @@ -15,9 +15,9 @@ #include #include #include -#include #include #include +#include #include "zvec/core/framework/index_framework.h" using namespace zvec::core; diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc index 6f95fa621..458de1c7c 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc @@ -20,9 +20,9 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" -#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc index ffa383749..befd9aea0 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc @@ -21,10 +21,10 @@ #include #include #include -#include "tests/test_util.h" #include #include #include +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; diff --git a/tests/core/algorithm/hnsw/hnsw_builder_test.cc b/tests/core/algorithm/hnsw/hnsw_builder_test.cc index 3d65c9af8..fd151227f 100644 --- a/tests/core/algorithm/hnsw/hnsw_builder_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_builder_test.cc @@ -18,8 +18,8 @@ #include #include #include -#include "zvec/core/framework/index_framework.h" #include "tests/test_util.h" +#include "zvec/core/framework/index_framework.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push diff --git a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp index 5b6fce16a..cd1ab3970 100644 --- a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp @@ -20,11 +20,11 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_builder.h" #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" #include "hnsw_params.h" -#include "tests/test_util.h" using namespace std; using namespace testing; diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc index ee25ba44a..9a3a09d8e 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "hnsw_streamer.h" -#include #include #include +#include #ifndef _MSC_VER #include #include @@ -3408,8 +3408,7 @@ TEST_F(HnswStreamerTest, TestGroupInBruteforceSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, - storage->open(dir_ + "TestGroupInBruteforceSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroupInBruteforceSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc index ae0d73c83..0e35dacd2 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc @@ -18,9 +18,9 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_framework.h" #include "hnsw_sparse_params.h" -#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp index cba21a8f0..8196e736e 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp @@ -20,8 +20,8 @@ #include #include #include -#include "zvec/core/framework/index_framework.h" #include "tests/test_util.h" +#include "zvec/core/framework/index_framework.h" #include "hnsw_sparse_params.h" #if defined(__GNUC__) || defined(__GNUG__) diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp index 70405d0fd..32dfb55b2 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp @@ -20,8 +20,8 @@ #include #include #include -#include "hnsw_sparse_streamer.h" #include "tests/test_util.h" +#include "hnsw_sparse_streamer.h" using namespace std; using namespace testing; @@ -48,7 +48,8 @@ class HnswSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswSparseStreamerTest::dir_("hnsw_sparse_streamer_buffer_test_dir/"); +std::string HnswSparseStreamerTest::dir_( + "hnsw_sparse_streamer_buffer_test_dir/"); shared_ptr HnswSparseStreamerTest::index_meta_ptr_; void HnswSparseStreamerTest::generate_sparse_data( diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc index 1cec7c4f2..623ac708a 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "hnsw_sparse_streamer.h" -#include #include #include +#include #ifndef _MSC_VER #include #include diff --git a/tests/core/interface/index_interface_test.cc b/tests/core/interface/index_interface_test.cc index b97405488..419798b97 100644 --- a/tests/core/interface/index_interface_test.cc +++ b/tests/core/interface/index_interface_test.cc @@ -17,12 +17,12 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/ailego/buffer/buffer_manager.h" #include "zvec/core/interface/index.h" #include "zvec/core/interface/index_factory.h" #include "zvec/core/interface/index_param.h" #include "zvec/core/interface/index_param_builders.h" -#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push diff --git a/tests/core/metric/quantized_integer_metric_test.cc b/tests/core/metric/quantized_integer_metric_test.cc index 501d8c7b9..835a07fb7 100644 --- a/tests/core/metric/quantized_integer_metric_test.cc +++ b/tests/core/metric/quantized_integer_metric_test.cc @@ -251,7 +251,7 @@ void TestDistanceMatrixInt8(const std::string &metric_name) { const size_t batch_size = M; const size_t query_size = N; - size_t dimension = (std::uniform_int_distribution(1, 65))(gen)*4; + size_t dimension = (std::uniform_int_distribution(1, 65))(gen) * 4; auto holder = GetHolder(dimension, batch_size, dist); IndexMeta meta(IndexMeta::DT_FP32, dimension); meta.set_metric(metric_name, 0, Params()); @@ -453,7 +453,7 @@ void TestDistanceMatrixInt4(const std::string &metric_name) { const size_t batch_size = M; const size_t query_size = N; - size_t dimension = (std::uniform_int_distribution(1, 65))(gen)*8; + size_t dimension = (std::uniform_int_distribution(1, 65))(gen) * 8; auto holder = GetHolder(dimension, batch_size, dist); IndexMeta meta(IndexMeta::DT_FP32, dimension); meta.set_metric(metric_name, 0, Params()); diff --git a/tests/db/crash_recovery/data_generator.cc b/tests/db/crash_recovery/data_generator.cc index 9baa548f9..9fd2f308e 100644 --- a/tests/db/crash_recovery/data_generator.cc +++ b/tests/db/crash_recovery/data_generator.cc @@ -13,9 +13,9 @@ // limitations under the License. -#include #include #include +#include #include #include "zvec/ailego/logger/logger.h" #include "utility.h" diff --git a/tests/db/index/column/vector_column_indexer_test.cc b/tests/db/index/column/vector_column_indexer_test.cc index 208a41077..5c91cdfb9 100644 --- a/tests/db/index/column/vector_column_indexer_test.cc +++ b/tests/db/index/column/vector_column_indexer_test.cc @@ -18,10 +18,10 @@ #include #include #include "db/index/column/vector_column/vector_column_params.h" +#include "tests/test_util.h" #include "zvec/ailego/utility/float_helper.h" #include "zvec/db/doc.h" #include "zvec/db/index_params.h" -#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -1297,9 +1297,8 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{1}; auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; - auto query = - vector_column_params::VectorData{vector_column_params::DenseVector{ - query_vec.data()}}; + auto query = vector_column_params::VectorData{ + vector_column_params::DenseVector{query_vec.data()}}; vector_column_params::QueryParams query_params; query_params.topk = 10; query_params.filter = nullptr; @@ -1327,9 +1326,8 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{1, 2}; auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; - auto query = - vector_column_params::VectorData{vector_column_params::DenseVector{ - query_vec.data()}}; + auto query = vector_column_params::VectorData{ + vector_column_params::DenseVector{query_vec.data()}}; vector_column_params::QueryParams query_params; query_params.topk = 10; query_params.filter = nullptr; @@ -1357,9 +1355,8 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{2}; auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; - auto query = - vector_column_params::VectorData{vector_column_params::DenseVector{ - query_vec.data()}}; + auto query = vector_column_params::VectorData{ + vector_column_params::DenseVector{query_vec.data()}}; vector_column_params::QueryParams query_params; query_params.topk = 10; query_params.filter = nullptr; diff --git a/tests/db/index/segment/segment_test.cc b/tests/db/index/segment/segment_test.cc index c15502adc..9530b8cf1 100644 --- a/tests/db/index/segment/segment_test.cc +++ b/tests/db/index/segment/segment_test.cc @@ -1150,7 +1150,8 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexerWithQuantVectorIndex) { query_params.topk = 10; query_params.filter = nullptr; query_params.fetch_vector = false; - query_params.query_params = std::make_shared(IndexType::HNSW); + query_params.query_params = + std::make_shared(IndexType::HNSW); query_params.query_params->set_is_using_refiner(true); auto results = combined_indexer->Search(query, query_params); @@ -1202,7 +1203,8 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexerQueryWithPks) { query_params.topk = 10; query_params.filter = nullptr; query_params.fetch_vector = false; - query_params.query_params = std::make_shared(IndexType::HNSW); + query_params.query_params = + std::make_shared(IndexType::HNSW); query_params.bf_pks = bf_pks; auto results = combined_indexer->Search(query, query_params); diff --git a/tests/db/index/storage/mem_store_test.cc b/tests/db/index/storage/mem_store_test.cc index 3b98440a5..d061b2aea 100644 --- a/tests/db/index/storage/mem_store_test.cc +++ b/tests/db/index/storage/mem_store_test.cc @@ -807,12 +807,13 @@ TEST_F(MemStoreTest, ThreadSafety) { std::vector> futures; for (int t = 0; t < num_threads; ++t) { - futures.push_back(std::async(std::launch::async, [this, t, inserts_per_thread]() { - for (int i = 0; i < inserts_per_thread; ++i) { - uint64_t doc_id = t * inserts_per_thread + i; - store_->insert(CreateDoc(doc_id)); - } - })); + futures.push_back( + std::async(std::launch::async, [this, t, inserts_per_thread]() { + for (int i = 0; i < inserts_per_thread; ++i) { + uint64_t doc_id = t * inserts_per_thread + i; + store_->insert(CreateDoc(doc_id)); + } + })); } // Wait for all threads to complete diff --git a/tests/db/index/storage/wal_file_test.cc b/tests/db/index/storage/wal_file_test.cc index 243e2a0eb..eabb91720 100644 --- a/tests/db/index/storage/wal_file_test.cc +++ b/tests/db/index/storage/wal_file_test.cc @@ -26,11 +26,11 @@ #include #include #include -#include "tests/test_util.h" #include #include #include #include "db/common/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push diff --git a/tests/test_util.h b/tests/test_util.h index e9b35eadd..00fedd81b 100644 --- a/tests/test_util.h +++ b/tests/test_util.h @@ -19,9 +19,9 @@ #include #ifdef _MSC_VER -#include #include #include +#include #include #endif