diff --git a/CMakeLists.txt b/CMakeLists.txt index 52a597548..3f25d14cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,8 +3,36 @@ cmake_policy(SET CMP0077 NEW) project(zvec) set(CC_CXX_STANDARD 17) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type") +if(MSVC) + set(INTTYPES_FORMAT VC7) + add_compile_options(/FS) #handle .pdb + add_compile_options(/EHsc) # def c++ exception behavior + add_compile_options(/Zc:preprocessor) + add_compile_options(/we4716) # -Werror=return-type + + + ###### reduce output length to make vibe coding work better :), which should be removed or solved later + add_compile_options(/wd4267 /wd4244) + add_compile_options(/wd4146) # unary minus operator applied to unsigned type #usage: uint32_t seg_id_{-1U}; + add_compile_options(/wd4310) # warning C4310: cast truncates constant value + add_compile_options(/wd4530) # warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc + add_compile_options(/wd4251) #protobuf + add_compile_options(/wd4245 /wd4334 /wd4702 /wd4305 /wd4099) + add_compile_options(/wd4200) # flexible structure + + # //warning C4334: '<<': result of 32-bit shift implicitly converted to 64 bits (was 64-bit shift intended?) + # keeps |= (1 << k); + add_compile_options(/wd4334) + ###### + + #TODO(windows): to be verified + add_definitions(-DARROW_STATIC -DPARQUET_STATIC -DARROW_ACERO_STATIC -DARROW_DS_STATIC -DARROW_COMPUTE_STATIC) + + +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type") +endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-as-needed") @@ -21,6 +49,7 @@ include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) include_directories(${PROJECT_ROOT_DIR}/src/include) include_directories(${PROJECT_ROOT_DIR}/src) +include_directories(${PROJECT_ROOT_DIR}) option(BUILD_PYTHON_BINDINGS "Build Python bindings using pybind11" OFF) message(STATUS "BUILD_PYTHON_BINDINGS:${BUILD_PYTHON_BINDINGS}") diff --git a/cmake/bazel.cmake b/cmake/bazel.cmake index 2e4f1ccf7..a531c3b08 100644 --- a/cmake/bazel.cmake +++ b/cmake/bazel.cmake @@ -621,6 +621,11 @@ function(_target_link_libraries _NAME) get_target_property(ALWAYS_LINK ${LIB} ALWAYS_LINK) if(ALWAYS_LINK) list(APPEND LOCAL_RESULT ${LIB}) + elseif(MSVC AND TARGET ${LIB}_static) + get_target_property(_SIBLING_AL ${LIB}_static ALWAYS_LINK) + if(_SIBLING_AL) + list(APPEND LOCAL_RESULT ${LIB}_static) + endif() endif() get_target_property(DEP_LIBS ${LIB} INTERFACE_LINK_LIBRARIES) @@ -652,6 +657,30 @@ function(_target_link_libraries _NAME) list(REMOVE_DUPLICATES ALL_LIBS_TO_PROCESS) + # On MSVC, each DLL has its own copy of template statics (e.g. Factory + # singletons), so registrations inside a DLL are invisible to the exe. + # Substitute SHARED libs with their ALWAYS_LINK _static counterparts and + # use /WHOLEARCHIVE so all registration code lives in the same module. + if(MSVC) + set(_SUBSTITUTED_LIBS "") + foreach(LIB ${ALL_LIBS_TO_PROCESS}) + if(TARGET ${LIB} AND TARGET ${LIB}_static) + get_target_property(_LIB_TYPE ${LIB} TYPE) + get_target_property(_STATIC_AL ${LIB}_static ALWAYS_LINK) + if("${_LIB_TYPE}" STREQUAL "SHARED_LIBRARY" AND _STATIC_AL) + list(APPEND _SUBSTITUTED_LIBS ${LIB}_static) + list(APPEND ALL_ALWAYS_LINK_LIBS ${LIB}_static) + continue() + endif() + endif() + list(APPEND _SUBSTITUTED_LIBS ${LIB}) + endforeach() + set(ALL_LIBS_TO_PROCESS ${_SUBSTITUTED_LIBS}) + if(ALL_ALWAYS_LINK_LIBS) + list(REMOVE_DUPLICATES ALL_ALWAYS_LINK_LIBS) + endif() + endif() + foreach(LIB ${ALL_LIBS_TO_PROCESS}) if(NOT TARGET ${LIB}) list(APPEND LINK_LIBS ${LIB}) @@ -672,7 +701,7 @@ function(_target_link_libraries _NAME) endif() else() # Microsoft Visual C++ - list(APPEND LINK_LIBS /WHOLEARCHIVE:$) + list(APPEND MSVC_WHOLEARCHIVE_OPTS /WHOLEARCHIVE:$) get_target_property(OTHER_LINK_LIBS ${LIB} INTERFACE_LINK_LIBRARIES) if(OTHER_LINK_LIBS) foreach(OTHER_LIB ${OTHER_LINK_LIBS}) @@ -691,6 +720,9 @@ function(_target_link_libraries _NAME) endforeach() target_link_libraries(${_NAME} ${LINK_LIBS}) + if(MSVC_WHOLEARCHIVE_OPTS) + target_link_options(${_NAME} PRIVATE ${MSVC_WHOLEARCHIVE_OPTS}) + endif() if(LIBS_DEPS) add_dependencies(${_NAME} ${LIBS_DEPS}) target_include_directories(${_NAME} PRIVATE "${LIBS_INCS}") diff --git a/cmake/utils.cmake b/cmake/utils.cmake index faccff08b..cda8a055a 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -12,7 +12,7 @@ function(apply_patch_once patch_name target_dir patch_file) #message(STATUS "Applying patch '${patch_name}' to ${target_dir} ...") execute_process( - COMMAND patch -p1 -i "${patch_file}" + COMMAND git apply --ignore-space-change --ignore-whitespace "${patch_file}" WORKING_DIRECTORY "${target_dir}" RESULT_VARIABLE patch_result OUTPUT_VARIABLE patch_stdout diff --git a/src/ailego/buffer/buffer_manager.cc b/src/ailego/buffer/buffer_manager.cc index ac2945b08..be3b6755d 100644 --- a/src/ailego/buffer/buffer_manager.cc +++ b/src/ailego/buffer/buffer_manager.cc @@ -103,7 +103,7 @@ struct BufferManager::BufferContext { ~BufferContext() { if (vector) { - free(vector); + ailego_aligned_free(vector); } } @@ -256,15 +256,15 @@ bool BufferManager::BufferContext::read_vector() { } AILEGO_DEFER([this] { file.close(); }); uint32_t len = id.vector().length; - auto ret = posix_memalign((void **)&vector, 64, len); // 64-byte alignment - if (ret != 0 || vector == nullptr) { + vector = (uint8_t *)ailego_aligned_malloc(len, 64); // 64-byte alignment + if (vector == nullptr) { LOG_ERROR("Failed to allocate buffer for file[%s]", file_name.c_str()); return false; } uint32_t offset = id.vector().offset; if (file.read(offset, vector, len) != len) { LOG_ERROR("Failed to read file[%s]", file_name.c_str()); - free(vector); + ailego_aligned_free(vector); vector = nullptr; return false; } @@ -390,7 +390,7 @@ class BufferManager::BufferPool { if (victim->id.type == BufferID::TYPE::PARQUET) { victim->arrow_refs.clear(); } else { - free(victim->vector); + ailego_aligned_free(victim->vector); victim->vector = nullptr; } victim->state = BufferContext::State::IDLE; @@ -585,10 +585,15 @@ uint64_t BufferManager::total_size_in_bytes() const { } -BufferManager::~BufferManager() { +void BufferManager::cleanup() { for (auto pool : pools_) { delete pool; } + pools_.clear(); +} + +BufferManager::~BufferManager() { + cleanup(); } diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 4bf139ae0..06e1444d5 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -15,7 +15,9 @@ #include "cpu_features.h" #include -#if !defined(_MSC_VER) && !defined(__ARM_ARCH) +#if defined(_MSC_VER) +#include +#elif !defined(__ARM_ARCH) #include #endif diff --git a/src/ailego/io/file.cc b/src/ailego/io/file.cc index 79e58a1c1..1a458cf3d 100644 --- a/src/ailego/io/file.cc +++ b/src/ailego/io/file.cc @@ -639,14 +639,28 @@ ssize_t File::offset(void) const { } void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { - LARGE_INTEGER file_size; - file_size.QuadPart = len; + // Root cause: Windows MapViewOfFile requires the file offset to be aligned to + // the allocation granularity (64 KB), but segment offsets were only + // page-aligned (4 KB). Also, CreateFileMapping was using len instead of + // off + len as the max size. + // + // Fix: Align the view offset down to allocation granularity, adjust the map + // length, and return base + excess. MemoryUnmap recovers the base by rounding + // down to granularity. + + SYSTEM_INFO si; + GetSystemInfo(&si); + DWORD granularity = si.dwAllocationGranularity; + ssize_t aligned_off = (off / (ssize_t)granularity) * (ssize_t)granularity; + size_t excess = (size_t)(off - aligned_off); + + LARGE_INTEGER max_size; + max_size.QuadPart = off + len; - // Create map object HANDLE file_mapping = CreateFileMapping( handle, nullptr, ((opts & File::MMAP_READONLY) ? PAGE_READONLY : PAGE_READWRITE), - file_size.HighPart, file_size.LowPart, nullptr); + max_size.HighPart, max_size.LowPart, nullptr); ailego_null_if_false(file_mapping != nullptr); DWORD desired_access = FILE_MAP_READ; @@ -656,14 +670,17 @@ void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { if (!(opts & File::MMAP_SHARED)) { desired_access |= FILE_MAP_COPY; } - file_size.QuadPart = off; - // Map the whole file to memory and close handle - void *addr = MapViewOfFile(file_mapping, desired_access, file_size.HighPart, - file_size.LowPart, 0); + LARGE_INTEGER view_offset; + view_offset.QuadPart = aligned_off; + size_t view_len = len + excess; + + void *base = MapViewOfFile(file_mapping, desired_access, view_offset.HighPart, + view_offset.LowPart, view_len); CloseHandle(file_mapping); - ailego_null_if_false(addr); + ailego_null_if_false(base); + void *addr = (char *)base + excess; if (opts & File::MMAP_LOCKED) { VirtualLock(addr, len); } @@ -673,8 +690,17 @@ void *File::MemoryMap(NativeHandle handle, ssize_t off, size_t len, int opts) { return addr; } -void *File::MemoryMap(size_t, int) { - return nullptr; +void *File::MemoryMap(size_t len, int opts) { + void *addr = + VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + ailego_null_if_false(addr); + if (opts & File::MMAP_LOCKED) { + VirtualLock(addr, len); + } + if (opts & File::MMAP_WARMUP) { + File::MemoryWarmup(addr, len); + } + return addr; } void *File::MemoryRemap(void *, size_t, void *, size_t) { @@ -683,7 +709,14 @@ void *File::MemoryRemap(void *, size_t, void *, size_t) { void File::MemoryUnmap(void *addr, size_t /*len*/) { ailego_return_if_false(addr); - UnmapViewOfFile(addr); + MEMORY_BASIC_INFORMATION mbi; + if (VirtualQuery(addr, &mbi, sizeof(mbi))) { + if (mbi.Type == MEM_MAPPED) { + UnmapViewOfFile(mbi.AllocationBase); + } else { + VirtualFree(mbi.AllocationBase, 0, MEM_RELEASE); + } + } } bool File::MemoryFlush(void *addr, size_t /*len*/) { diff --git a/src/ailego/math/distance_matrix_accum_fp16.i b/src/ailego/math/distance_matrix_accum_fp16.i index 43cc68d33..6f94b06b8 100644 --- a/src/ailego/math/distance_matrix_accum_fp16.i +++ b/src/ailego/math/distance_matrix_accum_fp16.i @@ -82,7 +82,7 @@ __m128 xmm_sum_0_0 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_0), \ _mm256_extractf128_ps(ymm_sum_0_0, 1)); \ if (qe >= qe_aligned + 2) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ __m128 xmm_q = _mm_cvtph_ps( \ _mm_shufflelo_epi16(_mm_broadcast_si32(q), _MM_SHUFFLE(1, 1, 0, 0))); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_q, xmm_sum_0_0) \ @@ -120,8 +120,8 @@ __m128 xmm_sum_0_1 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_1), \ _mm256_extractf128_ps(ymm_sum_0_1, 1)); \ if (qe >= qe_aligned + 4) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ - __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(q))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ + __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(q))); \ __m128 xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(2, 2, 0, 0)); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_p, xmm_sum_0_0) \ xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(3, 3, 1, 1)); \ @@ -164,7 +164,7 @@ __m128 xmm_sum_0_0 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_0), \ _mm256_extractf128_ps(ymm_sum_0_0, 1)); \ if (q != qe) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi16(*(const short *)(q))); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_q, xmm_sum_0_0) \ } \ @@ -198,7 +198,7 @@ if (q != qe) { \ __m128 xmm_q_0 = _mm_cvtph_ps(_mm_set1_epi16(*(const short *)(q + 0))); \ __m128 xmm_q_1 = _mm_cvtph_ps(_mm_set1_epi16(*(const short *)(q + 1))); \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ MATRIX_VAR_PROC(1, 2, 0, xmm_m, xmm_q, xmm_sum, ACCUM_FP32_STEP_SSE) \ } \ if (((uintptr_t)out & 0xf) == 0) { \ @@ -233,8 +233,8 @@ __m128 xmm_sum_0_3 = _mm_add_ps(_mm256_castps256_ps128(ymm_sum_0_3), \ _mm256_extractf128_ps(ymm_sum_0_3, 1)); \ if (q != qe) { \ - __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(m))); \ - __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64(*(const __m64 *)(q))); \ + __m128 xmm_m = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(m))); \ + __m128 xmm_q = _mm_cvtph_ps(_mm_set1_epi64x(*(const long long *)(q))); \ __m128 xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(0, 0, 0, 0)); \ ACCUM_FP32_STEP_SSE(xmm_m, xmm_p, xmm_sum_0_0) \ xmm_p = _mm_permute_ps(xmm_q, _MM_SHUFFLE(1, 1, 1, 1)); \ diff --git a/src/ailego/math/distance_matrix_fp16.i b/src/ailego/math/distance_matrix_fp16.i index 26ab75f08..11f05eb2f 100644 --- a/src/ailego/math/distance_matrix_fp16.i +++ b/src/ailego/math/distance_matrix_fp16.i @@ -67,9 +67,9 @@ } \ case 4: { \ __m256 ymm_lhs = _mm256_cvtph_ps( \ - _mm_set_epi64((__m64)(_MASK), *((const __m64 *)(lhs)))); \ + _mm_set_epi64x((long long)(_MASK), *(const long long *)(lhs))); \ __m256 ymm_rhs = _mm256_cvtph_ps( \ - _mm_set_epi64((__m64)(_MASK), *((const __m64 *)(rhs)))); \ + _mm_set_epi64x((long long)(_MASK), *(const long long *)(rhs))); \ _PROC(ymm_lhs, ymm_rhs, _RES##_0_0) \ break; \ } \ @@ -125,7 +125,7 @@ { \ __m256 ymm_m = _mm256_cvtph_ps(_LOAD((const __m128i *)(m))); \ __m256 ymm_q = _mm256_cvtph_ps(_mm_shufflehi_epi16( \ - _mm_shufflelo_epi16(_mm_set1_epi64(*(const __m64 *)(q)), \ + _mm_shufflelo_epi16(_mm_set1_epi64x(*(const long long *)(q)), \ _MM_SHUFFLE(1, 1, 0, 0)), \ _MM_SHUFFLE(3, 3, 2, 2))); \ _PROC(ymm_m, ymm_q, _RES##_0_0) \ @@ -155,7 +155,7 @@ //! Iterative process of computing distance (FP16, M=4, N=2) #define MATRIX_FP16_ITER_4X2_AVX(m, q, _RES, _LOAD, _PROC) \ { \ - __m128i xmm_qi = _mm_set1_epi64(*(const __m64 *)(q)); \ + __m128i xmm_qi = _mm_set1_epi64x(*(const long long *)(q)); \ __m256 ymm_m = _mm256_cvtph_ps(_LOAD((const __m128i *)(m))); \ __m256 ymm_q_0 = _mm256_cvtph_ps(_mm_shufflehi_epi16( \ _mm_shufflelo_epi16(xmm_qi, _MM_SHUFFLE(0, 0, 0, 0)), \ diff --git a/src/ailego/math/inner_product_matrix.h b/src/ailego/math/inner_product_matrix.h index d141722b8..f7e3eb294 100644 --- a/src/ailego/math/inner_product_matrix.h +++ b/src/ailego/math/inner_product_matrix.h @@ -852,7 +852,7 @@ struct MinusInnerProductSparseMatrix { const void *sparse_value, std::string &buffer); - static inline float ComputeInnerProductSparseInSegment( + static float ComputeInnerProductSparseInSegment( uint32_t m_sparse_count, const uint16_t *m_sparse_index, const ValueType *m_sparse_value, uint32_t q_sparse_count, const uint16_t *q_sparse_index, const ValueType *q_sparse_value); diff --git a/src/ailego/math/inner_product_matrix_fp16.cc b/src/ailego/math/inner_product_matrix_fp16.cc new file mode 100644 index 000000000..e69de29bb diff --git a/src/ailego/math/inner_product_matrix_fp16_avx512.cc b/src/ailego/math/inner_product_matrix_fp16_avx512.cc index 7e07952ee..0aa86af6a 100644 --- a/src/ailego/math/inner_product_matrix_fp16_avx512.cc +++ b/src/ailego/math/inner_product_matrix_fp16_avx512.cc @@ -735,7 +735,7 @@ float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count, _mm_loadu_ph(val_start_2 + k))); } - Float16 __attribute__((aligned(16))) tmp_res[8]; + alignas(16) Float16 tmp_res[8]; _mm_store_ph(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3] + tmp_res[4] + tmp_res[5] + tmp_res[6] + tmp_res[7]); diff --git a/src/ailego/math/inner_product_matrix_fp32.cc b/src/ailego/math/inner_product_matrix_fp32.cc new file mode 100644 index 000000000..e69de29bb diff --git a/src/ailego/math/inner_product_matrix_fp32_sse.cc b/src/ailego/math/inner_product_matrix_fp32_sse.cc index f90801eec..accd70ece 100644 --- a/src/ailego/math/inner_product_matrix_fp32_sse.cc +++ b/src/ailego/math/inner_product_matrix_fp32_sse.cc @@ -298,7 +298,7 @@ float InnerProductSparseInSegmentSSE(uint32_t m_sparse_count, _mm_loadu_ps(val_start_2 + k))); } - float __attribute__((aligned(16))) tmp_res[4]; + alignas(16) float tmp_res[4]; _mm_store_ps(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); } diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc new file mode 100644 index 000000000..e69de29bb diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc index 357703db6..ff9f29ebc 100644 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc @@ -328,7 +328,7 @@ float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count, _mm_loadu_ps(val_start_2 + k))); } - float __attribute__((aligned(16))) tmp_res[4]; + alignas(16) float tmp_res[4]; _mm_store_ps(tmp_res, sum128); sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); } diff --git a/src/ailego/math/norm_matrix_fp16.i b/src/ailego/math/norm_matrix_fp16.i index fec074211..c1d3bd219 100644 --- a/src/ailego/math/norm_matrix_fp16.i +++ b/src/ailego/math/norm_matrix_fp16.i @@ -49,7 +49,7 @@ } \ case 4: { \ __m256 ymm_m = _mm256_cvtph_ps( \ - _mm_set_epi64((__m64)(0ull), *((const __m64 *)(m)))); \ + _mm_set_epi64x(0LL, *(const long long *)(m))); \ NORM_FP32_STEP_AVX(ymm_m, _RES##_0_0) \ break; \ } \ diff --git a/src/ailego/math_batch/inner_product_distance_batch.h b/src/ailego/math_batch/inner_product_distance_batch.h index f42345bf1..6375482a2 100644 --- a/src/ailego/math_batch/inner_product_distance_batch.h +++ b/src/ailego/math_batch/inner_product_distance_batch.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl_fp32_avx2.cc b/src/ailego/math_batch/inner_product_distance_batch_impl_fp32_avx2.cc index ffda66e9f..bead50745 100644 --- a/src/ailego/math_batch/inner_product_distance_batch_impl_fp32_avx2.cc +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_fp32_avx2.cc @@ -24,7 +24,7 @@ namespace zvec::ailego::DistanceBatch { inline float sum4(__m128 v) { v = _mm_add_ps(v, _mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(v), 8))); - return v[0] + v[1]; + return _mm_cvtss_f32(v) + _mm_cvtss_f32(_mm_shuffle_ps(v, v, 1)); } inline __m128 sum_top_bottom_avx(__m256 v) { diff --git a/src/ailego/utility/string_helper.cc b/src/ailego/utility/string_helper.cc index 6c2015782..74d3affd6 100644 --- a/src/ailego/utility/string_helper.cc +++ b/src/ailego/utility/string_helper.cc @@ -14,6 +14,7 @@ #include #include +#include #include namespace zvec { @@ -64,10 +65,6 @@ std::string StringHelper::CopyTrim(std::string str) { return str; } -#if defined(_MSC_VER) -#define strncasecmp _strnicmp -#endif - bool StringHelper::CompareIgnoreCase(const std::string &a, const std::string &b) { if (a.size() != b.size()) { diff --git a/src/core/algorithm/cluster/kmeans_cluster.cc b/src/core/algorithm/cluster/kmeans_cluster.cc index 507159b86..e73a803a6 100644 --- a/src/core/algorithm/cluster/kmeans_cluster.cc +++ b/src/core/algorithm/cluster/kmeans_cluster.cc @@ -891,8 +891,10 @@ void KmeansCluster::update_features_thread(size_t column, for (size_t i = column, j = 0; i < shard_cluster_features_.size(); i += cluster_count) { const std::vector &it = shard_cluster_features_[i]; - std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); - j += it.size(); + if (!it.empty()) { + std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); + j += it.size(); + } } } diff --git a/src/core/algorithm/cluster/opt_kmeans_cluster.cc b/src/core/algorithm/cluster/opt_kmeans_cluster.cc index 49ae8f5f3..aa2269f4d 100644 --- a/src/core/algorithm/cluster/opt_kmeans_cluster.cc +++ b/src/core/algorithm/cluster/opt_kmeans_cluster.cc @@ -327,8 +327,10 @@ void OptKmeansAlgorithm::update_features_thread( for (size_t i = column, j = 0; i < shard_cluster_features_.size(); i += cluster_count) { const std::vector &it = shard_cluster_features_[i]; - std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); - j += it.size(); + if (!it.empty()) { + std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *)); + j += it.size(); + } } } diff --git a/src/core/algorithm/flat/flat_builder.h b/src/core/algorithm/flat/flat_builder.h index 5a48c6e50..665179bcb 100644 --- a/src/core/algorithm/flat/flat_builder.h +++ b/src/core/algorithm/flat/flat_builder.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once +#include #include #include #include "flat_utility.h" diff --git a/src/core/algorithm/flat/flat_index_format.h b/src/core/algorithm/flat/flat_index_format.h index d91385abd..08d93c793 100644 --- a/src/core/algorithm/flat/flat_index_format.h +++ b/src/core/algorithm/flat/flat_index_format.h @@ -41,7 +41,6 @@ struct LinearIndexHeader { block_count(0), index_meta_size(0) { memset(reserved_, 0, sizeof(reserved_)); - memset(index_meta, 0, sizeof(index_meta)); } uint32_t header_size{0}; uint32_t total_vector_count{0}; @@ -52,7 +51,7 @@ struct LinearIndexHeader { uint32_t block_count{0}; uint32_t index_meta_size{0}; char reserved_[28] = {0}; - char index_meta[0]; + char index_meta[]; }; /*! Index Format of Linear Index Meta for each Linear list diff --git a/src/core/algorithm/flat/flat_streamer.h b/src/core/algorithm/flat/flat_streamer.h index 66ceba6a0..567b84da5 100644 --- a/src/core/algorithm/flat/flat_streamer.h +++ b/src/core/algorithm/flat/flat_streamer.h @@ -180,10 +180,10 @@ class FlatStreamer : public IndexStreamer { State state_{STATE_INIT}; mutable std::mutex mapping_mutex_{}; ailego::SharedMutex dump_mutex_{}; - FlatStreamerEntity entity_; bool column_major_order_{false}; bool use_key_info_map_{true}; uint32_t read_block_size_{0}; + FlatStreamerEntity entity_; }; } // namespace core diff --git a/src/core/algorithm/flat/flat_streamer_dumper.h b/src/core/algorithm/flat/flat_streamer_dumper.h index 7d3044bc0..30b2994bf 100644 --- a/src/core/algorithm/flat/flat_streamer_dumper.h +++ b/src/core/algorithm/flat/flat_streamer_dumper.h @@ -14,6 +14,7 @@ #pragma once +#include #include "flat_streamer.h" #include "flat_utility.h" diff --git a/src/core/algorithm/flat/flat_streamer_entity.h b/src/core/algorithm/flat/flat_streamer_entity.h index 6ac410a82..7cf915429 100644 --- a/src/core/algorithm/flat/flat_streamer_entity.h +++ b/src/core/algorithm/flat/flat_streamer_entity.h @@ -389,7 +389,6 @@ class FlatStreamerEntity { IndexStorage::Pointer storage_{}; IndexMetric::MatrixDistance row_distance_{}, column_distance_{}; mutable std::vector segments_{}; - StreamerLinearMeta meta_{}; IndexStreamer::Stats &stats_; mutable std::shared_ptr key_info_map_lock_{}; std::unordered_map key_info_map_{}; @@ -401,6 +400,7 @@ class FlatStreamerEntity { uint32_t vec_unit_size_{0}; uint32_t vec_cols_{0}; mutable std::string vec_buf_{}; + StreamerLinearMeta meta_{}; }; } // namespace core diff --git a/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc b/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc index 53d8a24ce..1930d4325 100644 --- a/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc +++ b/src/core/algorithm/flat_sparse/flat_sparse_streamer_entity.cc @@ -15,6 +15,7 @@ #include "flat_sparse_streamer_entity.h" #include #include +#include #include #include #include diff --git a/src/core/algorithm/hnsw/hnsw_builder_entity.cc b/src/core/algorithm/hnsw/hnsw_builder_entity.cc index 472d98ee5..10591ac10 100644 --- a/src/core/algorithm/hnsw/hnsw_builder_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_builder_entity.cc @@ -136,8 +136,8 @@ int HnswBuilderEntity::get_vector(const node_id_t *ids, uint32_t count, int HnswBuilderEntity::get_vector( const node_id_t *ids, uint32_t count, std::vector &vec_blocks) const { - const void *vecs[count]; - get_vector(ids, count, vecs); + std::vector vecs(count); + get_vector(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } diff --git a/src/core/algorithm/hnsw/hnsw_chunk.h b/src/core/algorithm/hnsw/hnsw_chunk.h index b19d30943..7968dff95 100644 --- a/src/core/algorithm/hnsw/hnsw_chunk.h +++ b/src/core/algorithm/hnsw/hnsw_chunk.h @@ -15,7 +15,6 @@ #include #include -#include #include #include #include diff --git a/src/core/algorithm/hnsw/hnsw_dist_calculator.h b/src/core/algorithm/hnsw/hnsw_dist_calculator.h index 84faba40b..caf6e6d15 100644 --- a/src/core/algorithm/hnsw/hnsw_dist_calculator.h +++ b/src/core/algorithm/hnsw/hnsw_dist_calculator.h @@ -145,11 +145,11 @@ class HnswDistCalculator { return dist(vec); } - dist_t operator()(id_t i) { + dist_t operator()(node_id_t i) { return dist(i); } - dist_t operator()(id_t lhs, id_t rhs) { + dist_t operator()(node_id_t lhs, node_id_t rhs) { return dist(lhs, rhs); } diff --git a/src/core/algorithm/hnsw/hnsw_entity.cc b/src/core/algorithm/hnsw/hnsw_entity.cc index 53f07dd44..aa0c0a3fe 100644 --- a/src/core/algorithm/hnsw/hnsw_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_entity.cc @@ -318,7 +318,8 @@ int64_t HnswEntity::dump_upper_neighbors( ailego_assert_with(!!neighbors.data, "invalid neighbors"); ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level), "invalid neighbors"); - memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size()); + size_t buffer_bytes = buffer.size() * sizeof(node_id_t); + memset(buffer.data(), 0, buffer_bytes); buffer[0] = neighbors.size(); if (neighbor_mapping.empty()) { memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t)); @@ -327,15 +328,13 @@ int64_t HnswEntity::dump_upper_neighbors( buffer[i + 1] = neighbor_mapping[neighbors[i]]; } } - if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) != - sizeof(node_id_t) * buffer.size()) { + if (dumper->write(buffer.data(), buffer_bytes) != buffer_bytes) { LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, - sizeof(node_id_t) * buffer.size()); + buffer_bytes); return IndexError_WriteData; } - crc = ailego::Crc32c::Hash(buffer.data(), - sizeof(node_id_t) * buffer.size(), crc); - offset += sizeof(node_id_t) * buffer.size(); + crc = ailego::Crc32c::Hash(buffer.data(), buffer_bytes, crc); + offset += buffer_bytes; } } size_t padding_size = 0; diff --git a/src/core/algorithm/hnsw/hnsw_entity.h b/src/core/algorithm/hnsw/hnsw_entity.h index e5f2077f7..816676dff 100644 --- a/src/core/algorithm/hnsw/hnsw_entity.h +++ b/src/core/algorithm/hnsw/hnsw_entity.h @@ -138,7 +138,7 @@ struct HNSWHeader { struct NeighborsHeader { uint32_t neighbor_cnt; - node_id_t neighbors[0]; + node_id_t neighbors[]; }; struct Neighbors { diff --git a/src/core/algorithm/hnsw/hnsw_searcher_entity.cc b/src/core/algorithm/hnsw/hnsw_searcher_entity.cc index 6661c1db3..6829f71d7 100644 --- a/src/core/algorithm/hnsw/hnsw_searcher_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_searcher_entity.cc @@ -148,8 +148,8 @@ int HnswSearcherEntity::get_vector(const node_id_t *ids, uint32_t count, int HnswSearcherEntity::get_vector( const node_id_t *ids, uint32_t count, std::vector &vec_blocks) const { - const void *vecs[count]; - get_vector(ids, count, vecs); + std::vector vecs(count); + get_vector(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { vec_blocks.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } diff --git a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc index 71e2e477b..90a2a373e 100644 --- a/src/core/algorithm/hnsw/hnsw_streamer_entity.cc +++ b/src/core/algorithm/hnsw/hnsw_streamer_entity.cc @@ -89,7 +89,7 @@ int HnswStreamerEntity::update_neighbors( } auto loc = get_neighbor_chunk_loc(level, id); - size_t size = reinterpret_cast(&hd->neighbors[i]) - &buffer[0]; + size_t size = reinterpret_cast(&hd->neighbors[i]) - buffer.data(); size_t ret = loc.first->write(loc.second, hd, size); if (ailego_unlikely(ret != size)) { LOG_ERROR("Write neighbor header failed, ret=%zu", ret); @@ -485,7 +485,9 @@ int HnswStreamerEntity::check_hnsw_index(const HNSWHeader *hd) const { int HnswStreamerEntity::add_vector(level_t level, key_t key, const void *vec, node_id_t *id) { Chunk::Pointer node_chunk; - size_t chunk_offset = -1UL; + // On MSVC, unsigned long is 32-bit, so -1UL is 0xFFFFFFFF not + // 0xFFFFFFFFFFFFFFFF. + size_t chunk_offset = static_cast(-1); std::lock_guard lock(mutex_); // duplicate check @@ -558,7 +560,7 @@ int HnswStreamerEntity::add_vector(level_t level, key_t key, const void *vec, int HnswStreamerEntity::add_vector_with_id(level_t level, node_id_t id, const void *vec) { Chunk::Pointer node_chunk; - size_t chunk_offset = -1UL; + size_t chunk_offset = static_cast(-1); key_t key = id; std::lock_guard lock(mutex_); diff --git a/src/core/algorithm/hnsw/hnsw_streamer_entity.h b/src/core/algorithm/hnsw/hnsw_streamer_entity.h index 1a01b1416..9e3a95cfd 100644 --- a/src/core/algorithm/hnsw/hnsw_streamer_entity.h +++ b/src/core/algorithm/hnsw/hnsw_streamer_entity.h @@ -331,10 +331,10 @@ class HnswStreamerEntity : public HnswEntity { return 0; } Chunk::Pointer chunk; - uint64_t chunk_offset = -1UL; + uint64_t chunk_offset = UINT64_MAX; size_t neighbors_size = get_total_upper_neighbors_size(level); - uint64_t chunk_index = upper_neighbor_chunks_.size() - 1UL; - if (chunk_index == -1UL || + uint64_t chunk_index = upper_neighbor_chunks_.size() - 1ULL; + if (chunk_index == UINT64_MAX || (upper_neighbor_chunks_[chunk_index]->padding_size() < neighbors_size)) { // no space left and need to alloc chunk_index++; diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc index 48c20d726..7d912d54b 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_builder_entity.cc @@ -168,8 +168,8 @@ int HnswSparseBuilderEntity::get_vector_metas(const node_id_t *ids, int HnswSparseBuilderEntity::get_vector_metas( const node_id_t *ids, uint32_t count, std::vector &block_vecs) const { - const void *vecs[count]; - get_vector_metas(ids, count, vecs); + std::vector vecs(count); + get_vector_metas(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { block_vecs.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h index a02698914..c0c3da08a 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_chunk.h @@ -15,7 +15,6 @@ #include #include -#include #include #include #include diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h index 06cb76e4d..5bbfc1b6a 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_dist_calculator.h @@ -138,11 +138,11 @@ class HnswSparseDistCalculator { return dist(vec); } - dist_t operator()(id_t i) { + dist_t operator()(node_id_t i) { return dist(i); } - dist_t operator()(id_t lhs, id_t rhs) { + dist_t operator()(node_id_t lhs, node_id_t rhs) { return dist(lhs, rhs); } diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc index a0ecb2fc1..29ec39b94 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "hnsw_sparse_entity.h" +#include namespace zvec { namespace core { @@ -381,6 +382,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors( size_t offset = 0; uint32_t crc = 0; std::vector buffer(upper_neighbor_cnt() + 1); + size_t buffer_bytes = buffer.size() * sizeof(node_id_t); for (node_id_t id = 0; id < doc_cnt(); ++id) { node_id_t new_id = reorder_mapping.empty() ? id : reorder_mapping[id]; auto level = get_level(new_id); @@ -395,7 +397,7 @@ int64_t HnswSparseEntity::dump_upper_neighbors( ailego_assert_with(!!neighbors.data, "invalid neighbors"); ailego_assert_with(neighbors.size() <= neighbor_cnt(cur_level), "invalid neighbors"); - memset(buffer.data(), 0, sizeof(node_id_t) * buffer.size()); + memset(buffer.data(), 0, buffer_bytes); buffer[0] = neighbors.size(); if (neighbor_mapping.empty()) { memcpy(&buffer[1], &neighbors[0], neighbors.size() * sizeof(node_id_t)); @@ -404,15 +406,13 @@ int64_t HnswSparseEntity::dump_upper_neighbors( buffer[i + 1] = neighbor_mapping[neighbors[i]]; } } - if (dumper->write(buffer.data(), sizeof(node_id_t) * buffer.size()) != - sizeof(node_id_t) * buffer.size()) { + if (dumper->write(buffer.data(), buffer_bytes) != buffer_bytes) { LOG_ERROR("Dump graph neighbor id=%u failed, size %lu", id, - sizeof(node_id_t) * buffer.size()); + buffer_bytes); return IndexError_WriteData; } - crc = ailego::Crc32c::Hash(buffer.data(), - sizeof(node_id_t) * buffer.size(), crc); - offset += sizeof(node_id_t) * buffer.size(); + crc = ailego::Crc32c::Hash(buffer.data(), buffer_bytes, crc); + offset += buffer_bytes; } } size_t padding_size = 0; diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h index 37166027e..65e918372 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_entity.h @@ -161,7 +161,7 @@ struct HNSWSparseHeader { struct NeighborsHeader { uint32_t neighbor_cnt; - node_id_t neighbors[0]; + node_id_t neighbors[]; }; struct Neighbors { diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc index 8ef79adf2..4b23ab9a6 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_entity.cc @@ -155,8 +155,8 @@ int HnswSparseSearcherEntity::get_vector_metas(const node_id_t *ids, int HnswSparseSearcherEntity::get_vector_metas( const node_id_t *ids, uint32_t count, std::vector &block_vecs) const { - const void *vecs[count]; - get_vector_metas(ids, count, vecs); + std::vector vecs(count); + get_vector_metas(ids, count, vecs.data()); for (uint32_t i = 0; i < count; ++i) { block_vecs.emplace_back(IndexStorage::MemoryBlock((void *)vecs[i])); } diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc index aaffd9c12..811347f61 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.cc @@ -93,7 +93,7 @@ int HnswSparseStreamerEntity::update_neighbors( } auto loc = get_neighbor_chunk_loc(level, id); - size_t size = reinterpret_cast(&hd->neighbors[i]) - &buffer[0]; + size_t size = reinterpret_cast(&hd->neighbors[i]) - buffer.data(); size_t ret = loc.first->write(loc.second, hd, size); if (ailego_unlikely(ret != size)) { LOG_ERROR("Write neighbor header failed, ret=%zu", ret); @@ -607,8 +607,8 @@ int HnswSparseStreamerEntity::add_vector(level_t level, key_t key, SparseChunk::Pointer node_chunk; SparseChunk::Pointer sparse_node_chunk; - size_t chunk_offset = -1UL; - size_t sparse_chunk_offset = -1UL; + size_t chunk_offset = static_cast(-1); + size_t sparse_chunk_offset = static_cast(-1); std::lock_guard lock(mutex_); // duplicate check @@ -744,8 +744,8 @@ int HnswSparseStreamerEntity::add_vector_with_id(level_t level, node_id_t id, key_t key = id; SparseChunk::Pointer node_chunk; SparseChunk::Pointer sparse_node_chunk; - size_t chunk_offset = -1UL; - size_t sparse_chunk_offset = -1UL; + size_t chunk_offset = static_cast(-1); + size_t sparse_chunk_offset = static_cast(-1); // allocat sparse chunk uint32_t sparse_vector_len = sparse_vec.size(); diff --git a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h index 84087afc3..67a7d89ad 100644 --- a/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h +++ b/src/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_entity.h @@ -360,10 +360,10 @@ class HnswSparseStreamerEntity : public HnswSparseEntity { return 0; } SparseChunk::Pointer chunk; - uint64_t chunk_offset = -1UL; + uint64_t chunk_offset = UINT64_MAX; size_t neighbors_size = get_total_upper_neighbors_size(level); - uint64_t chunk_index = upper_neighbor_chunks_.size() - 1UL; - if (chunk_index == -1UL || + uint64_t chunk_index = upper_neighbor_chunks_.size() - 1ULL; + if (chunk_index == UINT64_MAX || (upper_neighbor_chunks_[chunk_index]->padding_size() < neighbors_size)) { // no space left and need to alloc chunk_index++; diff --git a/src/core/algorithm/ivf/ivf_dumper.h b/src/core/algorithm/ivf/ivf_dumper.h index d8ab9935b..81c03c835 100644 --- a/src/core/algorithm/ivf/ivf_dumper.h +++ b/src/core/algorithm/ivf/ivf_dumper.h @@ -249,11 +249,11 @@ class IVFDumper { size_t block_vector_count_{kDefaultBlockCount}; std::vector inverted_lists_meta_{}; std::vector keys_{}; - InvertedIndexHeader header_{}; uint32_t cur_list_id_{0}; uint32_t dumped_feature_count_{0}; size_t dumped_features_size_{0}; mutable size_t dumped_size_{0}; + InvertedIndexHeader header_; }; } // namespace core diff --git a/src/core/algorithm/ivf/ivf_entity.h b/src/core/algorithm/ivf/ivf_entity.h index c9d3d9647..d27f53e36 100644 --- a/src/core/algorithm/ivf/ivf_entity.h +++ b/src/core/algorithm/ivf/ivf_entity.h @@ -333,7 +333,6 @@ class IVFEntity { IndexMeta meta_{}; mutable IVFReformerWrapper reformer_{}; IVFDistanceCalculator::Pointer calculator_{}; - InvertedIndexHeader header_{}; IndexStorage::Pointer container_{}; IndexStorage::Segment::Pointer inverted_{}; IndexStorage::Segment::Pointer inverted_meta_{}; @@ -345,6 +344,7 @@ class IVFEntity { mutable std::string vector_{}; // temporary buffer for colomn major order float norm_value_{0.0f}; // normalize the inverted vector to orignal score bool norm_value_sqrt_{false}; // does the norm value need to sqrt + InvertedIndexHeader header_; }; } // namespace core diff --git a/src/core/algorithm/ivf/ivf_index_format.h b/src/core/algorithm/ivf/ivf_index_format.h index d86ae52b9..bf125fca8 100644 --- a/src/core/algorithm/ivf/ivf_index_format.h +++ b/src/core/algorithm/ivf/ivf_index_format.h @@ -19,6 +19,9 @@ namespace zvec { namespace core { +using node_id_t = uint32_t; +using key_t = uint64_t; + static constexpr uint64_t kInvalidKey = std::numeric_limits::max(); /*! Index Format of Inverted Index Header @@ -33,7 +36,7 @@ struct InvertedIndexHeader { uint32_t block_count{0}; uint32_t index_meta_size{0}; char reserved_[28]; - char index_meta[0]; + char index_meta[]; }; /*! Index Format of Inverted Index Meta for each Inverted list diff --git a/src/core/algorithm/ivf/ivf_utility.h b/src/core/algorithm/ivf/ivf_utility.h index 38e41a4fd..e7d213194 100644 --- a/src/core/algorithm/ivf/ivf_utility.h +++ b/src/core/algorithm/ivf/ivf_utility.h @@ -35,22 +35,22 @@ namespace core { #endif #ifndef ivf_check_with_msg -#define ivf_check_with_msg(code, fmt, args...) \ - do { \ - if (ailego_unlikely((code) != 0)) { \ - LOG_ERROR(fmt, ##args); \ - return code; \ - } \ +#define ivf_check_with_msg(code, fmt, ...) \ + do { \ + if (ailego_unlikely((code) != 0)) { \ + LOG_ERROR(fmt, ##__VA_ARGS__); \ + return code; \ + } \ } while (0) #endif #ifndef ivf_assert_with_msg -#define ivf_assert_with_msg(cond, err, fmt, args...) \ - do { \ - if (ailego_unlikely(!(cond))) { \ - LOG_ERROR(fmt, ##args); \ - return err; \ - } \ +#define ivf_assert_with_msg(cond, err, fmt, ...) \ + do { \ + if (ailego_unlikely(!(cond))) { \ + LOG_ERROR(fmt, ##__VA_ARGS__); \ + return err; \ + } \ } while (0) #endif diff --git a/src/core/utility/sparse_utility.h b/src/core/utility/sparse_utility.h index 6f057c557..d99b8d82b 100644 --- a/src/core/utility/sparse_utility.h +++ b/src/core/utility/sparse_utility.h @@ -15,7 +15,6 @@ #include #include -#include #include #include #include @@ -25,6 +24,8 @@ namespace zvec { namespace core { +using key_t = uint64_t; + constexpr uint32_t SEGMENT_ID_BITS = 16; constexpr uint32_t SEGMENT_ID_MASK = 0xFFFF; diff --git a/src/core/utility/visit_filter.h b/src/core/utility/visit_filter.h index ea76c14c5..959a0b450 100644 --- a/src/core/utility/visit_filter.h +++ b/src/core/utility/visit_filter.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include diff --git a/src/db/common/cgroup_util.cc b/src/db/common/cgroup_util.cc index 6a56dfcda..e358c2d20 100644 --- a/src/db/common/cgroup_util.cc +++ b/src/db/common/cgroup_util.cc @@ -75,6 +75,8 @@ uint64_t CgroupUtil::getUptime() { time_t csec = time(NULL); return csec - bsec; } +#elif defined(PLATFORM_WINDOWS) + return GetTickCount64() / 1000; #endif return 0; } @@ -97,6 +99,13 @@ void CgroupUtil::updateCpuCores() { if (cpu_cores_ <= 0) { cpu_cores_ = 1; } +#elif defined(PLATFORM_WINDOWS) + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + cpu_cores_ = static_cast(sysinfo.dwNumberOfProcessors); + if (cpu_cores_ <= 0) { + cpu_cores_ = 1; + } #endif } @@ -162,6 +171,14 @@ void CgroupUtil::updateMemoryLimit() { } else { memory_limit_ = 0; } +#elif defined(PLATFORM_WINDOWS) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + memory_limit_ = statex.ullTotalPhys; + } else { + memory_limit_ = 0; + } #endif } @@ -253,6 +270,15 @@ uint64_t CgroupUtil::getCurrentMemoryUsage() { return readMemoryUsageProc(); #elif defined(PLATFORM_MACOS) return getMacOSMemoryUsage(); +#elif defined(PLATFORM_WINDOWS) + MEMORYSTATUSEX statex; + statex.dwLength = sizeof(statex); + if (GlobalMemoryStatusEx(&statex)) { + return statex.ullTotalPhys - statex.ullAvailPhys; + } + return 0; +#else + return 0; #endif } @@ -369,8 +395,9 @@ double CgroupUtil::calculateCpuUsage() { return calculateLinuxCpuUsage(); #elif defined(PLATFORM_MACOS) return calculateMacOSCpuUsage(); -#endif +#else return 0.0; +#endif } #if defined(PLATFORM_LINUX) diff --git a/src/db/common/cgroup_util.h b/src/db/common/cgroup_util.h index 7df82aee0..759e2317c 100644 --- a/src/db/common/cgroup_util.h +++ b/src/db/common/cgroup_util.h @@ -32,6 +32,9 @@ #define PLATFORM_LINUX 1 #include #include +#elif defined(_WIN32) || defined(_WIN64) +#define PLATFORM_WINDOWS 1 +#include #endif namespace zvec { diff --git a/src/db/common/constants.h b/src/db/common/constants.h index 39b13f445..339664228 100644 --- a/src/db/common/constants.h +++ b/src/db/common/constants.h @@ -24,7 +24,7 @@ const float DEFAULT_MEMORY_LIMIT_RATIO = 0.8f; const uint32_t MIN_MEMORY_LIMIT_BYTES = 100 * 1024 * 1024; -const uint64_t INVALID_DOC_ID = -1UL; +const uint64_t INVALID_DOC_ID = UINT64_MAX; const std::string LOCAL_ROW_ID = "_zvec_row_id_"; diff --git a/src/db/common/file_helper.cc b/src/db/common/file_helper.cc index aa766f8ca..c82131681 100644 --- a/src/db/common/file_helper.cc +++ b/src/db/common/file_helper.cc @@ -13,14 +13,19 @@ // limitations under the License. #include "file_helper.h" -#include -#include #include -#include #include -#include #include #include +#ifdef _MSC_VER +#include +#include +#else +#include +#include +#include +#include +#endif #include namespace zvec { @@ -32,6 +37,18 @@ const std::string FileHelper::RECOVER_SUFFIX = ".recovering"; bool FileHelper::CopyFile(const std::string &src_file_path, const std::string &dst_file_path) { +#ifdef _MSC_VER + std::string dst_file_path_tmp = dst_file_path + ".tmp"; + std::error_code ec; + std::filesystem::copy_file(src_file_path, dst_file_path_tmp, + std::filesystem::copy_options::overwrite_existing, + ec); + if (ec) { + return false; + } + std::filesystem::rename(dst_file_path_tmp, dst_file_path, ec); + return !ec; +#else int src_fd = open(src_file_path.c_str(), O_RDONLY, 0); if (src_fd < 0) { return false; @@ -54,10 +71,19 @@ bool FileHelper::CopyFile(const std::string &src_file_path, } } return rename(dst_file_path_tmp.c_str(), dst_file_path.c_str()) == 0; +#endif } bool FileHelper::CopyDirectory(const std::string &src_dir_path, const std::string &dst_dir_path) { +#ifdef _MSC_VER + std::error_code ec; + std::filesystem::copy(src_dir_path, dst_dir_path, + std::filesystem::copy_options::recursive | + std::filesystem::copy_options::overwrite_existing, + ec); + return !ec; +#else DIR *dir = opendir(src_dir_path.c_str()); if (!dir) { return false; @@ -91,6 +117,7 @@ bool FileHelper::CopyDirectory(const std::string &src_dir_path, } } return true; +#endif } void FileHelper::CleanupDirectory(const std::string &backup_dir, @@ -100,6 +127,18 @@ void FileHelper::CleanupDirectory(const std::string &backup_dir, return; } +#ifdef _MSC_VER + size_t prefix_len = strlen(prefix_name); + std::vector candidates; + std::error_code ec; + for (const auto &entry : + std::filesystem::directory_iterator(backup_dir, ec)) { + std::string name = entry.path().filename().string(); + if (name.compare(0, prefix_len, prefix_name) == 0) { + candidates.emplace_back(name); + } + } +#else DIR *dir = opendir(backup_dir.c_str()); if (!dir) { return; @@ -115,6 +154,7 @@ void FileHelper::CleanupDirectory(const std::string &backup_dir, candidates.emplace_back(dent->d_name); } } +#endif if (candidates.size() <= max_backup_count) { return; } diff --git a/src/db/index/column/vector_column/combined_vector_column_indexer.cc b/src/db/index/column/vector_column/combined_vector_column_indexer.cc index 14fd2193b..f64001e83 100644 --- a/src/db/index/column/vector_column/combined_vector_column_indexer.cc +++ b/src/db/index/column/vector_column/combined_vector_column_indexer.cc @@ -14,6 +14,7 @@ #include "combined_vector_column_indexer.h" #include #include +#include namespace zvec { diff --git a/src/db/index/column/vector_column/engine_helper.hpp b/src/db/index/column/vector_column/engine_helper.hpp index de1cfc6c0..74359d79e 100644 --- a/src/db/index/column/vector_column/engine_helper.hpp +++ b/src/db/index/column/vector_column/engine_helper.hpp @@ -109,12 +109,14 @@ class ProximaEngineHelper { engine_query_param->is_linear = db_query_params.query_params->is_linear(); } if (db_query_params.refiner_param) { - engine_query_param->refiner_param = - std::make_shared( - core_interface::RefinerParam{ - .scale_factor_ = db_query_params.refiner_param->scale_factor_, - .reference_index = - db_query_params.refiner_param->reference_indexer->index}); + { + core_interface::RefinerParam rp; + rp.scale_factor_ = db_query_params.refiner_param->scale_factor_; + rp.reference_index = + db_query_params.refiner_param->reference_indexer->index; + engine_query_param->refiner_param = + std::make_shared(rp); + } } return engine_query_param; diff --git a/src/db/index/common/doc.cc b/src/db/index/common/doc.cc index 6d411bfb2..2e0c6d3f3 100644 --- a/src/db/index/common/doc.cc +++ b/src/db/index/common/doc.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include #include "db/common/constants.h" #include "db/index/common/type_helper.h" @@ -117,16 +118,16 @@ template T byte_swap(T value) { if constexpr (std::is_same_v) { uint16_t val = *reinterpret_cast(&value); - val = __builtin_bswap16(val); + val = ailego_bswap16(val); return *reinterpret_cast(&val); } else if constexpr (sizeof(T) == 1) { return value; } else if constexpr (sizeof(T) == 2) { return (value << 8) | ((value >> 8) & 0xFF); } else if constexpr (sizeof(T) == 4) { - return __builtin_bswap32(value); + return static_cast(ailego_bswap32(static_cast(value))); } else if constexpr (sizeof(T) == 8) { - return __builtin_bswap64(value); + return static_cast(ailego_bswap64(static_cast(value))); } else { T result = 0; for (size_t i = 0; i < sizeof(T); ++i) { diff --git a/src/db/index/segment/segment.cc b/src/db/index/segment/segment.cc index 43928f39d..6a5341753 100644 --- a/src/db/index/segment/segment.cc +++ b/src/db/index/segment/segment.cc @@ -608,7 +608,7 @@ Status SegmentImpl::InsertVector(VectorColumnIndexer::Ptr &indexer, (void *)sparse_value.data()}; } else { vector_data.vector = - vector_column_params::DenseVector{.data = value.value().data()}; + vector_column_params::DenseVector{value.value().data()}; } auto &mem_block_meta = segment_meta_->writing_forward_block().value(); diff --git a/src/db/index/segment/segment_helper.cc b/src/db/index/segment/segment_helper.cc index 45b2dfb30..2c1038f2c 100644 --- a/src/db/index/segment/segment_helper.cc +++ b/src/db/index/segment/segment_helper.cc @@ -642,6 +642,9 @@ Status SegmentHelper::ReduceVectorIndex( s = vector_indexer->Flush(); CHECK_RETURN_STATUS(s); + s = vector_indexer->Close(); + CHECK_RETURN_STATUS(s); + BlockMeta new_block_meta; new_block_meta.set_id(vector_block_id); new_block_meta.set_type(BlockType::VECTOR_INDEX); @@ -687,6 +690,9 @@ Status SegmentHelper::ReduceVectorIndex( s = vector_indexer->Flush(); CHECK_RETURN_STATUS(s); + s = vector_indexer->Close(); + CHECK_RETURN_STATUS(s); + BlockMeta new_block_meta; new_block_meta.set_id(vector_block_id); new_block_meta.set_type(BlockType::VECTOR_INDEX); @@ -719,6 +725,9 @@ Status SegmentHelper::ReduceVectorIndex( s = vector_indexer_quantize->Flush(); CHECK_RETURN_STATUS(s); + s = vector_indexer_quantize->Close(); + CHECK_RETURN_STATUS(s); + new_block_meta.set_id(vector_quan_block_id); new_block_meta.set_type(BlockType::VECTOR_INDEX_QUANTIZE); new_block_meta.set_columns({field->name()}); diff --git a/src/db/index/storage/store_helper.h b/src/db/index/storage/store_helper.h index f9b3b5155..138941a47 100644 --- a/src/db/index/storage/store_helper.h +++ b/src/db/index/storage/store_helper.h @@ -40,7 +40,7 @@ namespace zvec { inline FileFormat InferFileFormat(const std::string &file_path) { - std::string ext = std::filesystem::path(file_path).extension(); + std::string ext = std::filesystem::path(file_path).extension().string(); std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower); if (ext == ".parquet") { return FileFormat::PARQUET; diff --git a/src/db/index/storage/wal/local_wal_file.cc b/src/db/index/storage/wal/local_wal_file.cc index 91490b212..6a50e5c34 100644 --- a/src/db/index/storage/wal/local_wal_file.cc +++ b/src/db/index/storage/wal/local_wal_file.cc @@ -13,7 +13,9 @@ // limitations under the License. #include "local_wal_file.h" +#ifndef _MSC_VER #include +#endif #include #include #include "db/common/error_code.h" diff --git a/src/db/sqlengine/common/generic_node.h b/src/db/sqlengine/common/generic_node.h index a90de1fbc..2275c3d57 100644 --- a/src/db/sqlengine/common/generic_node.h +++ b/src/db/sqlengine/common/generic_node.h @@ -15,6 +15,7 @@ #pragma once #include +#include namespace zvec::sqlengine { diff --git a/src/db/sqlengine/common/util.cc b/src/db/sqlengine/common/util.cc index 3f18bfe4d..568830ad6 100644 --- a/src/db/sqlengine/common/util.cc +++ b/src/db/sqlengine/common/util.cc @@ -13,7 +13,9 @@ // limitations under the License. #include "util.h" +#ifndef _MSC_VER #include +#endif #include #include #include diff --git a/src/include/zvec/ailego/buffer/buffer_manager.h b/src/include/zvec/ailego/buffer/buffer_manager.h index 8aebf759d..1e3496196 100644 --- a/src/include/zvec/ailego/buffer/buffer_manager.h +++ b/src/include/zvec/ailego/buffer/buffer_manager.h @@ -179,6 +179,8 @@ class BufferManager : public Singleton { uint64_t total_size_in_bytes() const; + void cleanup(); + ~BufferManager(); private: diff --git a/src/include/zvec/ailego/internal/platform.h b/src/include/zvec/ailego/internal/platform.h index 598ca97ad..ccd33971e 100644 --- a/src/include/zvec/ailego/internal/platform.h +++ b/src/include/zvec/ailego/internal/platform.h @@ -44,6 +44,39 @@ #endif #endif +//! MSVC platform compatibility (before extern "C" so available in C++) +#if defined(_MSC_VER) + +#if !defined(S_ISDIR) +#define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif + +#if !defined(_SSIZE_T_DEFINED) +#define _SSIZE_T_DEFINED +typedef intptr_t ssize_t; +#endif + +#if !defined(_ID_T_DEFINED) +#define _ID_T_DEFINED +typedef unsigned int id_t; +#endif + +#include +#define ailego_bswap16(x) _byteswap_ushort(x) +#define ailego_bswap32(x) _byteswap_ulong(x) +#define ailego_bswap64(x) _byteswap_uint64(x) + +#define strncasecmp _strnicmp + +#else // !_MSC_VER + +#define ailego_bswap16(x) __builtin_bswap16(x) +#define ailego_bswap32(x) __builtin_bswap32(x) +#define ailego_bswap64(x) __builtin_bswap64(x) + +#endif // _MSC_VER + + #if defined(__cplusplus) extern "C" { #endif @@ -104,11 +137,6 @@ extern "C" { #endif #endif -//! Add 'ssize_t' for MSVC -#if defined(_MSC_VER) -typedef intptr_t ssize_t; -#endif - #if defined(_MSC_VER) //! Returns the number of trailing 0-bits in x static inline int ailego_ctz32(uint32_t x) { @@ -174,7 +202,7 @@ static inline int ailego_clz64(uint64_t x) { #define ailego_likely(x) (x) #define ailego_unlikely(x) (x) #ifdef __SSE__ -#define ailego_prefetch(p) _mm_prefetch((p), 0) +#define ailego_prefetch(p) _mm_prefetch((const char *)(p), 0) #else #define ailego_prefetch(p) ((void)(p)) #endif diff --git a/src/include/zvec/ailego/io/file.h b/src/include/zvec/ailego/io/file.h index e884bdbf2..18c5f4922 100644 --- a/src/include/zvec/ailego/io/file.h +++ b/src/include/zvec/ailego/io/file.h @@ -14,6 +14,7 @@ #pragma once +#include #include namespace zvec { diff --git a/src/include/zvec/ailego/logger/logger.h b/src/include/zvec/ailego/logger/logger.h index 872d529f2..a7c90a9be 100644 --- a/src/include/zvec/ailego/logger/logger.h +++ b/src/include/zvec/ailego/logger/logger.h @@ -19,6 +19,14 @@ #include #include +// Define printf format attribute for GCC/Clang, empty for MSVC +#if defined(__GNUC__) || defined(__clang__) +#define AILEGO_PRINTF_FORMAT(fmt_idx, arg_idx) \ + __attribute__((format(printf, fmt_idx, arg_idx))) +#else +#define AILEGO_PRINTF_FORMAT(fmt_idx, arg_idx) +#endif + //! Register Index Logger #define FACTORY_REGISTER_LOGGER_ALIAS(__NAME__, __IMPL__, ...) \ AILEGO_FACTORY_REGISTER(__NAME__, zvec::ailego::Logger, __IMPL__, \ @@ -150,8 +158,9 @@ class LoggerBroker { } //! Log Message - __attribute__((format(printf, 4, 5))) static void Log( - int level, const char *file, int line, const char *format, ...) { + AILEGO_PRINTF_FORMAT(4, 5) + static void Log(int level, const char *file, int line, const char *format, + ...) { if (IsLevelEnabled(level)) { va_list args; va_start(args, format); diff --git a/src/include/zvec/ailego/utility/file_helper.h b/src/include/zvec/ailego/utility/file_helper.h index 00f4f5d0d..36b5fa940 100644 --- a/src/include/zvec/ailego/utility/file_helper.h +++ b/src/include/zvec/ailego/utility/file_helper.h @@ -16,6 +16,7 @@ #include #include +#include namespace zvec { namespace ailego { diff --git a/src/include/zvec/core/framework/index_logger.h b/src/include/zvec/core/framework/index_logger.h index 3263850ff..842c6e98f 100644 --- a/src/include/zvec/core/framework/index_logger.h +++ b/src/include/zvec/core/framework/index_logger.h @@ -143,8 +143,9 @@ class IndexLoggerBroker { } //! Log Message - __attribute__((format(printf, 4, 5))) static void Log( - int level, const char *file, int line, const char *format, ...) { + AILEGO_PRINTF_FORMAT(4, 5) + static void Log(int level, const char *file, int line, const char *format, + ...) { if (IsLevelEnabled(level)) { va_list args; va_start(args, format); diff --git a/src/include/zvec/db/schema.h b/src/include/zvec/db/schema.h index 09885c16a..bce2a1fd4 100644 --- a/src/include/zvec/db/schema.h +++ b/src/include/zvec/db/schema.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once +#include #include #include #include @@ -46,6 +47,9 @@ class FieldSchema { nullable_(nullable), dimension_(0), index_params_(index_params ? index_params->clone() : nullptr) {} + FieldSchema(const std::string &name, DataType type, bool nullable, + std::nullptr_t) + : FieldSchema(name, type, nullable, IndexParams::Ptr(nullptr)) {} FieldSchema(const std::string &name, DataType type, uint32_t dimension, bool nullable, const IndexParams::Ptr &index_params = nullptr) : name_(name), diff --git a/tests/ailego/buffer/buffer_manager_test.cc b/tests/ailego/buffer/buffer_manager_test.cc index 637c3ee9f..90c9f4e93 100644 --- a/tests/ailego/buffer/buffer_manager_test.cc +++ b/tests/ailego/buffer/buffer_manager_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -35,9 +36,7 @@ class BufferManagerTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); if (!File::MakePath(working_dir)) { LOG_ERROR("Failed to create working directory."); @@ -59,7 +58,10 @@ class BufferManagerTest : public testing::Test { BufferManager::Instance().init(4 * 1024 * 1024, 1); } - static void TearDownTestCase() {} + static void TearDownTestCase() { + BufferManager::Instance().cleanup(); + zvec::test_util::RemoveTestPath(working_dir); + } /***** Global initialization and cleanup - End *****/ ; }; diff --git a/tests/ailego/io/file_test.cc b/tests/ailego/io/file_test.cc index b71873616..84b916ce5 100644 --- a/tests/ailego/io/file_test.cc +++ b/tests/ailego/io/file_test.cc @@ -233,7 +233,7 @@ TEST(File, MemoryMap) { const char *file_path = "file_map_testing.tmp"; size_t file_size = 2u * 1024u * 1024u + 12u * 1024; size_t map_offset = MemoryHelper::PageSize() * 16; - size_t map_size = file_size - MemoryHelper::PageSize(); + size_t map_size = file_size - map_offset; File::Delete(file_path); EXPECT_FALSE(File::IsRegular(file_path)); diff --git a/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc b/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc index 7a536d282..f5e157c2c 100644 --- a/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc +++ b/tests/ailego/math/mips_euclidean_distance_matrix_int4_test.cc @@ -115,13 +115,13 @@ TEST(DistanceMatrix, GeneralRepeatedQuadraticInjection) { const uint32_t dim = (std::uniform_int_distribution(2, 128))(gen) * 2; const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim / 2); std::vector vec2(dim / 2); for (size_t d = 0; d < dim / 2; ++d) { - vec1[d] = dist(gen); - vec2[d] = dist(gen); + vec1[d] = static_cast(dist(gen)); + vec2[d] = static_cast(dist(gen)); } ASSERT_NEAR( ConvertAndComputeByMips(vec1.data(), vec2.data(), dim, m_val, e2), @@ -212,12 +212,12 @@ void TestSquaredEuclideanMatrixRepeatedQuadraticInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } float squared_l2_norm = 0.0f; for (size_t i = 0; i < matrix_size; i += dimension) { @@ -421,12 +421,12 @@ void MipsRepeatedQuadraticInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < block_size; ++i) { @@ -596,13 +596,13 @@ TEST(DistanceMatrix, GeneralSphericalInjection) { const uint32_t dim = (std::uniform_int_distribution(2, 128))(gen) * 2; const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim / 2); std::vector vec2(dim / 2); for (size_t d = 0; d < dim / 2; ++d) { - vec1[d] = dist(gen); - vec2[d] = dist(gen); + vec1[d] = static_cast(dist(gen)); + vec2[d] = static_cast(dist(gen)); } ASSERT_NEAR(ConvertAndComputeByMips(vec1.data(), vec2.data(), dim, e2), MipsSquaredEuclidean(vec1.data(), vec2.data(), dim, e2), @@ -687,12 +687,12 @@ void TestSquaredEuclideanMatrixSphericalInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } float squared_l2_norm = 0.0f; for (size_t i = 0; i < matrix_size; i += dimension) { @@ -895,12 +895,12 @@ void MipsSphericalInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(0, 255); + std::uniform_int_distribution dist(0, 255); for (size_t i = 0; i < matrix_size; ++i) { - matrix1[i] = dist(gen); + matrix1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < query_matrix_size; ++i) { - query1[i] = dist(gen); + query1[i] = static_cast(dist(gen)); } for (size_t i = 0; i < block_size; ++i) { diff --git a/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc b/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc index 8a5f7c5bc..36f49a480 100644 --- a/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc +++ b/tests/ailego/math/mips_euclidean_distance_matrix_int8_test.cc @@ -100,7 +100,7 @@ TEST(DistanceMatrix, GeneralRepeatedQuadraticInjection) { const float epsilon = 1e-6; const uint32_t dim = std::uniform_int_distribution(2, 128)(gen); const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim); std::vector vec2(dim); @@ -193,7 +193,7 @@ void TestSquaredEuclideanMatrixRepeatedQuadraticInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } @@ -402,7 +402,7 @@ void MipsRepeatedQuadraticInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } @@ -561,7 +561,7 @@ TEST(DistanceMatrix, GeneralSphericalInjection) { const float epsilon = 1e-6; const uint32_t dim = std::uniform_int_distribution(2, 128)(gen); const uint32_t count = std::uniform_int_distribution(1, 1000)(gen); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < count; ++i) { std::vector vec1(dim); std::vector vec2(dim); @@ -649,7 +649,7 @@ void TestSquaredEuclideanMatrixSphericalInjection(void) { std::vector result1(batch_size * query_size); std::vector result2(batch_size * query_size); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } @@ -853,7 +853,7 @@ void MipsSphericalInjectionBenchMark(void) { std::vector query2(query_matrix_size); std::mt19937 gen((std::random_device())()); - std::uniform_int_distribution dist(-127, 127); + std::uniform_int_distribution dist(-127, 127); for (size_t i = 0; i < matrix_size; ++i) { matrix1[i] = dist(gen); } diff --git a/tests/ailego/parallel/lock_test.cc b/tests/ailego/parallel/lock_test.cc index 2a98e8e94..b54875964 100644 --- a/tests/ailego/parallel/lock_test.cc +++ b/tests/ailego/parallel/lock_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include diff --git a/tests/ailego/parallel/multi_thread_list_test.cc b/tests/ailego/parallel/multi_thread_list_test.cc index 1d3be229f..17a896f1b 100644 --- a/tests/ailego/parallel/multi_thread_list_test.cc +++ b/tests/ailego/parallel/multi_thread_list_test.cc @@ -14,6 +14,7 @@ #include #include +#include #include #define private public @@ -68,8 +69,7 @@ TEST(MultiThreadListTest, General) { uint32_t num_of_producer = 100; uint32_t num_of_producer_done = 100; - std::vector consumer_results(num_of_consumer); - std::fill(consumer_results.begin(), consumer_results.end(), 0); + std::vector consumer_results(num_of_consumer, 0); for (uint32_t i = 0; i < num_of_consumer; i++) { consumer_pool.execute(consumer, i + 1, &consumer_results[i]); @@ -121,8 +121,8 @@ TEST(MultiThreadListTest, ConsumeStopResume) { ailego::ThreadPool producer_pool; ailego::ThreadPool consumer_pool; - uint32_t num_of_consumer = 100; - uint32_t num_of_producer = 100; + constexpr uint32_t num_of_consumer = 100; + constexpr uint32_t num_of_producer = 100; std::vector consumer_results(2 * num_of_consumer); std::fill(consumer_results.begin(), consumer_results.end(), 0); @@ -227,8 +227,7 @@ TEST(MultiThreadListTest, General_Moveable) { uint32_t num_of_producer = 100; uint32_t num_of_producer_done = 100; - std::vector consumer_results(num_of_consumer); - std::fill(consumer_results.begin(), consumer_results.end(), 0); + std::vector consumer_results(num_of_consumer, 0); for (uint32_t i = 0; i < num_of_consumer; i++) { consumer_pool.execute(consumer_moveable, i + 1, &consumer_results[i]); diff --git a/tests/ailego/utility/string_helper_test.cc b/tests/ailego/utility/string_helper_test.cc index 137600de3..705cd352c 100644 --- a/tests/ailego/utility/string_helper_test.cc +++ b/tests/ailego/utility/string_helper_test.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include using namespace zvec; diff --git a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc index d9197f1a3..9e0461960 100644 --- a/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc +++ b/tests/core/algorithm/cluster/opt_kmeans_cluster_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include #include "zvec/core/framework/index_framework.h" using namespace zvec::core; @@ -261,7 +262,7 @@ TEST(OptKmeansCluster, BinaryGeneral) { for (const auto &it : centroids) { const auto &vec = it.vector(); - uint mask = 0x1; + unsigned int mask = 0x1; std::cout << it.follows() << " (" << it.score() << ") { " << !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", " << !!(vec[0] & (mask << 2)) << ", ... , " @@ -277,7 +278,7 @@ TEST(OptKmeansCluster, BinaryGeneral) { for (const auto &it : centroids) { const auto &vec = it.vector(); - uint mask = 0x1; + unsigned int mask = 0x1; std::cout << it.follows() << " (" << it.score() << ") { " << !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", " << !!(vec[0] & (mask << 2)) << ", ... , " @@ -293,7 +294,7 @@ TEST(OptKmeansCluster, BinaryGeneral) { for (const auto &it : centroids) { const auto &vec = it.vector(); - uint mask = 0x1; + unsigned int mask = 0x1; std::cout << it.follows() << " (" << it.score() << ") { " << !!(vec[0] & mask) << ", " << !!(vec[0] & (mask << 1)) << ", " << !!(vec[0] & (mask << 2)) << ", ... , " @@ -330,7 +331,7 @@ TEST(OptKmeansCluster, IN4General) { std::random_device rd; std::mt19937 gen(rd()); - std::uniform_int_distribution dist(0, UINT8_MAX); + std::uniform_int_distribution dist(0, UINT8_MAX); for (uint32_t i = 0; i < count; ++i) { std::vector vec(dimension / 2); diff --git a/tests/core/algorithm/flat/flat_builder_test.cc b/tests/core/algorithm/flat/flat_builder_test.cc index ac3c3b2d7..1a447dfbc 100644 --- a/tests/core/algorithm/flat/flat_builder_test.cc +++ b/tests/core/algorithm/flat/flat_builder_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -43,7 +44,7 @@ class FlatBuilderTest : public testing::Test { static IndexMeta meta_; }; -std::string FlatBuilderTest ::dir_("flat_builder_test"); +std::string FlatBuilderTest ::dir_("flat_builder_test/"); IndexMeta FlatBuilderTest::meta_; void FlatBuilderTest::SetUp(void) { @@ -54,9 +55,7 @@ void FlatBuilderTest::SetUp(void) { //! self-check column-major and row-major search. void FlatBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void build_process(IndexBuilder::Pointer &builder, @@ -68,7 +67,7 @@ void build_process(IndexBuilder::Pointer &builder, auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - std::string path = FlatBuilderTest::dir_ + "/TestGeneral"; + std::string path = FlatBuilderTest::dir_ + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -128,7 +127,7 @@ TEST_F(FlatBuilderTest, TestBinaryInvalidColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; } TEST_F(FlatBuilderTest, TestBuildWithRowMajor) { @@ -138,7 +137,7 @@ TEST_F(FlatBuilderTest, TestBuildWithRowMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -166,7 +165,7 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithRowMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -195,7 +194,7 @@ TEST_F(FlatBuilderTest, TestBinaryBuildWithRowMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); @@ -223,7 +222,7 @@ TEST_F(FlatBuilderTest, TestBuildWithColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -252,7 +251,7 @@ TEST_F(FlatBuilderTest, TestInt8BuildWithColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); size_t doc_cnt = 128UL; @@ -280,7 +279,7 @@ TEST_F(FlatBuilderTest, TestBinaryBuildWithColumnMajor) { ASSERT_NE(builder, nullptr); Params params; ASSERT_EQ(0, builder->init(meta_, params)); - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); @@ -307,7 +306,7 @@ TEST_F(FlatBuilderTest, TestWithRowMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -332,7 +331,7 @@ TEST_F(FlatBuilderTest, TestInt8WithRowMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -358,7 +357,7 @@ TEST_F(FlatBuilderTest, TestBinaryWithRowMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); @@ -383,7 +382,7 @@ TEST_F(FlatBuilderTest, TestWithColumnMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(DIMENSION); @@ -409,7 +408,7 @@ TEST_F(FlatBuilderTest, TestInt8WithColumnMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); size_t doc_cnt = 128UL; @@ -434,7 +433,7 @@ TEST_F(FlatBuilderTest, TestBinaryWithColumnMajor) { IndexBuilder::Pointer builder = IndexFactory::CreateBuilder("FlatBuilder"); ASSERT_NE(builder, nullptr); Params params; - std::string path = dir_ + "/TestGeneral"; + std::string path = dir_ + "TestGeneral"; auto holder = std::make_shared>(dim); diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp index e9988692a..616dfe256 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -30,7 +31,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("streamer_test/"); +std::string FlatStreamerTest::dir_("flat_streamer_buffer_test_dir/"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -38,15 +39,11 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void FlatStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatStreamerTest, TestLinearSearch) { @@ -61,7 +58,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearch", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -79,7 +76,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); - + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); @@ -87,7 +84,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); @@ -183,7 +180,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchMMap", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchMMap", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -201,6 +198,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); ElapsedTime elapsed_time; IndexStreamer::Pointer read_streamer = @@ -209,7 +207,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchMMap", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchMMap", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); @@ -278,7 +276,7 @@ TEST_F(FlatStreamerTest, TestBufferStorage) { ASSERT_NE(nullptr, storage); Params stg_params; EXPECT_EQ(0, storage->init(stg_params)); - EXPECT_EQ(0, storage->open(dir_ + "/Test/LinearSearch", true)); + EXPECT_EQ(0, storage->open(dir_ + "Test/LinearSearch", true)); EXPECT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -295,6 +293,7 @@ TEST_F(FlatStreamerTest, TestBufferStorage) { } streamer->flush(0UL); streamer.reset(); + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); @@ -303,7 +302,7 @@ TEST_F(FlatStreamerTest, TestBufferStorage) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); EXPECT_EQ(0, read_storage->init(stg_params)); - EXPECT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearch", false)); + EXPECT_EQ(0, read_storage->open(dir_ + "Test/LinearSearch", false)); EXPECT_EQ(0, read_streamer->open(read_storage)); auto read_ctx = read_streamer->create_context(); auto provider = read_streamer->create_provider(); diff --git a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp index c919e9fee..5265878d8 100644 --- a/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp +++ b/tests/core/algorithm/flat/flat_streamer_buffer_time_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -30,7 +31,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("streamer_test/"); +std::string FlatStreamerTest::dir_("flat_streamer_buffer_time_test_dir/"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -38,15 +39,11 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void FlatStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatStreamerTest, TestLinearSearchMMap) { @@ -61,7 +58,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/LinearSearchMMap", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/LinearSearchMMap", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -79,6 +76,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); @@ -86,7 +84,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchMMap) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/LinearSearchMMap", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/LinearSearchMMap", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 30; ElapsedTime elapsed_time; diff --git a/tests/core/algorithm/flat/flat_streamer_test.cc b/tests/core/algorithm/flat/flat_streamer_test.cc index 1ddbd2d9b..cd8c6ff13 100644 --- a/tests/core/algorithm/flat/flat_streamer_test.cc +++ b/tests/core/algorithm/flat/flat_streamer_test.cc @@ -24,6 +24,7 @@ #include #include #include "algorithm/flat/flat_utility.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -47,7 +48,7 @@ class FlatStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatStreamerTest::dir_("streamer_test/"); +std::string FlatStreamerTest::dir_("flat_streamer_test_dir/"); std::shared_ptr FlatStreamerTest::index_meta_ptr_; void FlatStreamerTest::SetUp(void) { @@ -55,15 +56,11 @@ void FlatStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void FlatStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatStreamerTest, TestAddVector) { @@ -77,7 +74,7 @@ TEST_F(FlatStreamerTest, TestAddVector) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/AddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -112,7 +109,7 @@ TEST_F(FlatStreamerTest, TestLinearSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/AddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -186,7 +183,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -219,7 +216,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearcherSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearcherSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearcherSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -236,7 +233,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearcherSearch) { streamer->add_impl(i, vec.data(), qmeta, ctx); } - std::string path1 = dir_ + "/TestAddAndSearcherSearchDump"; + std::string path1 = dir_ + "TestAddAndSearcherSearchDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_EQ(0, dumper->init(Params())); ASSERT_EQ(0, dumper->create(path1)); @@ -277,7 +274,7 @@ TEST_F(FlatStreamerTest, TestLinearSearchRandomData) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearchRandomData", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearchRandomData", true)); ASSERT_EQ(0, streamer->init(meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -451,7 +448,7 @@ TEST_F(FlatStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -491,7 +488,7 @@ TEST_F(FlatStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -936,7 +933,7 @@ TEST_F(FlatStreamerTest, TestGroup) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -1044,7 +1041,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -1125,7 +1122,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) { Params write_stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(write_stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, write_streamer->init(*index_meta_ptr_, write_params)); ASSERT_EQ(0, write_streamer->open(write_storage)); auto ctx = write_streamer->create_context(); @@ -1148,7 +1145,8 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) { } write_streamer->flush(0UL); write_streamer->close(); - write_streamer.reset(); // + write_streamer.reset(); + write_storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatStreamer"); @@ -1157,7 +1155,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) { Params read_stg_params; auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, read_storage->init(read_stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, read_storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, read_streamer->init(*index_meta_ptr_, read_params)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc index af770255f..d50fc983b 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_builder_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -36,7 +37,7 @@ class FlatSparseBuilderTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string FlatSparseBuilderTest::_dir("FlatSparseBuilderTest"); +std::string FlatSparseBuilderTest::_dir("FlatSparseBuilderTest/"); shared_ptr FlatSparseBuilderTest::_index_meta_ptr; void FlatSparseBuilderTest::SetUp(void) { @@ -46,9 +47,7 @@ void FlatSparseBuilderTest::SetUp(void) { } void FlatSparseBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(_dir); } TEST_F(FlatSparseBuilderTest, TestGeneral) { @@ -84,7 +83,7 @@ TEST_F(FlatSparseBuilderTest, TestGeneral) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -188,7 +187,7 @@ TEST_F(FlatSparseBuilderTest, TestIndexThreads) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndexThreads"; + string path = _dir + "TestIndexThreads"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder1->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -245,7 +244,7 @@ TEST_F(FlatSparseBuilderTest, TestHalfFloatConverter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestHalFloatConverter"; + string path = _dir + "TestHalFloatConverter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc index f0553dd89..458de1c7c 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_searcher_test.cc @@ -20,6 +20,7 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" @@ -87,15 +88,11 @@ void FlatSparseSearcherTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void FlatSparseSearcherTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatSparseSearcherTest, TestGeneral) { @@ -104,7 +101,7 @@ TEST_F(FlatSparseSearcherTest, TestGeneral) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -165,7 +162,7 @@ TEST_F(FlatSparseSearcherTest, TestGeneral) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -215,7 +212,7 @@ TEST_F(FlatSparseSearcherTest, TestStreamerDump) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -236,7 +233,7 @@ TEST_F(FlatSparseSearcherTest, TestStreamerDump) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestStreamerDump"; + auto path = dir_ + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -301,7 +298,7 @@ TEST_F(FlatSparseSearcherTest, TestLoadClose) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -362,7 +359,7 @@ TEST_F(FlatSparseSearcherTest, TestLoadClose) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -420,7 +417,7 @@ TEST_F(FlatSparseSearcherTest, TestSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -442,7 +439,7 @@ TEST_F(FlatSparseSearcherTest, TestSearch) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -514,7 +511,7 @@ TEST_F(FlatSparseSearcherTest, TestSearchPKeys) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -542,7 +539,7 @@ TEST_F(FlatSparseSearcherTest, TestSearchPKeys) { } // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -727,7 +724,7 @@ TEST_F(FlatSparseSearcherTest, TestMultiThread) { ASSERT_EQ(2999, max); // test dump - auto path = dir_ + "/TestGeneral_dump"; + auto path = dir_ + "TestGeneral_dump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc index 11c84e8a5..b6bc546e0 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_buffer_test.cc @@ -22,6 +22,7 @@ #include #include #include +#include "tests/test_util.h" using namespace std; using namespace testing; @@ -86,15 +87,11 @@ void FlatSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void FlatSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatSparseStreamerTest, TestGeneral) { @@ -113,7 +110,7 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { ailego::Params stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/Test/FlatSparseSearch", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "Test/FlatSparseSearch", true)); ASSERT_EQ(0, write_streamer->init(index_meta, params)); ASSERT_EQ(0, write_streamer->open(write_storage)); @@ -137,6 +134,8 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + write_storage->close(); + write_storage.reset(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("FlatSparseStreamer"); @@ -144,7 +143,7 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/FlatSparseSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/FlatSparseSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); diff --git a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc index cad6a4d35..befd9aea0 100644 --- a/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc +++ b/tests/core/algorithm/flat_sparse/flat_sparse_streamer_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -48,7 +50,7 @@ class FlatSparseStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string FlatSparseStreamerTest::dir_("streamer_test/"); +std::string FlatSparseStreamerTest::dir_("flat_sparse_streamer_test_dir/"); std::shared_ptr FlatSparseStreamerTest::index_meta_ptr_; void FlatSparseStreamerTest::generate_sparse_data( @@ -88,15 +90,11 @@ void FlatSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void FlatSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(FlatSparseStreamerTest, TestGeneral) { @@ -105,7 +103,7 @@ TEST_F(FlatSparseStreamerTest, TestGeneral) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -175,7 +173,7 @@ TEST_F(FlatSparseStreamerTest, TestLinearSearch) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -249,7 +247,7 @@ TEST_F(FlatSparseStreamerTest, TestLinearSearchByKeys) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -374,7 +372,7 @@ TEST_F(FlatSparseStreamerTest, TestCreateIterator) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCreateIterator", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -573,7 +571,7 @@ TEST_F(FlatSparseStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -623,7 +621,7 @@ TEST_F(FlatSparseStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -1108,7 +1106,7 @@ TEST_F(FlatSparseStreamerTest, TestProvider) { } // dump - // auto path1 = dir_ + "/TestProvider"; + // auto path1 = dir_ + "TestProvider"; // auto dumper1 = IndexFactory::CreateDumper("FileDumper"); // ASSERT_NE(dumper1, nullptr); // ASSERT_EQ(0, dumper1->create(path1)); @@ -1211,7 +1209,7 @@ TEST_F(FlatSparseStreamerTest, TestParamsMaxDocCount) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -1292,7 +1290,7 @@ TEST_F(FlatSparseStreamerTest, TestParamsDataChunkSize) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_TRUE(storage != nullptr); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); // init streamer @@ -1484,7 +1482,7 @@ TEST_F(FlatSparseStreamerTest, TestGroupBy) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1559,7 +1557,7 @@ TEST_F(FlatSparseStreamerTest, TestGroupByNotEnoughNum) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1634,7 +1632,7 @@ TEST_F(FlatSparseStreamerTest, TestAddAndSearchWithID) { Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); diff --git a/tests/core/algorithm/hnsw/hnsw_builder_test.cc b/tests/core/algorithm/hnsw/hnsw_builder_test.cc index 402ae9720..fd151227f 100644 --- a/tests/core/algorithm/hnsw/hnsw_builder_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_builder_test.cc @@ -18,6 +18,7 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_framework.h" #if defined(__GNUC__) || defined(__GNUG__) @@ -42,7 +43,7 @@ class HnswBuilderTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswBuilderTest::_dir("hnswBuilderTest"); +std::string HnswBuilderTest::_dir("hnswBuilderTest/"); shared_ptr HnswBuilderTest::_index_meta_ptr; void HnswBuilderTest::SetUp(void) { @@ -52,9 +53,7 @@ void HnswBuilderTest::SetUp(void) { } void HnswBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(_dir); } TEST_F(HnswBuilderTest, TestGeneral) { @@ -83,7 +82,7 @@ TEST_F(HnswBuilderTest, TestGeneral) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -191,7 +190,7 @@ TEST_F(HnswBuilderTest, TestIndexThreads) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndexThreads"; + string path = _dir + "TestIndexThreads"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder1->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -243,7 +242,7 @@ TEST_F(HnswBuilderTest, TestCosine) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosine"; + string path = _dir + "TestCosine"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -327,7 +326,7 @@ TEST_F(HnswBuilderTest, TestCosineFp16Converter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosineFp16Converter"; + string path = _dir + "TestCosineFp16Converter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -409,7 +408,7 @@ TEST_F(HnswBuilderTest, TestCosineInt8Converter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosineInt8Converter"; + string path = _dir + "TestCosineInt8Converter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -491,7 +490,7 @@ TEST_F(HnswBuilderTest, TestCosineInt4Converter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestCosineInt4Converter"; + string path = _dir + "TestCosineInt4Converter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); diff --git a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp index d3a7004fa..cd1ab3970 100644 --- a/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_searcher_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_builder.h" #include "zvec/core/framework/index_factory.h" #include "zvec/core/framework/index_meta.h" @@ -58,9 +59,7 @@ void HnswSearcherTest::SetUp(void) { } void HnswSearcherTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(_dir); } TEST_F(HnswSearcherTest, TestRnnSearch) { @@ -82,7 +81,7 @@ TEST_F(HnswSearcherTest, TestRnnSearch) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRnnSearch"; + string path = _dir + "TestRnnSearch"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -148,7 +147,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchInnerProduct) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRnnSearchInnerProduct"; + string path = _dir + "TestRnnSearchInnerProduct"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -230,7 +229,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchCosine) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRnnSearchCosine"; + string path = _dir + "TestRnnSearchCosine"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -301,7 +300,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchMipsSquaredEuclidean) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -345,7 +344,7 @@ TEST_F(HnswSearcherTest, TestRnnSearchMipsSquaredEuclidean) { ASSERT_LT(-radius, results[topk - 1].score()); } - auto path = _dir + "/TestStreamerDump"; + auto path = _dir + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -416,7 +415,7 @@ TEST_F(HnswSearcherTest, TestGeneral) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -561,7 +560,7 @@ TEST_F(HnswSearcherTest, TestClearAndReload) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -659,7 +658,7 @@ TEST_F(HnswSearcherTest, TestFilter) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -759,7 +758,7 @@ TEST_F(HnswSearcherTest, TestStreamerDump) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(*_index_meta_ptr, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -774,7 +773,7 @@ TEST_F(HnswSearcherTest, TestStreamerDump) { } streamer->add_impl(i, vec.data(), qmeta, ctx); } - auto path = _dir + "/TestStreamerDump"; + auto path = _dir + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -887,9 +886,9 @@ TEST_F(HnswSearcherTest, TestSharedContext) { size_t docs1 = rand() % 500 + 100; size_t docs2 = rand() % 5000 + 100; size_t docs3 = rand() % 50000 + 100; - auto path1 = _dir + "/TestSharedContext.index1"; - auto path2 = _dir + "/TestSharedContext.index2"; - auto path3 = _dir + "/TestSharedContext.index3"; + auto path1 = _dir + "TestSharedContext.index1"; + auto path2 = _dir + "TestSharedContext.index2"; + auto path3 = _dir + "TestSharedContext.index3"; auto searcher1 = gen_index(0, docs1, path1); auto searcher2 = gen_index(1, docs2, path2); auto searcher3 = gen_index(2, docs3, path3); @@ -957,7 +956,7 @@ TEST_F(HnswSearcherTest, TestSharedContext) { IndexFactory::CreateStreamer("HnswStreamer"); auto storage = IndexFactory::CreateStorage("MMapFileStorage"); storage->init(ailego::Params()); - storage->open(_dir + "/TestSharedContext.index4", true); + storage->open(_dir + "TestSharedContext.index4", true); streamer->init(*_index_meta_ptr, ailego::Params()); streamer->open(storage); NumericalVector query(dim); @@ -1011,7 +1010,7 @@ TEST_F(HnswSearcherTest, TestProvider) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestProvider"; + string path = _dir + "TestProvider"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1077,7 +1076,7 @@ TEST_F(HnswSearcherTest, TestMipsEuclideanMetric) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestMipsEuclideanMetric"; + string path = _dir + "TestMipsEuclideanMetric"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1132,7 +1131,7 @@ TEST_F(HnswSearcherTest, TestRandomPaddingTopk) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestRandomPadding"; + string path = _dir + "TestRandomPadding"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1200,7 +1199,7 @@ TEST_F(HnswSearcherTest, TestBruteForceSetupInContext) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1337,7 +1336,7 @@ TEST_F(HnswSearcherTest, TestCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestCosine.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1365,7 +1364,7 @@ TEST_F(HnswSearcherTest, TestCosine) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestCosine"; + auto path = _dir + "TestCosine"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1474,7 +1473,7 @@ TEST_F(HnswSearcherTest, TestFetchVector) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVector.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestFetchVector.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1492,7 +1491,7 @@ TEST_F(HnswSearcherTest, TestFetchVector) { streamer->add_impl(i, vec.data(), qmeta, ctx); } - auto path = _dir + "/TestFetchVector"; + auto path = _dir + "TestFetchVector"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1594,7 +1593,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosine.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1624,7 +1623,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosine) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestFetchVectorCosine"; + auto path = _dir + "TestFetchVectorCosine"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1752,7 +1751,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineHalfFloatConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ( - 0, storage->open(_dir + "/TestFetchVectorCosineHalfFloatConverter.index", + 0, storage->open(_dir + "TestFetchVectorCosineHalfFloatConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1787,7 +1786,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineHalfFloatConverter) { vecs.push_back(vec); } - auto path = _dir + "/TestFetchVectorCosineHalfFloatConverter"; + auto path = _dir + "TestFetchVectorCosineHalfFloatConverter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1916,7 +1915,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineFp16Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosineFp16Converter.index", + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosineFp16Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1950,7 +1949,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineFp16Converter) { vecs.push_back(vec); } - auto path = _dir + "/TestFetchVectorCosineFp16Converter"; + auto path = _dir + "TestFetchVectorCosineFp16Converter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -2071,7 +2070,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt8Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosineInt8Converter.index", + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosineInt8Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2102,7 +2101,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt8Converter) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestFetchVectorCosineInt8Converter"; + auto path = _dir + "TestFetchVectorCosineInt8Converter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -2229,7 +2228,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt4Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestFetchVectorCosineInt4Converter.index", + ASSERT_EQ(0, storage->open(_dir + "TestFetchVectorCosineInt4Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2260,7 +2259,7 @@ TEST_F(HnswSearcherTest, TestFetchVectorCosineInt4Converter) { ASSERT_EQ(0, streamer->add_impl(i, new_vec.data(), new_meta, ctx)); } - auto path = _dir + "/TestFetchVectorCosineInt4Converter"; + auto path = _dir + "TestFetchVectorCosineInt4Converter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -2375,7 +2374,7 @@ TEST_F(HnswSearcherTest, TestGroup) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGroup"; + string path = _dir + "TestGroup"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -2502,7 +2501,7 @@ TEST_F(HnswSearcherTest, TestGroupNotEnoughNum) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGroupNotEnoughNum"; + string path = _dir + "TestGroupNotEnoughNum"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -2591,7 +2590,7 @@ TEST_F(HnswSearcherTest, TestGroupInBruteforceSearch) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGroupInBruteforceSearch"; + string path = _dir + "TestGroupInBruteforceSearch"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -2696,7 +2695,7 @@ TEST_F(HnswSearcherTest, TestBinaryConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(_dir + "/TestBinaryConverter.index", true)); + ASSERT_EQ(0, storage->open(_dir + "TestBinaryConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2727,7 +2726,7 @@ TEST_F(HnswSearcherTest, TestBinaryConverter) { vecs.push_back(vec); } - auto path = _dir + "/TestBinaryConverter"; + auto path = _dir + "TestBinaryConverter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp index bd96789a4..94e35c938 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw/hnsw_streamer_buffer_test.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "tests/test_util.h" using namespace zvec::core; using namespace zvec::ailego; @@ -31,7 +32,7 @@ class HnswStreamerTest : public testing::Test { static std::shared_ptr index_meta_ptr_; }; -std::string HnswStreamerTest::dir_("streamer_test/"); +std::string HnswStreamerTest::dir_("hnsw_streamer_buffer_test_dir/"); std::shared_ptr HnswStreamerTest::index_meta_ptr_; void HnswStreamerTest::SetUp(void) { @@ -39,15 +40,11 @@ void HnswStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void HnswStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswStreamerTest, TestHnswSearch) { @@ -64,7 +61,7 @@ TEST_F(HnswStreamerTest, TestHnswSearch) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/HnswSearch", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/HnswSearch", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -82,6 +79,7 @@ TEST_F(HnswStreamerTest, TestHnswSearch) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("HnswStreamer"); @@ -89,7 +87,7 @@ TEST_F(HnswStreamerTest, TestHnswSearch) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); @@ -186,7 +184,7 @@ TEST_F(HnswStreamerTest, TestHnswSearchMMap) { ASSERT_NE(nullptr, storage); Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/HnswSearchMMap", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/HnswSearchMMap", true)); ASSERT_EQ(0, write_streamer->open(storage)); auto ctx = write_streamer->create_context(); @@ -204,6 +202,7 @@ TEST_F(HnswStreamerTest, TestHnswSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + storage->close(); ElapsedTime elapsed_time; IndexStreamer::Pointer read_streamer = @@ -212,7 +211,7 @@ TEST_F(HnswStreamerTest, TestHnswSearchMMap) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSearchMMap", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSearchMMap", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); size_t topk = 3; auto provider = read_streamer->create_provider(); diff --git a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc index c04f67124..9a3a09d8e 100644 --- a/tests/core/algorithm/hnsw/hnsw_streamer_test.cc +++ b/tests/core/algorithm/hnsw/hnsw_streamer_test.cc @@ -14,12 +14,17 @@ #include "hnsw_streamer.h" #include #include +#include +#ifndef _MSC_VER #include +#include +#endif #include #include #include #include #include +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -44,7 +49,7 @@ class HnswStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswStreamerTest::dir_("streamer_test/"); +std::string HnswStreamerTest::dir_("hnsw_streamer_test_dir/"); shared_ptr HnswStreamerTest::index_meta_ptr_; void HnswStreamerTest::SetUp(void) { @@ -52,15 +57,11 @@ void HnswStreamerTest::SetUp(void) { IndexMeta(IndexMeta::DataType::DT_FP32, dim)); index_meta_ptr_->set_metric("SquaredEuclidean", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void HnswStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswStreamerTest, TestAddVector) { @@ -77,7 +78,7 @@ TEST_F(HnswStreamerTest, TestAddVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/Test/AddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "Test/AddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -109,7 +110,7 @@ TEST_F(HnswStreamerTest, TestLinearSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -179,7 +180,7 @@ TEST_F(HnswStreamerTest, TestLinearSearchByKeys) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -295,7 +296,7 @@ TEST_F(HnswStreamerTest, TestKnnSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -379,7 +380,7 @@ TEST_F(HnswStreamerTest, TestAddAndSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -470,7 +471,7 @@ TEST_F(HnswStreamerTest, TestKnnSearchRandomData) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearchRandomData", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearchRandomData", true)); ASSERT_EQ(0, streamer->init(meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -644,7 +645,7 @@ TEST_F(HnswStreamerTest, TestCreateIterator) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCreateIterator", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -717,7 +718,7 @@ TEST_F(HnswStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -758,7 +759,7 @@ TEST_F(HnswStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -1434,7 +1435,7 @@ TEST_F(HnswStreamerTest, TestCheckStats) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - std::string path = dir_ + "/TestCheckStats.index"; + std::string path = dir_ + "TestCheckStats.index"; ASSERT_EQ(0, storage->open(path, true)); ailego::Params params; params.set(PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT, 100); @@ -1530,7 +1531,7 @@ TEST_F(HnswStreamerTest, TestCheckStats) { ASSERT_EQ(createTime3, createTime1); ASSERT_GT(updateTime3, updateTime2); - auto dpath = dir_ + "/dumpIndex"; + auto dpath = dir_ + "dumpIndex"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(dpath)); @@ -1639,10 +1640,12 @@ TEST_F(HnswStreamerTest, TestDumpIndexAndAdd) { IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, dim); ASSERT_NE(nullptr, ctx); int code = 0; - std::mutex mutex; - auto addVector = [&](int a, int b) { + std::atomic async_started{false}; + auto addVector = [&](int a, int b, bool signal_start) { int success = 0; - mutex.unlock(); + if (signal_start) { + async_started.store(true, std::memory_order_release); + } for (int i = a; i < b; i++) { for (size_t j = 0; j < dim; ++j) { vec[j] = i; @@ -1659,18 +1662,17 @@ TEST_F(HnswStreamerTest, TestDumpIndexAndAdd) { } std::cout << "addVector: " << success << " success" << std::endl; }; - mutex.lock(); - addVector(0, 2000); - mutex.lock(); - auto t2 = std::async(std::launch::async, addVector, 2000, 3000); - auto path1 = dir_ + "/dumpIndex1"; + addVector(0, 2000, false); + auto t2 = std::async(std::launch::async, addVector, 2000, 3000, true); + auto path1 = dir_ + "dumpIndex1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); - mutex.lock(); // sync: wait addVector start and release lock + while (!async_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } auto test_dumper = std::make_shared(); ASSERT_EQ(0, streamer->dump(test_dumper)); - mutex.unlock(); ASSERT_EQ(0, streamer->dump(dumper1)); ASSERT_EQ(0, dumper1->close()); t2.get(); @@ -1768,7 +1770,7 @@ TEST_F(HnswStreamerTest, TestProvider) { streamer->add_impl(keys[i], vec.data(), qmeta, ctx); } - auto path1 = dir_ + "/TestGetVector1"; + auto path1 = dir_ + "TestGetVector1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); @@ -2003,7 +2005,7 @@ TEST_F(HnswStreamerTest, TestBruteForceSetupInContext) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ(0, - storage->open(dir_ + "/TestBruteForceSetupInContext.index", true)); + storage->open(dir_ + "TestBruteForceSetupInContext.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2136,7 +2138,7 @@ TEST_F(HnswStreamerTest, TestKnnSearchCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestKnnSearchCosine.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestKnnSearchCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2252,7 +2254,7 @@ TEST_F(HnswStreamerTest, TestFetchVector) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVector.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVector.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2349,7 +2351,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosine.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2478,7 +2480,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineHalfFloatConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ( - 0, storage->open(dir_ + "/TestFetchVectorCosineHalfFloatConverter.index", + 0, storage->open(dir_ + "TestFetchVectorCosineHalfFloatConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2618,7 +2620,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineFp16Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosineFp16Converter.index", + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosineFp16Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2752,7 +2754,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt8Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosineInt8Converter.index", + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosineInt8Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2883,7 +2885,7 @@ TEST_F(HnswStreamerTest, TestFetchVectorCosineInt4Converter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestFetchVectorCosineInt4Converter.index", + ASSERT_EQ(0, storage->open(dir_ + "TestFetchVectorCosineInt4Converter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2998,7 +3000,7 @@ TEST_F(HnswStreamerTest, TestRnnSearch) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestRnnSearchInnerProduct.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestRnnSearchInnerProduct.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3061,7 +3063,7 @@ TEST_F(HnswStreamerTest, TestRnnSearchInnerProduct) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestRnnSearchInnerProduct.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestRnnSearchInnerProduct.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3138,7 +3140,7 @@ TEST_F(HnswStreamerTest, TestRnnSearchCosine) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestRnnSearchCosine.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestRnnSearchCosine.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3213,7 +3215,7 @@ TEST_F(HnswStreamerTest, TestGroup) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroup.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroup.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3328,7 +3330,7 @@ TEST_F(HnswStreamerTest, TestGroupNotEnoughNum) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGroupNotEnoughNum.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroupNotEnoughNum.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3406,8 +3408,7 @@ TEST_F(HnswStreamerTest, TestGroupInBruteforceSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, - storage->open(dir_ + "/TestGroupInBruteforceSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGroupInBruteforceSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3502,7 +3503,7 @@ TEST_F(HnswStreamerTest, TestBinaryConverter) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestBinaryConverter.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestBinaryConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3567,7 +3568,7 @@ TEST_F(HnswStreamerTest, TestAddAndSearchWithID) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddAndSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddAndSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -3702,7 +3703,7 @@ TEST_F(HnswStreamerTest, TestBasicRefiner) { ailego::Params base_stg_params; auto base_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, base_storage->init(base_stg_params)); - ASSERT_EQ(0, base_storage->open(dir_ + "/TestBasicRefinerBase.index", true)); + ASSERT_EQ(0, base_storage->open(dir_ + "TestBasicRefinerBase.index", true)); ASSERT_EQ(0, base_streamer->init(index_meta_binary, params)); ASSERT_EQ(0, base_streamer->open(base_storage)); @@ -3714,7 +3715,7 @@ TEST_F(HnswStreamerTest, TestBasicRefiner) { auto refine_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, refine_storage->init(refine_stg_params)); ASSERT_EQ(0, - refine_storage->open(dir_ + "/TestBasicRefinerRefine.index", true)); + refine_storage->open(dir_ + "TestBasicRefinerRefine.index", true)); ASSERT_EQ(0, refine_streamer->init(index_meta, params)); ASSERT_EQ(0, refine_streamer->open(refine_storage)); auto refine_ctx = refine_streamer->create_context(); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc index 0f46dab99..0e35dacd2 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_builder_test.cc @@ -18,6 +18,7 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_framework.h" #include "hnsw_sparse_params.h" @@ -42,7 +43,7 @@ class HnswSparseBuilderTest : public testing::Test { static shared_ptr _index_meta_ptr; }; -std::string HnswSparseBuilderTest::_dir("HnswSparseBuilderTest"); +std::string HnswSparseBuilderTest::_dir("HnswSparseBuilderTest/"); shared_ptr HnswSparseBuilderTest::_index_meta_ptr; void HnswSparseBuilderTest::SetUp(void) { @@ -52,9 +53,7 @@ void HnswSparseBuilderTest::SetUp(void) { } void HnswSparseBuilderTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", _dir.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(_dir); } TEST_F(HnswSparseBuilderTest, TestGeneral) { @@ -92,7 +91,7 @@ TEST_F(HnswSparseBuilderTest, TestGeneral) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestGeneral"; + string path = _dir + "TestGeneral"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -230,7 +229,7 @@ TEST_F(HnswSparseBuilderTest, TestIndexThreads) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndexThreads"; + string path = _dir + "TestIndexThreads"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder1->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -289,7 +288,7 @@ TEST_F(HnswSparseBuilderTest, TestHalfFloatConverter) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestHalFloatConverter"; + string path = _dir + "TestHalFloatConverter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -385,7 +384,7 @@ TEST_F(HnswSparseBuilderTest, TestIndptr) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndptr"; + string path = _dir + "TestIndptr"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -447,7 +446,7 @@ TEST_F(HnswSparseBuilderTest, TestIndptrFp16) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = _dir + "/TestIndptrFp16"; + string path = _dir + "TestIndptrFp16"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp index dcbff13e2..8196e736e 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_searcher_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/core/framework/index_framework.h" #include "hnsw_sparse_params.h" @@ -90,9 +91,7 @@ void HnswSparseSearcherTest::SetUp(void) { } void HnswSparseSearcherTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswSparseSearcherTest, TestGeneral) { @@ -116,7 +115,7 @@ TEST_F(HnswSparseSearcherTest, TestGeneral) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -139,7 +138,7 @@ TEST_F(HnswSparseSearcherTest, TestGeneral) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestGeneral"; + auto path = dir_ + "TestGeneral"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -249,7 +248,7 @@ TEST_F(HnswSparseSearcherTest, TestRnnSearch) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestRnnSearch"; + string path = dir_ + "TestRnnSearch"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -334,7 +333,7 @@ TEST_F(HnswSparseSearcherTest, TestClearAndReload) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestClearAndReload"; + string path = dir_ + "TestClearAndReload"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -469,7 +468,7 @@ TEST_F(HnswSparseSearcherTest, TestFilter) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestFilter"; + string path = dir_ + "TestFilter"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -621,7 +620,7 @@ TEST_F(HnswSparseSearcherTest, TestBatchQuery) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestBatchQuery"; + string path = dir_ + "TestBatchQuery"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -756,7 +755,7 @@ TEST_F(HnswSparseSearcherTest, TestStreamerDump) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestStreamerDump.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestStreamerDump.index", true)); ASSERT_EQ(0, streamer->init(*_index_meta_ptr, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -778,7 +777,7 @@ TEST_F(HnswSparseSearcherTest, TestStreamerDump) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestStreamerDump"; + auto path = dir_ + "TestStreamerDump"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -913,9 +912,9 @@ TEST_F(HnswSparseSearcherTest, TestSharedContext) { size_t docs1 = rand() % 500 + 100; size_t docs2 = rand() % 5000 + 100; size_t docs3 = rand() % 50000 + 100; - auto path1 = dir_ + "/TestSharedContext.index1"; - auto path2 = dir_ + "/TestSharedContext.index2"; - auto path3 = dir_ + "/TestSharedContext.index3"; + auto path1 = dir_ + "TestSharedContext.index1"; + auto path2 = dir_ + "TestSharedContext.index2"; + auto path3 = dir_ + "TestSharedContext.index3"; auto searcher1 = gen_index(0, docs1, path1); auto searcher2 = gen_index(1, docs2, path2); auto searcher3 = gen_index(2, docs3, path3); @@ -1000,7 +999,7 @@ TEST_F(HnswSparseSearcherTest, TestSharedContext) { IndexFactory::CreateStreamer("HnswSparseStreamer"); auto storage = IndexFactory::CreateStorage("MMapFileStorage"); storage->init(ailego::Params()); - storage->open(dir_ + "/TestSharedContext.index4", true); + storage->open(dir_ + "TestSharedContext.index4", true); streamer->init(*_index_meta_ptr, ailego::Params()); streamer->open(storage); @@ -1075,7 +1074,7 @@ TEST_F(HnswSparseSearcherTest, TestProvider) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestProvider"; + string path = dir_ + "TestProvider"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1162,7 +1161,7 @@ TEST_F(HnswSparseSearcherTest, TestRandomPaddingTopk) { auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestRandomPadding"; + string path = dir_ + "TestRandomPadding"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1249,7 +1248,7 @@ TEST_F(HnswSparseSearcherTest, TestBruteForceSetupInContext) { ASSERT_EQ(0, builder->build(holder)); auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); - string path = dir_ + "/TestBruteForceSetupInContext"; + string path = dir_ + "TestBruteForceSetupInContext"; ASSERT_EQ(0, dumper->create(path)); ASSERT_EQ(0, builder->dump(dumper)); ASSERT_EQ(0, dumper->close()); @@ -1395,7 +1394,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatConverter) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestHalfFloatConverter.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestHalfFloatConverter.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1426,7 +1425,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatConverter) { new_vec.data(), new_meta, ctx)); } - auto path = dir_ + "/TestHalfFloatConverter"; + auto path = dir_ + "TestHalfFloatConverter"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1532,7 +1531,7 @@ TEST_F(HnswSparseSearcherTest, TestQueryFilteringRatio) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestQueryFilteringRatio.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestQueryFilteringRatio.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1555,7 +1554,7 @@ TEST_F(HnswSparseSearcherTest, TestQueryFilteringRatio) { sparse_vec_list[i].data(), qmeta, ctx)); } - auto path = dir_ + "/TestQueryFilteringRatio"; + auto path = dir_ + "TestQueryFilteringRatio"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); @@ -1672,7 +1671,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatRevert) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestHalfFloatRevert.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestHalfFloatRevert.index", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -1730,7 +1729,7 @@ TEST_F(HnswSparseSearcherTest, TestHalfFloatRevert) { } } - auto path = dir_ + "/TestHalfFloatRevert"; + auto path = dir_ + "TestHalfFloatRevert"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(path)); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp index 8c7aadb24..32dfb55b2 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_buffer_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include "tests/test_util.h" #include "hnsw_sparse_streamer.h" using namespace std; @@ -47,7 +48,8 @@ class HnswSparseStreamerTest : public testing::Test { static shared_ptr index_meta_ptr_; }; -std::string HnswSparseStreamerTest::dir_("HnswSparseStreamerTest/"); +std::string HnswSparseStreamerTest::dir_( + "hnsw_sparse_streamer_buffer_test_dir/"); shared_ptr HnswSparseStreamerTest::index_meta_ptr_; void HnswSparseStreamerTest::generate_sparse_data( @@ -85,15 +87,11 @@ void HnswSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void HnswSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswSparseStreamerTest, TestGeneral) { @@ -118,7 +116,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { ailego::Params stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/Test/HnswSparseSearch", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "Test/HnswSparseSearch", true)); ASSERT_EQ(0, write_streamer->init(index_meta, params)); ASSERT_EQ(0, write_streamer->open(write_storage)); @@ -142,6 +140,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + write_storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("HnswSparseStreamer"); @@ -149,7 +148,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { auto read_storage = IndexFactory::CreateStorage("BufferStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSparseSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSparseSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); @@ -246,7 +245,7 @@ TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) { ailego::Params stg_params; auto write_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, write_storage->init(stg_params)); - ASSERT_EQ(0, write_storage->open(dir_ + "/Test/HnswSparseSearch", true)); + ASSERT_EQ(0, write_storage->open(dir_ + "Test/HnswSparseSearch", true)); ASSERT_EQ(0, write_streamer->init(index_meta, params)); ASSERT_EQ(0, write_streamer->open(write_storage)); @@ -270,6 +269,7 @@ TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) { write_streamer->flush(0UL); write_streamer->close(); write_streamer.reset(); + write_storage->close(); IndexStreamer::Pointer read_streamer = IndexFactory::CreateStreamer("HnswSparseStreamer"); @@ -277,7 +277,7 @@ TEST_F(HnswSparseStreamerTest, TestHnswSearchMMap) { auto read_storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, read_storage); ASSERT_EQ(0, read_storage->init(stg_params)); - ASSERT_EQ(0, read_storage->open(dir_ + "/Test/HnswSparseSearch", false)); + ASSERT_EQ(0, read_storage->open(dir_ + "Test/HnswSparseSearch", false)); ASSERT_EQ(0, read_streamer->open(read_storage)); auto linearCtx = read_streamer->create_context(); diff --git a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc index 9192fdb12..623ac708a 100644 --- a/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc +++ b/tests/core/algorithm/hnsw_sparse/hnsw_sparse_streamer_test.cc @@ -14,13 +14,18 @@ #include "hnsw_sparse_streamer.h" #include #include +#include +#ifndef _MSC_VER #include +#include +#endif #include #include #include #include #include #include +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -87,15 +92,11 @@ void HnswSparseStreamerTest::SetUp(void) { IndexMeta::DataType::DT_FP32)); index_meta_ptr_->set_metric("InnerProductSparse", 0, ailego::Params()); - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } void HnswSparseStreamerTest::TearDown(void) { - char cmdBuf[100]; - snprintf(cmdBuf, 100, "rm -rf %s", dir_.c_str()); - system(cmdBuf); + zvec::test_util::RemoveTestPath(dir_); } TEST_F(HnswSparseStreamerTest, TestGeneral) { @@ -119,7 +120,7 @@ TEST_F(HnswSparseStreamerTest, TestGeneral) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -229,7 +230,7 @@ TEST_F(HnswSparseStreamerTest, TestAddVector) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestAddVector", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestAddVector", true)); ASSERT_EQ(0, streamer->open(storage)); auto ctx = streamer->create_context(); @@ -265,7 +266,7 @@ TEST_F(HnswSparseStreamerTest, TestLinearSearch) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearch.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearch.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -345,7 +346,7 @@ TEST_F(HnswSparseStreamerTest, TestLinearSearchByKeys) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestLinearSearchByKeys.index", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestLinearSearchByKeys.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -574,7 +575,7 @@ TEST_F(HnswSparseStreamerTest, TestCreateIterator) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestCreateIterator", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestCreateIterator", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -662,7 +663,7 @@ TEST_F(HnswSparseStreamerTest, TestForceFlush) { stg_params.set("proxima.mmap_file.storage.copy_on_write", true); stg_params.set("proxima.mmap_file.storage.force_flush", true); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -713,7 +714,7 @@ TEST_F(HnswSparseStreamerTest, TestForceFlush) { storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_NE(nullptr, storage); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestForceFlush", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestForceFlush", true)); ASSERT_EQ(0, streamer->open(storage)); checkIter(cnt, streamer); @@ -1480,7 +1481,7 @@ TEST_F(HnswSparseStreamerTest, TestCheckStats) { ASSERT_NE(nullptr, storage); ailego::Params stg_params; ASSERT_EQ(0, storage->init(stg_params)); - std::string path = dir_ + "/TestCheckStats.index"; + std::string path = dir_ + "TestCheckStats.index"; ASSERT_EQ(0, storage->open(path, true)); ailego::Params params; params.set(PARAM_HNSW_SPARSE_STREAMER_MAX_NEIGHBOR_COUNT, 100); @@ -1594,7 +1595,7 @@ TEST_F(HnswSparseStreamerTest, TestCheckStats) { ASSERT_EQ(createTime3, createTime1); ASSERT_GT(updateTime3, updateTime2); - auto dpath = dir_ + "/dumpIndex"; + auto dpath = dir_ + "dumpIndex"; auto dumper = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper, nullptr); ASSERT_EQ(0, dumper->create(dpath)); @@ -1725,10 +1726,12 @@ TEST_F(HnswSparseStreamerTest, TestDumpIndexAndAdd) { IndexMeta::DataType::DT_FP32); ASSERT_NE(nullptr, ctx); int code = 0; - std::mutex mutex; + std::atomic async_started{false}; - auto addVector = [&](int a, int b) { - mutex.unlock(); + auto addVector = [&](int a, int b, bool signal_start) { + if (signal_start) { + async_started.store(true, std::memory_order_release); + } for (int i = a; i < b; i++) { NumericalVector sparse_indices(sparse_dim_count); NumericalVector sparse_velues(sparse_dim_count); @@ -1749,18 +1752,17 @@ TEST_F(HnswSparseStreamerTest, TestDumpIndexAndAdd) { } }; - mutex.lock(); - addVector(0, 2000); - mutex.lock(); - auto t2 = std::async(std::launch::async, addVector, 2000, 3000); - auto path1 = dir_ + "/dumpIndex1"; + addVector(0, 2000, false); + auto t2 = std::async(std::launch::async, addVector, 2000, 3000, true); + auto path1 = dir_ + "dumpIndex1"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); - mutex.lock(); // sync: wait addVector start and release lock + while (!async_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } auto test_dumper = std::make_shared(); ASSERT_EQ(0, streamer->dump(test_dumper)); - mutex.unlock(); ASSERT_EQ(0, streamer->dump(dumper1)); ASSERT_EQ(0, dumper1->close()); t2.get(); @@ -1877,7 +1879,7 @@ TEST_F(HnswSparseStreamerTest, TestProvider) { sparse_velues.data(), qmeta, ctx)); } - auto path1 = dir_ + "/TestProvider"; + auto path1 = dir_ + "TestProvider"; auto dumper1 = IndexFactory::CreateDumper("FileDumper"); ASSERT_NE(dumper1, nullptr); ASSERT_EQ(0, dumper1->create(path1)); @@ -2092,7 +2094,7 @@ TEST_F(HnswSparseStreamerTest, TestBruteForceSetupInContext) { auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); ASSERT_EQ(0, - storage->open(dir_ + "/TestBruteForceSetupInContext.index", true)); + storage->open(dir_ + "TestBruteForceSetupInContext.index", true)); ASSERT_EQ(0, streamer->init(*index_meta_ptr_, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2232,7 +2234,7 @@ TEST_F(HnswSparseStreamerTest, TestQueryFilteringRatio) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestQueryFilteringRatio", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestQueryFilteringRatio", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); @@ -2348,7 +2350,7 @@ TEST_F(HnswSparseStreamerTest, TestAddAndSearchWithID) { ailego::Params stg_params; auto storage = IndexFactory::CreateStorage("MMapFileStorage"); ASSERT_EQ(0, storage->init(stg_params)); - ASSERT_EQ(0, storage->open(dir_ + "/TestGeneral", true)); + ASSERT_EQ(0, storage->open(dir_ + "TestGeneral", true)); ASSERT_EQ(0, streamer->init(index_meta, params)); ASSERT_EQ(0, streamer->open(storage)); diff --git a/tests/core/algorithm/ivf/ivf_searcher_test.cc b/tests/core/algorithm/ivf/ivf_searcher_test.cc index 9911e0e2e..282bde9d2 100644 --- a/tests/core/algorithm/ivf/ivf_searcher_test.cc +++ b/tests/core/algorithm/ivf/ivf_searcher_test.cc @@ -392,7 +392,7 @@ TEST_F(IVFSearcherTest, TestSimpleCosine) { { size_t topk = 33; context->set_topk(topk); - + std::string new_vec; IndexQueryMeta new_meta; ASSERT_EQ(0, reformer->convert(query.data(), qmeta, &new_vec, &new_meta)); @@ -3001,7 +3001,7 @@ TEST_F(IVFSearcherTest, TestProvider) { } else { std::iota(keys.begin(), keys.end(), 0U); std::transform(keys.begin(), keys.end(), keys.begin(), - [&](key_t k) { return step * k; }); + [&](uint64_t k) { return step * k; }); if (rand_order) { uint32_t seed = Realtime::Seconds(); std::shuffle(keys.begin(), keys.end(), std::default_random_engine(seed)); @@ -3070,7 +3070,7 @@ TEST_F(IVFSearcherTest, TestProviderInt8) { auto holder = make_shared>( dimension_); size_t doc_cnt = 5000UL; - std::vector keys(doc_cnt); + std::vector keys(doc_cnt); srand(Realtime::MilliSeconds()); bool rand_key = rand() % 2; bool rand_order = rand() % 2; @@ -3086,7 +3086,7 @@ TEST_F(IVFSearcherTest, TestProviderInt8) { } else { std::iota(keys.begin(), keys.end(), 0U); std::transform(keys.begin(), keys.end(), keys.begin(), - [&](key_t k) { return step * k; }); + [&](uint64_t k) { return step * k; }); if (rand_order) { uint32_t seed = Realtime::Seconds(); std::shuffle(keys.begin(), keys.end(), std::default_random_engine(seed)); diff --git a/tests/core/interface/index_interface_test.cc b/tests/core/interface/index_interface_test.cc index b38eea559..419798b97 100644 --- a/tests/core/interface/index_interface_test.cc +++ b/tests/core/interface/index_interface_test.cc @@ -17,6 +17,7 @@ #include #include #include +#include "tests/test_util.h" #include "zvec/ailego/buffer/buffer_manager.h" #include "zvec/core/interface/index.h" #include "zvec/core/interface/index_factory.h" @@ -33,12 +34,10 @@ using namespace zvec::core_interface; TEST(IndexInterface, General) { constexpr uint32_t kDimension = 64; const std::string index_name{"test.index"}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_name.c_str()); auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_name); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -86,7 +85,7 @@ TEST(IndexInterface, General) { ASSERT_FLOAT_EQ(1.0f, fetched_vector[1]); ASSERT_FLOAT_EQ(2.0f, fetched_vector[2]); index->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_name); }; @@ -155,13 +154,11 @@ TEST(IndexInterface, BufferGeneral) { zvec::ailego::BufferManager::Instance().init(100 * 1024 * 1024, 1); constexpr uint32_t kDimension = 64; const std::string index_name{"test.index"}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s*", index_name.c_str()); auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { std::string real_index_name = index_name; - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_name + "*"); auto write_index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, write_index); @@ -214,7 +211,7 @@ TEST(IndexInterface, BufferGeneral) { ASSERT_FLOAT_EQ(1.0f, fetched_vector[1]); ASSERT_FLOAT_EQ(2.0f, fetched_vector[2]); read_index->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_name + "*"); }; @@ -260,18 +257,17 @@ TEST(IndexInterface, BufferGeneral) { .with_fetch_vector(true) .with_ef_search(20) .build()); + zvec::ailego::BufferManager::Instance().cleanup(); } TEST(IndexInterface, SparseGeneral) { constexpr uint32_t kSparseCount = 3; const std::string index_name{"test.index"}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_name.c_str()); auto func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer &query_param) { - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_name); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -337,7 +333,7 @@ TEST(IndexInterface, SparseGeneral) { ASSERT_EQ(i, fetched_values[i]); } index->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_name); }; @@ -386,8 +382,7 @@ TEST(IndexInterface, Merge) { const std::string index_name{"test.index"}; auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + zvec::test_util::RemoveTestFiles(file_name); }; auto create_index_func = @@ -762,7 +757,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test invalid vector data type for sparse operations @@ -785,7 +780,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test fetch non-existent document @@ -806,7 +801,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test search with invalid vector data @@ -839,7 +834,7 @@ TEST(IndexInterface, Failure) { ASSERT_NE(0, ret); index->Close(); - system("rm -f test.index"); + zvec::test_util::RemoveTestFiles("test.index"); } // Test merge with invalid write concurrency @@ -883,7 +878,9 @@ TEST(IndexInterface, Failure) { index1->Close(); index2->Close(); index3->Close(); - system("rm -f test1.index test2.index test3.index"); + zvec::test_util::RemoveTestFiles("test1.index"); + zvec::test_util::RemoveTestFiles("test2.index"); + zvec::test_util::RemoveTestFiles("test3.index"); } } @@ -1013,9 +1010,7 @@ TEST(IndexInterface, Score) { auto sparse_indices = std::vector{0, 1, 2}; auto query_vector = std::vector{1.0f, 2.0f, 3.0f}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto check_score = [&](const SearchResult &result, MetricType metric_type) { ASSERT_EQ(result.doc_list_.size(), 2); @@ -1086,7 +1081,7 @@ TEST(IndexInterface, Score) { auto dense_func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer query_param, MetricType metric_type) { - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -1108,13 +1103,13 @@ TEST(IndexInterface, Score) { check_score(result, metric_type); index->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; auto sparse_func = [&](const BaseIndexParam::Pointer ¶m, const BaseIndexQueryParam::Pointer query_param, MetricType metric_type) { - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto index = IndexFactory::CreateAndInitIndex(*param); ASSERT_NE(nullptr, index); @@ -1142,7 +1137,7 @@ TEST(IndexInterface, Score) { check_score(result, metric_type); index->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; constexpr uint32_t kDimension = 3; diff --git a/tests/core/metric/quantized_integer_metric_test.cc b/tests/core/metric/quantized_integer_metric_test.cc index 501d8c7b9..835a07fb7 100644 --- a/tests/core/metric/quantized_integer_metric_test.cc +++ b/tests/core/metric/quantized_integer_metric_test.cc @@ -251,7 +251,7 @@ void TestDistanceMatrixInt8(const std::string &metric_name) { const size_t batch_size = M; const size_t query_size = N; - size_t dimension = (std::uniform_int_distribution(1, 65))(gen)*4; + size_t dimension = (std::uniform_int_distribution(1, 65))(gen) * 4; auto holder = GetHolder(dimension, batch_size, dist); IndexMeta meta(IndexMeta::DT_FP32, dimension); meta.set_metric(metric_name, 0, Params()); @@ -453,7 +453,7 @@ void TestDistanceMatrixInt4(const std::string &metric_name) { const size_t batch_size = M; const size_t query_size = N; - size_t dimension = (std::uniform_int_distribution(1, 65))(gen)*8; + size_t dimension = (std::uniform_int_distribution(1, 65))(gen) * 8; auto holder = GetHolder(dimension, batch_size, dist); IndexMeta meta(IndexMeta::DT_FP32, dimension); meta.set_metric(metric_name, 0, Params()); diff --git a/tests/db/crash_recovery/data_generator.cc b/tests/db/crash_recovery/data_generator.cc index d0f5798fe..9fd2f308e 100644 --- a/tests/db/crash_recovery/data_generator.cc +++ b/tests/db/crash_recovery/data_generator.cc @@ -13,9 +13,9 @@ // limitations under the License. -#include #include #include +#include #include #include "zvec/ailego/logger/logger.h" #include "utility.h" diff --git a/tests/db/crash_recovery/write_recovery_test.cc b/tests/db/crash_recovery/write_recovery_test.cc index 1f53a5f41..d8b564c42 100644 --- a/tests/db/crash_recovery/write_recovery_test.cc +++ b/tests/db/crash_recovery/write_recovery_test.cc @@ -20,8 +20,20 @@ #include #include #include +#include "tests/test_util.h" #include "utility.h" +#ifdef _WIN32 +#include +#include +typedef HANDLE pid_t; +#define SIGKILL 9 +// 模拟简单的 WIFEXITED 逻辑 +#define WIFEXITED(status) true +#define WEXITSTATUS(status) (status) +#define WIFSIGNALED(status) ((status) != 0) +#endif + namespace zvec { @@ -34,8 +46,31 @@ const zvec::CollectionOptions options_{false, true}; static std::string LocateDataGenerator() { namespace fs = std::filesystem; - const std::vector candidates{"./data_generator", - "./bin/data_generator"}; + std::cout << "Current path: " << fs::current_path() << std::endl; + + const std::string base_name = "data_generator"; + std::vector candidates; + // Define potential search locations relative to the current working directory + const std::vector search_paths = {"./", "./bin/"}; + + for (const auto &p : search_paths) { + candidates.push_back(p); + } +#ifdef _WIN32 + for (const auto &p : search_paths) { + candidates.push_back(p + "Debug/"); + candidates.push_back(p + "Release/"); + } +#endif + + + for (auto &p : candidates) { + p += base_name; +#ifdef _WIN32 + p += ".exe"; // Append .exe suffix for Windows compatibility +#endif + } + for (const auto &p : candidates) { if (fs::exists(p)) { return fs::canonical(p).string(); @@ -45,93 +80,116 @@ static std::string LocateDataGenerator() { } -void RunGenerator(const std::string &start, const std::string &end, - const std::string &op, const std::string &version) { - pid_t pid = fork(); - ASSERT_GE(pid, 0); +/** + * Internal helper to execute the data generator process. + * If 'kill_after_seconds' is greater than 0, the process will be forcibly + * terminated after the specified duration to simulate a crash. + */ +void ExecuteProcess(const std::string &start, const std::string &end, + const std::string &op, const std::string &version, + int kill_after_seconds = -1) { + bool should_crash = kill_after_seconds >= 0; + +#ifdef _WIN32 + // 1. Build the command line string with quotes to handle paths with spaces + std::string cmd_str = data_generator_bin_ + " --path " + dir_path_ + + " --start " + start + " --end " + end + " --op " + op + + " --version " + version; + + STARTUPINFOA si = {sizeof(si)}; + PROCESS_INFORMATION pi; + + std::cout << cmd_str << std::endl; + + std::vector cmd_buf(cmd_str.begin(), cmd_str.end()); + cmd_buf.push_back('\0'); + if (!CreateProcessA(NULL, cmd_buf.data(), NULL, NULL, FALSE, 0, NULL, NULL, + &si, &pi)) { + FAIL() << "CreateProcess failed (" << GetLastError() << ")"; + } - if (pid == 0) { // Child process - char arg_path[] = "--path"; - char arg_start[] = "--start"; - char arg_end[] = "--end"; - char arg_op[] = "--op"; - char arg_version[] = "--version"; - char *args[] = {const_cast(data_generator_bin_.c_str()), - arg_path, - const_cast(dir_path_.c_str()), - arg_start, - const_cast(start.c_str()), - arg_end, - const_cast(end.c_str()), - arg_op, - const_cast(op.c_str()), - arg_version, - const_cast(version.c_str()), - nullptr}; - execvp(args[0], args); - perror("execvp failed"); - _exit(1); + if (should_crash) { + std::this_thread::sleep_for(std::chrono::seconds(kill_after_seconds)); + // Simulate a crash by killing the process + TerminateProcess(pi.hProcess, 1); } - int status; - waitpid(pid, &status, 0); - ASSERT_TRUE(WIFEXITED(status)) - << "Child process did not exit normally. Terminated by signal?"; - int exit_code = WEXITSTATUS(status); - ASSERT_EQ(exit_code, 0) << "data_generator failed with exit code: " - << exit_code; -} + WaitForSingleObject(pi.hProcess, INFINITE); + DWORD exit_code; + GetExitCodeProcess(pi.hProcess, &exit_code); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); -void RunGeneratorAndCrash(const std::string &start, const std::string &end, - const std::string &op, const std::string &version, - int seconds) { + if (should_crash) { + // In crash tests, we expect the process to have been terminated + ASSERT_NE(exit_code, 0) << "Process was expected to crash/terminate."; + } else { + ASSERT_EQ(exit_code, 0) << "Process failed with exit code: " << exit_code; + } + +#else + // POSIX Implementation pid_t pid = fork(); ASSERT_GE(pid, 0); if (pid == 0) { // Child process - char arg_path[] = "--path"; - char arg_start[] = "--start"; - char arg_end[] = "--end"; - char arg_op[] = "--op"; - char arg_version[] = "--version"; - char *args[] = {const_cast(data_generator_bin_.c_str()), - arg_path, - const_cast(dir_path_.c_str()), - arg_start, - const_cast(start.c_str()), - arg_end, - const_cast(end.c_str()), - arg_op, - const_cast(op.c_str()), - arg_version, - const_cast(version.c_str()), - nullptr}; - execvp(args[0], args); + // Use a vector to manage arguments cleanly + std::vector args = {data_generator_bin_.c_str(), + "--path", + dir_path_.c_str(), + "--start", + start.c_str(), + "--end", + end.c_str(), + "--op", + op.c_str(), + "--version", + version.c_str(), + nullptr}; + execvp(args[0], const_cast(args.data())); perror("execvp failed"); _exit(1); } - std::this_thread::sleep_for(std::chrono::seconds(seconds)); - if (kill(pid, 0) == 0) { + int status; + if (should_crash) { + std::this_thread::sleep_for(std::chrono::seconds(kill_after_seconds)); kill(pid, SIGKILL); + waitpid(pid, &status, 0); + ASSERT_TRUE(WIFSIGNALED(status)) << "Process did not crash as expected."; + } else { + waitpid(pid, &status, 0); + ASSERT_TRUE(WIFEXITED(status)) << "Process exited abnormally."; + ASSERT_EQ(WEXITSTATUS(status), 0); } - int status; - waitpid(pid, &status, 0); - ASSERT_TRUE(WIFSIGNALED(status)) - << "Child process was not killed by a signal. It exited normally?"; +#endif +} + +// Public API 1: Normal execution +void RunGenerator(const std::string &start, const std::string &end, + const std::string &op, const std::string &version) { + ExecuteProcess(start, end, op, version); +} + +// Public API 2: Execution with simulated crash +void RunGeneratorAndCrash(const std::string &start, const std::string &end, + const std::string &op, const std::string &version, + int seconds) { + ExecuteProcess(start, end, op, version, seconds); } class CrashRecoveryTest : public ::testing::Test { protected: void SetUp() override { - system("rm -rf ./crash_test_db"); - ASSERT_NO_THROW(data_generator_bin_ = LocateDataGenerator()); + zvec::test_util::RemoveTestPath("./crash_test_db"); + // ASSERT_NO_THROW(data_generator_bin_ = LocateDataGenerator()); + data_generator_bin_ = LocateDataGenerator(); } void TearDown() override { - system("rm -rf ./crash_test_db"); + zvec::test_util::RemoveTestPath("./crash_test_db"); } }; diff --git a/tests/db/index/CMakeLists.txt b/tests/db/index/CMakeLists.txt index 22a2932e1..9c28c70cb 100644 --- a/tests/db/index/CMakeLists.txt +++ b/tests/db/index/CMakeLists.txt @@ -18,11 +18,31 @@ if(APPLE) endif() file(GLOB_RECURSE ALL_TEST_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *_test.cc) + +if(MSVC) + set(_MSVC_EXCLUDE_TESTS + "storage/mmap_store_test.cc" + "storage/mem_store_test.cc" + "column/inverted_column/inverted_indexer_util_test.cc" + "segment/segment_test.cc" +# "buffer/buffer_manager_test.cc" +# "flat/flat_streamer_test.cc" +# "flat_sparse/flat_sparse_streamer_test.cc" +# "hnsw/hnsw_streamer_test.cc" +# "hnsw_sparse/hnsw_sparse_builder_test.cc" +# "hnsw_sparse/hnsw_sparse_streamer_test.cc" + ) + foreach(_excl ${_MSVC_EXCLUDE_TESTS}) + list(FILTER ALL_TEST_SRCS EXCLUDE REGEX "${_excl}") + endforeach() +endif() + foreach(CC_SRCS ${ALL_TEST_SRCS}) get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE) cc_gmock( NAME ${CC_TARGET} STRICT LIBS zvec_db + zvec_ailego zvec_proto core_metric_static core_utility_static diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc index 58e0a0a43..620b4ef6f 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_array_numbers_test.cc @@ -15,6 +15,7 @@ #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -229,9 +230,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -242,9 +241,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc index 2c78f05b0..953a45f7f 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_bool_test.cc @@ -15,6 +15,7 @@ #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -254,9 +255,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -267,9 +266,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc index bcda36898..07bc1e7da 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_cyclic_numbers_test.cc @@ -16,6 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -368,9 +369,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -381,9 +380,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc index 3cd1ae775..8dfe7cda1 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_sequential_numbers_test.cc @@ -16,6 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -479,9 +480,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -492,9 +491,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc b/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc index 8732ed24c..b1da86595 100644 --- a/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc +++ b/tests/db/index/column/inverted_column/inverted_column_indexer_string_test.cc @@ -16,6 +16,7 @@ #include #include #include "db/index/column/inverted_column/inverted_indexer.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -292,9 +293,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -305,9 +304,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc b/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc index 99045eacc..858af223d 100644 --- a/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc +++ b/tests/db/index/column/inverted_column/inverted_indexer_util_test.cc @@ -14,6 +14,7 @@ #include +#include "tests/test_util.h" #define private public #define protected public #include "db/index/column/inverted_column/inverted_indexer.h" @@ -38,9 +39,7 @@ class InvertedIndexTest : public testing::Test { /***** Global initialization and cleanup - Start *****/ public: static void SetUpTestCase() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); indexer_ = InvertedIndexer::CreateAndOpen(collection_name, working_dir, true, {}, false); @@ -51,9 +50,7 @@ class InvertedIndexTest : public testing::Test { static void TearDownTestCase() { indexer_.reset(); - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf %s", working_dir.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestPath(working_dir); } /***** Global initialization and cleanup - End *****/ diff --git a/tests/db/index/column/vector_column_indexer_test.cc b/tests/db/index/column/vector_column_indexer_test.cc index 483efcdee..5c91cdfb9 100644 --- a/tests/db/index/column/vector_column_indexer_test.cc +++ b/tests/db/index/column/vector_column_indexer_test.cc @@ -18,6 +18,7 @@ #include #include #include "db/index/column/vector_column/vector_column_params.h" +#include "tests/test_util.h" #include "zvec/ailego/utility/float_helper.h" #include "zvec/db/doc.h" #include "zvec/db/index_params.h" @@ -61,9 +62,7 @@ TEST(VectorColumnIndexerTest, General) { const std::string index_file_path = "test_indexer.index"; constexpr idx_t kDocId = 2345; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -121,15 +120,14 @@ TEST(VectorColumnIndexerTest, General) { ASSERT_NEAR(dense_vector[3], 0, 0.1); // 4. search - // https://stackoverflow.com/questions/69009389/how-to-get-away-with-using-designated-initializers-in-c17-or-why-is-it-seemi auto query_vector = std::vector{1.0f, 2.0f, 3.0f, 0}; auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto indexer_query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .query_params = query_params}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams indexer_query_params; + indexer_query_params.topk = 10; + indexer_query_params.filter = nullptr; + indexer_query_params.fetch_vector = true; + indexer_query_params.query_params = query_params; auto results = indexer->Search(query, indexer_query_params); ASSERT_TRUE(results.has_value()); @@ -173,7 +171,7 @@ TEST(VectorColumnIndexerTest, General) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP), @@ -215,9 +213,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeFP16) { constexpr idx_t kDocId = 2345; constexpr int dimension = 4; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -307,12 +303,12 @@ TEST(VectorColumnIndexerTest, DenseDataTypeFP16) { buffer.data()); auto query_vector = buffer; auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto indexer_query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .query_params = query_params}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams indexer_query_params; + indexer_query_params.topk = 10; + indexer_query_params.filter = nullptr; + indexer_query_params.fetch_vector = true; + indexer_query_params.query_params = query_params; auto results = indexer->Search(query, indexer_query_params); ASSERT_TRUE(results.has_value()); @@ -356,7 +352,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeFP16) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP), @@ -372,9 +368,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeINT8) { constexpr idx_t kDocId = 2345; constexpr int dimension = 4; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -445,15 +439,14 @@ TEST(VectorColumnIndexerTest, DenseDataTypeINT8) { } // 4. search - // https://stackoverflow.com/questions/69009389/how-to-get-away-with-using-designated-initializers-in-c17-or-why-is-it-seemi auto query_vector = std::vector{1, 2, 3, 0}; auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto indexer_query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .query_params = query_params}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams indexer_query_params; + indexer_query_params.topk = 10; + indexer_query_params.filter = nullptr; + indexer_query_params.fetch_vector = true; + indexer_query_params.query_params = query_params; auto results = indexer->Search(query, indexer_query_params); ASSERT_TRUE(results.has_value()); @@ -497,7 +490,7 @@ TEST(VectorColumnIndexerTest, DenseDataTypeINT8) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP), @@ -513,9 +506,7 @@ TEST(VectorColumnIndexerTest, SparseGeneral) { const std::string index_file_path = "test_indexer.index"; constexpr idx_t kDocId = 2345; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // create indexer auto indexer = std::make_shared( @@ -560,8 +551,10 @@ TEST(VectorColumnIndexerTest, SparseGeneral) { auto query = vector_column_params::VectorData{vector_column_params::SparseVector{ kSparseCount, indices.data(), values.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -607,7 +600,7 @@ TEST(VectorColumnIndexerTest, SparseGeneral) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP)); @@ -623,9 +616,7 @@ TEST(VectorColumnIndexerTest, SparseDataTypeFP16) { const std::string index_file_path = "test_indexer.index"; constexpr idx_t kDocId = 2345; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // create indexer auto indexer = std::make_shared( @@ -674,8 +665,10 @@ TEST(VectorColumnIndexerTest, SparseDataTypeFP16) { auto query = vector_column_params::VectorData{vector_column_params::SparseVector{ kSparseCount, indices.data(), values.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -721,7 +714,7 @@ TEST(VectorColumnIndexerTest, SparseDataTypeFP16) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::IP)); @@ -732,9 +725,8 @@ TEST(VectorColumnIndexerTest, Merge) { constexpr uint32_t kDimension = 64; const std::string index_name{"test_indexer.index"}; - auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = @@ -853,8 +845,10 @@ TEST(VectorColumnIndexerTest, Merge) { // search with fetch vector auto query = vector_column_params::VectorData{ vector_column_params::DenseVector{vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer2->Search(query, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = @@ -999,9 +993,8 @@ TEST(VectorColumnIndexerTest, SparseMerge) { constexpr uint32_t kUnitSize = sizeof(float); // VECTOR_FP32 const std::string index_name{"test_indexer.index"}; - auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = @@ -1277,9 +1270,7 @@ TEST(VectorColumnIndexerTest, BfPks) { auto func = [&](const IndexParams::Ptr index_params) { const std::string index_file_path = "test_indexer.index"; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // 1. create indexer auto indexer = std::make_shared( @@ -1305,14 +1296,14 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{1}; - auto query = - vector_column_params::VectorData{vector_column_params::DenseVector{ - .data = std::vector{1.0f, 2.0f, 3.0f}.data()}}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .bf_pks = {bf_pks}}; + auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; + auto query = vector_column_params::VectorData{ + vector_column_params::DenseVector{query_vec.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.bf_pks = {bf_pks}; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1334,14 +1325,14 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{1, 2}; - auto query = - vector_column_params::VectorData{vector_column_params::DenseVector{ - .data = std::vector{1.0f, 2.0f, 3.0f}.data()}}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .bf_pks = {bf_pks}}; + auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; + auto query = vector_column_params::VectorData{ + vector_column_params::DenseVector{query_vec.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.bf_pks = {bf_pks}; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1363,14 +1354,14 @@ TEST(VectorColumnIndexerTest, BfPks) { { auto bf_pks = std::vector{2}; - auto query = - vector_column_params::VectorData{vector_column_params::DenseVector{ - .data = std::vector{1.0f, 2.0f, 3.0f}.data()}}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = true, - .bf_pks = {bf_pks}}; + auto query_vec = std::vector{1.0f, 2.0f, 3.0f}; + auto query = vector_column_params::VectorData{ + vector_column_params::DenseVector{query_vec.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.bf_pks = {bf_pks}; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1392,7 +1383,7 @@ TEST(VectorColumnIndexerTest, BfPks) { indexer->Close(); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); }; func(std::make_shared(MetricType::COSINE)); @@ -1577,14 +1568,12 @@ TEST(VectorColumnIndexerTest, CosineGeneral) { const int kDim = 20; const int kCount = 20; // can't set too large, or the qunatization error // will be too large due to float's precision - const int kTopk = 10; + const uint32_t kTopk = 10; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto func = [&](const IndexParams::Ptr index_params, DataType data_type) { - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto indexer = std::make_shared( index_file_path, FieldSchema("test", data_type, kDim, false, index_params)); @@ -1623,11 +1612,11 @@ TEST(VectorColumnIndexerTest, CosineGeneral) { vector_column_params::DenseVector{buffer.data.data()}}; auto _t = std::make_shared(100); _t->set_is_linear(true); - auto query_params = - vector_column_params::QueryParams{.topk = kTopk, - .filter = nullptr, - .fetch_vector = true, - .query_params = _t}; + vector_column_params::QueryParams query_params; + query_params.topk = kTopk; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.query_params = _t; auto results = indexer->Search(data, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = @@ -1725,7 +1714,7 @@ TEST(VectorColumnIndexerTest, CosineGeneral) { TEST(VectorColumnIndexerTest, Score) { const std::string index_file_path = "test_indexer.index"; - const int kTopk = 10; + const uint32_t kTopk = 10; constexpr idx_t kDocId1 = 2345; constexpr idx_t kDocId2 = 5432; auto vector1 = std::vector{3.0f, 4.0f, 5.0f}; @@ -1737,9 +1726,7 @@ TEST(VectorColumnIndexerTest, Score) { auto sparse_indices = std::vector{0, 1, 2}; auto query_vector = std::vector{1.0f, 2.0f, 3.0f}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); auto check_score = [&](VectorIndexResults *vector_results, @@ -1833,9 +1820,11 @@ TEST(VectorColumnIndexerTest, Score) { .ok()); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = kTopk, .filter = nullptr, .fetch_vector = true}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = kTopk; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1884,8 +1873,10 @@ TEST(VectorColumnIndexerTest, Score) { vector_column_params::VectorData{vector_column_params::SparseVector{ 3, reinterpret_cast(sparse_indices.data()), query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1937,9 +1928,7 @@ TEST(VectorColumnIndexerTest, Failure) { constexpr idx_t kDocId = 1234; auto vector = std::vector{1.0f, 2.0f, 3.0f}; - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -f %s", index_file_path.c_str()); - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); // Test case 1: Operations on unopened indexer { @@ -1978,9 +1967,11 @@ TEST(VectorColumnIndexerTest, Failure) { // Test Search on unopened indexer auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = false}; + vector_column_params::DenseVector{vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; auto search_result = indexer->Search(query, query_params); ASSERT_FALSE(search_result.has_value()); ASSERT_EQ(search_result.error().message(), "Index not opened"); @@ -2115,14 +2106,14 @@ TEST(VectorColumnIndexerTest, Failure) { // Test search with bf_pks size > 1 auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = vector.data()}}; + vector_column_params::DenseVector{vector.data()}}; auto bf_pks1 = std::vector{1, 2}; auto bf_pks2 = std::vector{3, 4}; - auto query_params = - vector_column_params::QueryParams{.topk = 10, - .filter = nullptr, - .fetch_vector = false, - .bf_pks = {bf_pks1, bf_pks2}}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; + query_params.bf_pks = {bf_pks1, bf_pks2}; auto search_result = indexer->Search(query, query_params); ASSERT_FALSE(search_result.has_value()); @@ -2305,9 +2296,11 @@ TEST(VectorColumnIndexerTest, Failure) { // // // Test search with unsupported index type // auto query = vector_column_params::VectorData{ - // vector_column_params::DenseVector{.data = vector.data()}}; - // auto query_params = vector_column_params::QueryParams{ - // .topk = 10, .filter = nullptr, .fetch_vector = false}; + // vector_column_params::DenseVector{vector.data()}}; + // vector_column_params::QueryParams query_params; + // query_params.topk = 10; + // query_params.filter = nullptr; + // query_params.fetch_vector = false; // // auto search_result = indexer->Search(query, query_params); // ASSERT_FALSE(search_result.has_value()); @@ -2316,16 +2309,15 @@ TEST(VectorColumnIndexerTest, Failure) { // indexer->Close(); // } - system(cmd_buf); + zvec::test_util::RemoveTestFiles(index_file_path); } TEST(VectorColumnIndexerTest, CosineMerge) { constexpr uint32_t kDimension = 64; const std::string index_name{"test_indexer.index"}; - auto del_index_file_func = [&](const std::string file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = @@ -2465,8 +2457,10 @@ TEST(VectorColumnIndexerTest, CosineMerge) { // search with fetch vector auto query = vector_column_params::VectorData{ vector_column_params::DenseVector{vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .topk = 10, .filter = nullptr, .fetch_vector = true}; + vector_column_params::QueryParams query_params; + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = true; auto results = indexer2->Search(query, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = @@ -2573,11 +2567,10 @@ TEST(VectorColumnIndexerTest, Refiner) { const int kDim = 20; const int kCount = 20; // can't set too large, or the qunatization error // will be too large due to float's precision - const int kTopk = 10; + const uint32_t kTopk = 10; - auto del_index_file_func = [&](const std::string &file_name) { - auto cmd_buf = "rm -f " + file_name; - system(cmd_buf.c_str()); + auto del_index_file_func = [](const std::string &file_name) { + zvec::test_util::RemoveTestFiles(file_name); }; auto create_indexer_func = @@ -2624,15 +2617,14 @@ TEST(VectorColumnIndexerTest, Refiner) { auto data = vector_column_params::VectorData{ vector_column_params::DenseVector{buffer.data.data()}}; ; - auto query_params = vector_column_params::QueryParams{ - .topk = kTopk, - .filter = nullptr, - .fetch_vector = true, - .query_params = std::make_shared(100), - .refiner_param = std::make_shared( - vector_column_params::RefinerParam{ - .scale_factor_ = 10, - .reference_indexer = reference_indexer})}; + vector_column_params::QueryParams query_params; + query_params.topk = kTopk; + query_params.filter = nullptr; + query_params.fetch_vector = true; + query_params.query_params = std::make_shared(100); + query_params.refiner_param = + std::make_shared( + vector_column_params::RefinerParam{10, reference_indexer}); auto results = indexer->Search(data, query_params); ASSERT_TRUE(results.has_value()); auto vector_results = diff --git a/tests/db/index/segment/segment_test.cc b/tests/db/index/segment/segment_test.cc index 6ca6fffe4..9530b8cf1 100644 --- a/tests/db/index/segment/segment_test.cc +++ b/tests/db/index/segment/segment_test.cc @@ -1078,12 +1078,12 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexer) { auto dense_fp32_field = schema->get_field("dense_fp32"); auto query_vector = new_doc.get>("dense_fp32").value(); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .dimension = dense_fp32_field->dimension(), - .topk = 10, - .filter = nullptr, - .fetch_vector = false}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.dimension = dense_fp32_field->dimension(); + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; auto results = combined_indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); @@ -1144,13 +1144,14 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexerWithQuantVectorIndex) { auto dense_fp32_field = schema->get_field("dense_fp32"); auto query_vector = new_doc.get>("dense_fp32").value(); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .dimension = dense_fp32_field->dimension(), - .topk = 10, - .filter = nullptr, - .fetch_vector = false, - .query_params = std::make_shared(IndexType::HNSW)}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.dimension = dense_fp32_field->dimension(); + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; + query_params.query_params = + std::make_shared(IndexType::HNSW); query_params.query_params->set_is_using_refiner(true); auto results = combined_indexer->Search(query, query_params); @@ -1195,18 +1196,16 @@ TEST_P(SegmentTest, CombinedVectorColumnIndexerQueryWithPks) { auto dense_fp32_field = schema->get_field("dense_fp32"); auto query_vector = verify_doc.get>("dense_fp32").value(); auto query = vector_column_params::VectorData{ - vector_column_params::DenseVector{.data = query_vector.data()}}; - auto query_params = vector_column_params::QueryParams{ - .data_type = dense_fp32_field->data_type(), - .dimension = dense_fp32_field->dimension(), - .topk = 10, - .filter = nullptr, - .fetch_vector = false, - .query_params = std::make_shared(IndexType::HNSW), - .group_by = nullptr, - .bf_pks = bf_pks, - .refiner_param = nullptr, - .extra_params = {}}; + vector_column_params::DenseVector{query_vector.data()}}; + vector_column_params::QueryParams query_params; + query_params.data_type = dense_fp32_field->data_type(); + query_params.dimension = dense_fp32_field->dimension(); + query_params.topk = 10; + query_params.filter = nullptr; + query_params.fetch_vector = false; + query_params.query_params = + std::make_shared(IndexType::HNSW); + query_params.bf_pks = bf_pks; auto results = combined_indexer->Search(query, query_params); ASSERT_TRUE(results.has_value()); diff --git a/tests/db/index/storage/mem_store_test.cc b/tests/db/index/storage/mem_store_test.cc index 27c019144..d061b2aea 100644 --- a/tests/db/index/storage/mem_store_test.cc +++ b/tests/db/index/storage/mem_store_test.cc @@ -807,12 +807,13 @@ TEST_F(MemStoreTest, ThreadSafety) { std::vector> futures; for (int t = 0; t < num_threads; ++t) { - futures.push_back(std::async(std::launch::async, [this, t]() { - for (int i = 0; i < inserts_per_thread; ++i) { - uint64_t doc_id = t * inserts_per_thread + i; - store_->insert(CreateDoc(doc_id)); - } - })); + futures.push_back( + std::async(std::launch::async, [this, t, inserts_per_thread]() { + for (int i = 0; i < inserts_per_thread; ++i) { + uint64_t doc_id = t * inserts_per_thread + i; + store_->insert(CreateDoc(doc_id)); + } + })); } // Wait for all threads to complete diff --git a/tests/db/index/storage/wal_file_test.cc b/tests/db/index/storage/wal_file_test.cc index e2a073f17..eabb91720 100644 --- a/tests/db/index/storage/wal_file_test.cc +++ b/tests/db/index/storage/wal_file_test.cc @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifdef _MSC_VER +#define _ALLOW_KEYWORD_MACROS +#endif #define private public #define protected public #include "db/index/storage/wal/wal_file.h" @@ -27,6 +30,7 @@ #include #include #include "db/common/file_helper.h" +#include "tests/test_util.h" #if defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic push @@ -39,9 +43,7 @@ using SegmentID = uint32_t; class WalFileTest : public testing::Test { protected: void SetUp() { - char cmd_buf[100]; - snprintf(cmd_buf, 100, "rm -rf ./data.wal.*"); - system(cmd_buf); + zvec::test_util::RemoveTestFiles("./data.wal.*"); } void TearDown() {} diff --git a/tests/test_util.h b/tests/test_util.h new file mode 100644 index 000000000..00fedd81b --- /dev/null +++ b/tests/test_util.h @@ -0,0 +1,71 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +#ifdef _MSC_VER +#include +#include +#include +#include +#endif + +namespace zvec { +namespace test_util { + +inline void RemoveTestPath(const std::string &path) { + if (!ailego::FileHelper::RemovePath(path.c_str())) { +#ifdef _WIN32 + system(("rmdir /s /q \"" + path + "\" 2>NUL").c_str()); +#endif + } +} + +inline void RemoveTestFiles(const std::string &pattern) { + if (pattern.find('*') != std::string::npos || + pattern.find('?') != std::string::npos) { +#ifdef _WIN32 + system(("del /f /q " + pattern + " 2>NUL").c_str()); +#else + system(("rm -rf " + pattern).c_str()); +#endif + } else { + ailego::FileHelper::RemovePath(pattern.c_str()); + } +} + +} // namespace test_util +} // namespace zvec + +#ifdef _MSC_VER +inline void sleep(unsigned int seconds) { + std::this_thread::sleep_for(std::chrono::seconds(seconds)); +} + +inline void usleep(unsigned int microseconds) { + std::this_thread::sleep_for(std::chrono::microseconds(microseconds)); +} + +inline int truncate(const char *path, long length) { + int fd = _open(path, _O_RDWR); + if (fd < 0) return -1; + int ret = _chsize(fd, length); + _close(fd); + return ret; +} +#endif diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index a32eac5ef..fb4415973 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -12,7 +12,12 @@ set(EXTERNAL_LIB_DIR ${CMAKE_BINARY_DIR}/external/usr/local/lib) file(MAKE_DIRECTORY ${EXTERNAL_INC_DIR}) file(MAKE_DIRECTORY ${EXTERNAL_LIB_DIR}) +set(_SAVED_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) +set(BUILD_SHARED_LIBS OFF) +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) add_subdirectory(googletest googletest EXCLUDE_FROM_ALL) +set(BUILD_SHARED_LIBS ${_SAVED_BUILD_SHARED_LIBS}) +unset(_SAVED_BUILD_SHARED_LIBS) add_subdirectory(gflags gflags EXCLUDE_FROM_ALL) add_subdirectory(glog glog EXCLUDE_FROM_ALL) add_subdirectory(sparsehash sparsehash EXCLUDE_FROM_ALL) diff --git a/thirdparty/antlr/CMakeLists.txt b/thirdparty/antlr/CMakeLists.txt index fe620ef02..f137628f6 100644 --- a/thirdparty/antlr/CMakeLists.txt +++ b/thirdparty/antlr/CMakeLists.txt @@ -2,14 +2,23 @@ set(ANTLR_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/antlr4) set(ANTLR_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/antlr4.patch) apply_patch_once("antlr4_fix" "${ANTLR_SRC_DIR}" "${ANTLR_PATCH}") +set(WITH_STATIC_CRT OFF CACHE BOOL "Use dynamic CRT for antlr4" FORCE) add_subdirectory(antlr4/runtime/Cpp/) add_library(antlr4 UNKNOWN IMPORTED GLOBAL) +if(MSVC) + set(_antlr4_lib_name "antlr4-runtime-static.lib") +else() + set(_antlr4_lib_name "libantlr4-runtime.a") +endif() set_target_properties( antlr4 PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/runtime/src/" - IMPORTED_LOCATION "${EXTERNAL_LIB_DIR}/libantlr4-runtime.a" + IMPORTED_LOCATION "${EXTERNAL_LIB_DIR}/${_antlr4_lib_name}" ) +if(WIN32) + set_property(TARGET antlr4 PROPERTY INTERFACE_COMPILE_DEFINITIONS "ANTLR4CPP_STATIC") +endif() add_dependencies(antlr4 antlr4_static) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") diff --git a/thirdparty/arrow/CMakeLists.txt b/thirdparty/arrow/CMakeLists.txt index 4326ea92a..b72f85c97 100644 --- a/thirdparty/arrow/CMakeLists.txt +++ b/thirdparty/arrow/CMakeLists.txt @@ -12,12 +12,26 @@ include(ProcessorCount) ProcessorCount(NPROC) -set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/libparquet.a) -set(LIB_ARROW ${EXTERNAL_LIB_DIR}/libarrow.a) -set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/libarrow_compute.a) -set(LIB_ACERO ${EXTERNAL_LIB_DIR}/libarrow_acero.a) -set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/libarrow_bundled_dependencies.a) -set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/libarrow_dataset.a) +if(MSVC) + set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/parquet_static.lib) + set(LIB_ARROW ${EXTERNAL_LIB_DIR}/arrow_static.lib) + set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/arrow_compute_static.lib) + set(LIB_ACERO ${EXTERNAL_LIB_DIR}/arrow_acero_static.lib) + set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/arrow_bundled_dependencies.lib) + set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/arrow_dataset_static.lib) +else() + set(LIB_PARQUET ${EXTERNAL_LIB_DIR}/libparquet.a) + set(LIB_ARROW ${EXTERNAL_LIB_DIR}/libarrow.a) + set(LIB_COMPUTE ${EXTERNAL_LIB_DIR}/libarrow_compute.a) + set(LIB_ACERO ${EXTERNAL_LIB_DIR}/libarrow_acero.a) + set(LIB_ARROW_DEPENDS ${EXTERNAL_LIB_DIR}/libarrow_bundled_dependencies.a) + set(LIB_ARROW_DATASET ${EXTERNAL_LIB_DIR}/libarrow_dataset.a) +endif() + +# $ is a generator expression that evaluates to the current build configuration (e.g., Debug, Release) at build system generation time, not at CMake configure time. +# Multi-config vs. single-config generators: +# Multi-config generators (Visual Studio, Xcode, Ninja Multi-Config): $ works as expected and evaluates differently for each configuration. +# Single-config generators (Makefiles, Ninja): $ evaluates to the value of CMAKE_BUILD_TYPE (which must be set at configure time) set(CONFIGURE_ENV_LIST "") if(USE_OSS_MIRROR) @@ -49,14 +63,18 @@ if(ANDROID) LOG_INSTALL ON ) else() + set(ARROW_EXTRA_CMAKE_ARGS "") + if(MSVC) + set(ARROW_EXTRA_CMAKE_ARGS -DCMAKE_C_FLAGS=/FS -DCMAKE_CXX_FLAGS=/FS) + endif() ExternalProject_Add( ARROW.BUILD PREFIX arrow SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/apache-arrow-21.0.0 DOWNLOAD_COMMAND "" BUILD_IN_SOURCE false - CONFIGURE_COMMAND env ${CONFIGURE_ENV_LIST} "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" - BUILD_COMMAND "${CMAKE_COMMAND}" --build . --target all -- -j ${NPROC} - INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local + CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${CMAKE_CACHE_ARGS} ${ARROW_EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=$ -DCMAKE_DEBUG_POSTFIX= -DARROW_BUILD_SHARED=OFF -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_DATASET=ON -DARROW_PARQUET=ON -DARROW_COMPUTE=ON -DARROW_WITH_ZLIB=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_MIMALLOC=OFF -DCMAKE_INSTALL_LIBDIR=lib "/cpp" + BUILD_COMMAND "${CMAKE_COMMAND}" --build . -j ${NPROC} --config $ + INSTALL_COMMAND "${CMAKE_COMMAND}" --install "" --prefix=${EXTERNAL_BINARY_DIR}/usr/local --config $ BYPRODUCTS ${LIB_PARQUET} ${LIB_ARROW} ${LIB_COMPUTE} ${LIB_ACERO} ${LIB_ARROW_DEPENDS} ${LIB_ARROW_DATASET} LOG_DOWNLOAD ON LOG_CONFIGURE ON @@ -70,7 +88,7 @@ add_dependencies(arrow ARROW.BUILD) set(Arrow_FOUND TRUE PARENT_SCOPE) set(Arrow_INCLUDE_DIR ${EXTERNAL_INC_DIR} PARENT_SCOPE) -set(Arrow_LIBRARIES ${EXTERNAL_LIB_DIR}/libarrow.a PARENT_SCOPE) +set(Arrow_LIBRARIES ${LIB_ARROW} PARENT_SCOPE) set(Arrow_DIR ${EXTERNAL_BINARY_DIR} PARENT_SCOPE) set(Arrow_LIBRARY_DIR ${EXTERNAL_BINARY_DIR} PARENT_SCOPE) @@ -89,6 +107,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_ARROW}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends" + INTERFACE_COMPILE_DEFINITIONS "ARROW_STATIC" ) add_dependencies(Arrow::arrow_static ARROW.BUILD) @@ -98,6 +117,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_PARQUET}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static" + INTERFACE_COMPILE_DEFINITIONS "PARQUET_STATIC" ) add_dependencies(Arrow::parquet_static ARROW.BUILD) @@ -107,6 +127,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_COMPUTE}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static" + INTERFACE_COMPILE_DEFINITIONS "ARROW_COMPUTE_STATIC" ) add_dependencies(Arrow::arrow_compute ARROW.BUILD) @@ -116,6 +137,7 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_ACERO}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static;Arrow::arrow_compute" + INTERFACE_COMPILE_DEFINITIONS "ARROW_ACERO_STATIC" ) add_dependencies(Arrow::arrow_acero ARROW.BUILD) @@ -125,5 +147,6 @@ set_target_properties( INTERFACE_INCLUDE_DIRECTORIES ${EXTERNAL_INC_DIR} IMPORTED_LOCATION "${LIB_ARROW_DATASET}" INTERFACE_LINK_LIBRARIES "Arrow::arrow_depends;Arrow::arrow_static;Arrow::arrow_compute;Arrow::arrow_acero" + INTERFACE_COMPILE_DEFINITIONS "ARROW_DS_STATIC" ) add_dependencies(Arrow::arrow_dataset ARROW.BUILD) diff --git a/thirdparty/glog/CMakeLists.txt b/thirdparty/glog/CMakeLists.txt index 611f18efc..e460aa681 100644 --- a/thirdparty/glog/CMakeLists.txt +++ b/thirdparty/glog/CMakeLists.txt @@ -3,7 +3,9 @@ set(WITH_GFLAGS OFF CACHE BOOL "Disable find_package(gflags) in glog" FORCE) set(WITH_UNWIND OFF CACHE BOOL "Disable find_package(unwind) in glog" FORCE) set(HAVE_LIB_GFLAGS TRUE CACHE BOOL "") +if(NOT MSVC) add_compile_options(-Wno-deprecated-declarations) +endif() set(GLOG_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/glog-0.5.0) if (ANDROID) @@ -14,6 +16,15 @@ else() apply_patch_once("glog_fix" "${GLOG_SRC_DIR}" "${GLOG_PATCH}") endif() +if(MSVC) + set(INTTYPES_FORMAT VC7) +# # Pre-populate glog type checks that may fail under MSVC in subdirectory builds. +# # MSVC has both uint16_t (via ) and unsigned __int16 (built-in). +# set(HAVE___UINT16 "2" CACHE INTERNAL "sizeof unsigned __int16" FORCE) +# set(HAVE_UINT16_T "2" CACHE INTERNAL "sizeof uint16_t" FORCE) +# set(HAVE_U_INT16_T "" CACHE INTERNAL "sizeof u_int16_t" FORCE) +endif() + set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(glog-0.5.0) @@ -22,6 +33,12 @@ unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) add_dependencies(glog gflags) +if(MSVC) + # Modern MSVC (1900+/VS2015+) provides snprintf in the CRT. + # Suppress glog's own declaration which conflicts with the SDK header. + target_compile_definitions(glog PRIVATE HAVE_SNPRINTF) +endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") target_compile_options(glog PRIVATE -Wno-sign-compare) endif() diff --git a/thirdparty/googletest/CMakeLists.txt b/thirdparty/googletest/CMakeLists.txt index e72a0c134..151302dc3 100644 --- a/thirdparty/googletest/CMakeLists.txt +++ b/thirdparty/googletest/CMakeLists.txt @@ -1,4 +1,6 @@ +if(NOT MSVC) add_compile_options(-Wno-deprecated-copy) +endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_compile_options(-Wno-maybe-uninitialized) endif() diff --git a/thirdparty/lz4/CMakeLists.txt b/thirdparty/lz4/CMakeLists.txt index 7db62d4f5..e66bdd101 100644 --- a/thirdparty/lz4/CMakeLists.txt +++ b/thirdparty/lz4/CMakeLists.txt @@ -3,8 +3,17 @@ set(lz4_LIBRARY_DIR "${EXTERNAL_BINARY_DIR}/usr/local/lib/") file(MAKE_DIRECTORY ${lz4_INCLUDE_DIR}) file(MAKE_DIRECTORY ${lz4_LIBRARY_DIR}) +if(MSVC) + set(LZ4_LIB_NAME lz4.lib) +else() + set(LZ4_LIB_NAME liblz4.a) +endif() + include(ExternalProject) +include(ProcessorCount) +ProcessorCount(NPROC) + set(_lz4_env "") if(ANDROID) string(REGEX REPLACE "^android-([0-9]+)$" "\\1" ANDROID_API_LEVEL "${ANDROID_PLATFORM}") @@ -45,32 +54,55 @@ else() list(APPEND _lz4_env "CFLAGS=-fPIC") endif() -ExternalProject_Add( - Lz4.BUILD - PREFIX lz4 - URL "${CMAKE_CURRENT_SOURCE_DIR}/lz4-1.9.4" - CONFIGURE_COMMAND "" - BUILD_COMMAND env ${_lz4_env} BUILD_SHARED=no make -j - INSTALL_COMMAND make DESTDIR=${EXTERNAL_BINARY_DIR} BUILD_SHARED=no install - BUILD_IN_SOURCE ON - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON - LOG_INSTALL ON - BUILD_BYPRODUCTS ${lz4_LIBRARY_DIR}/liblz4.a -) - +if(MSVC) + ExternalProject_Add( + Lz4.BUILD + PREFIX lz4 + URL "${CMAKE_CURRENT_SOURCE_DIR}/lz4-1.9.4" + SOURCE_SUBDIR build/cmake + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${EXTERNAL_BINARY_DIR}/usr/local + -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF + -DBUILD_STATIC_LIBS=ON + -DLZ4_BUILD_CLI=OFF + -DLZ4_BUILD_LEGACY_LZ4C=OFF + BUILD_COMMAND "${CMAKE_COMMAND}" --build . --config $ -j + INSTALL_COMMAND "${CMAKE_COMMAND}" --install . --config $ + BUILD_IN_SOURCE OFF + LOG_DOWNLOAD ON + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON + BUILD_BYPRODUCTS ${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME} + ) +else() + ExternalProject_Add( + Lz4.BUILD + PREFIX lz4 + URL "${CMAKE_CURRENT_SOURCE_DIR}/lz4-1.9.4" + CONFIGURE_COMMAND "" + BUILD_COMMAND env ${_lz4_env} BUILD_SHARED=no make -j + INSTALL_COMMAND make DESTDIR=${EXTERNAL_BINARY_DIR} BUILD_SHARED=no install + BUILD_IN_SOURCE ON + LOG_DOWNLOAD ON + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON + BUILD_BYPRODUCTS ${lz4_LIBRARY_DIR}/liblz4.a + ) +endif() add_library(lz4 STATIC IMPORTED GLOBAL) set_target_properties( - lz4 PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}" - IMPORTED_LOCATION "${lz4_LIBRARY_DIR}/liblz4.a" + lz4 PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}" + IMPORTED_LOCATION "${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME}" ) add_dependencies(lz4 Lz4.BUILD) set(lz4_FOUND TRUE PARENT_SCOPE) -set(lz4_LIBRARY ${lz4_LIBRARY_DIR}/liblz4.a PARENT_SCOPE) -set(lz4_LIBRARIES ${lz4_LIBRARY_DIR}/liblz4.a PARENT_SCOPE) +set(lz4_LIBRARY ${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME} PARENT_SCOPE) +set(lz4_LIBRARIES ${lz4_LIBRARY_DIR}/${LZ4_LIB_NAME} PARENT_SCOPE) set(lz4_INCLUDE_DIR "${EXTERNAL_BINARY_DIR}/usr/local/include" PARENT_SCOPE) set(lz4_INCLUDE_DIRS "${EXTERNAL_BINARY_DIR}/usr/local/include" PARENT_SCOPE) set(lz4_VERSION 1.9.4 PARENT_SCOPE) diff --git a/thirdparty/rocksdb/CMakeLists.txt b/thirdparty/rocksdb/CMakeLists.txt index 9d756bba8..ab1d6ec66 100644 --- a/thirdparty/rocksdb/CMakeLists.txt +++ b/thirdparty/rocksdb/CMakeLists.txt @@ -13,17 +13,37 @@ set(WITH_CORE_TOOLS OFF CACHE BOOL "build with ldb and sst_dump" FORCE) set(WITH_TOOLS OFF CACHE BOOL "build with tools" FORCE) set(WITH_LZ4 ON CACHE BOOL "build with lz4" FORCE) set(USE_RTTI ON CACHE BOOL "build with RTTI" FORCE) +set(ROCKSDB_SKIP_THIRDPARTY ON CACHE BOOL "skip thirdparty.inc" FORCE) set(FAIL_ON_WARNINGS OFF CACHE BOOL "build with no Werror" FORCE) set(PORTABLE ON CACHE BOOL "build a portable lib" FORCE) +set(lz4_INCLUDE_DIRS "${lz4_INCLUDE_DIR}" CACHE PATH "lz4 include directory" FORCE) +set(lz4_LIBRARIES "${lz4_LIBRARY}" CACHE FILEPATH "lz4 library" FORCE) +set(lz4_FOUND TRUE CACHE BOOL "lz4 found" FORCE) + +if(NOT TARGET lz4::lz4) + add_library(lz4::lz4 UNKNOWN IMPORTED GLOBAL) + set_target_properties(lz4::lz4 PROPERTIES + IMPORTED_LOCATION "${lz4_LIBRARY}" + IMPORTED_LOCATION_DEBUG "${lz4_LIBRARY}" + IMPORTED_LOCATION_RELEASE "${lz4_LIBRARY}" + IMPORTED_LOCATION_MINSIZEREL "${lz4_LIBRARY}" + IMPORTED_LOCATION_RELWITHDEBINFO "${lz4_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${lz4_INCLUDE_DIR}" + ) + add_dependencies(lz4::lz4 Lz4.BUILD) +endif() + set(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${EXTERNAL_LIB_DIR}) add_subdirectory(rocksdb-8.1.1) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY}) + unset(_SAVED_CMAKE_ARCHIVE_OUTPUT_DIRECTORY) get_target_property(rocksdb_SOURCE_DIR rocksdb SOURCE_DIR) set(ROCKSDB_INCLUDE_DIR ${rocksdb_SOURCE_DIR}/include) target_include_directories(rocksdb PUBLIC $) +target_include_directories(rocksdb PRIVATE ${lz4_INCLUDE_DIR}) add_dependencies(rocksdb Lz4.BUILD) diff --git a/thirdparty/sparsehash/CMakeLists.txt b/thirdparty/sparsehash/CMakeLists.txt index 9743c31aa..c389505fc 100644 --- a/thirdparty/sparsehash/CMakeLists.txt +++ b/thirdparty/sparsehash/CMakeLists.txt @@ -5,6 +5,11 @@ if(NOT EXISTS "${SPARSE_DST}") file(COPY "${SPARSE_SRC}" DESTINATION "${DESTINATION_DIR}") endif() +if(MSVC) + set(SPARSEHASH_WINDOWS_PATCH ${CMAKE_CURRENT_SOURCE_DIR}/sparsehash.windows.patch) + apply_patch_once("sparsehash.windows.patch" "${DESTINATION_DIR}" "${SPARSEHASH_WINDOWS_PATCH}") +endif() + add_library(sparsehash INTERFACE) target_include_directories( sparsehash INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/sparsehash-2.0.4/src/" diff --git a/thirdparty/sparsehash/sparsehash.windows.patch b/thirdparty/sparsehash/sparsehash.windows.patch new file mode 100644 index 000000000..99aebf69d --- /dev/null +++ b/thirdparty/sparsehash/sparsehash.windows.patch @@ -0,0 +1,13 @@ +diff --git a/src/sparsehash/internal/hashtable-common.h b/src/sparsehash/internal/hashtable-common.h +index bac2b88..59c752c 100644 +--- a/src/sparsehash/internal/hashtable-common.h ++++ b/src/sparsehash/internal/hashtable-common.h +@@ -51,7 +51,7 @@ _START_GOOGLE_NAMESPACE_ + + template struct SparsehashCompileAssert { }; + #define SPARSEHASH_COMPILE_ASSERT(expr, msg) \ +- __attribute__((unused)) typedef SparsehashCompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] ++ static_assert(bool(expr), #msg) + + namespace sparsehash_internal { + diff --git a/tools/core/bench_result.h b/tools/core/bench_result.h index 9c53b13a3..177e13513 100644 --- a/tools/core/bench_result.h +++ b/tools/core/bench_result.h @@ -14,11 +14,15 @@ #pragma once -#include #include #include #include #include +#ifdef _MSC_VER +#include +#else +#include +#endif #include namespace zvec { @@ -52,15 +56,29 @@ class BenchResult { lock_.unlock(); } void mark_start() { +#ifdef _MSC_VER + start_ = std::chrono::steady_clock::now(); +#else gettimeofday(&start_, NULL); +#endif } void mark_end() { +#ifdef _MSC_VER + end_ = std::chrono::steady_clock::now(); +#else gettimeofday(&end_, NULL); +#endif } long get_duration_by_ms() { +#ifdef _MSC_VER + return static_cast( + std::chrono::duration_cast(end_ - start_) + .count()); +#else long duration = (end_.tv_sec - start_.tv_sec) * 1000 + (end_.tv_usec - start_.tv_usec) / 1000; return duration; +#endif } long get_total_query_count() { return total_query_count_; @@ -114,8 +132,13 @@ class BenchResult { long total_process_time_by_us_; long min_time_by_us_; long max_time_by_us_; +#ifdef _MSC_VER + std::chrono::steady_clock::time_point start_; + std::chrono::steady_clock::time_point end_; +#else struct timeval start_; struct timeval end_; +#endif ailego::SpinMutex lock_; std::map process_time_map_; // }; diff --git a/tools/core/local_builder.cc b/tools/core/local_builder.cc index 9d502a1e5..82bce65c9 100644 --- a/tools/core/local_builder.cc +++ b/tools/core/local_builder.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include diff --git a/tools/core/local_builder_original.cc b/tools/core/local_builder_original.cc index f6a44014d..d17e0bc7c 100644 --- a/tools/core/local_builder_original.cc +++ b/tools/core/local_builder_original.cc @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include #include #include #include