Add fresh algorithm

README.md

@@ -46,6 +46,8 @@ The following table summarizes the key features of each algorithm:
 | **[Odyssey](https://helios2.mi.parisdescartes.fr/~themisp/odyssey/)** | Distributed and parallel in-memory similarity search |
 | **[SOFA](https://helios2.mi.parisdescartes.fr/~themisp/publications/icde25-sofa.pdf)** | In-memory similarity search using Symbolic Fourier Approximation (SFA) |
 | **[Hercules](https://helios2.mi.parisdescartes.fr/~themisp/publications/pvldb22-hercules.pdf)** | In-memory hierarchical similarity search using EAPCA and SAX-based pruning |
+| **[DumpyOS](https://helios2.mi.parisdescartes.fr/~themisp/publications/vldbj24-dumpyos.pdf)** | In-memory scalable data series similarity search using an adaptive multi-ary iSAX index |
+| **[FreSH](http://publications.ics.forth.gr/tech-reports/2023/2023.TR489_FreSh_A_LockFree_Data_Series_Index.pdf)** | In-memory lock-free parallel similarity search using an iSAX index (SRDS 2023) |
 
 
 

benchmark/CMakeLists.txt

@@ -261,6 +261,27 @@ target_include_directories(bm_Sofa_L2Square
 )
 endif()
 
+# ////// DUMPYOS //////
+add_executable(bm_DumpyOS_L2Square
+    bm_DumpyOS_L2Square.cpp
+    bm_utils.cpp
+    ../commons/paramSetup.cpp
+    ../commons/test_bm_utils.cpp
+    ../commons/dataloaders.cpp
+)
+target_link_libraries(bm_DumpyOS_L2Square
+    PRIVATE
+    benchmark::benchmark
+    benchmark::benchmark_main
+    dino_lib
+    GTest::gtest
+)
+target_include_directories(bm_DumpyOS_L2Square
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../lib
+    ${CMAKE_CURRENT_SOURCE_DIR}/../commons
+)
+
 # ////// HERCULES //////
 add_executable(bm_Hercules_L2Square
     bm_Hercules_L2Square.cpp
@@ -282,6 +303,27 @@ target_include_directories(bm_Hercules_L2Square
     ${CMAKE_CURRENT_SOURCE_DIR}/../commons
 )
 
+# ////// FRESH //////
+add_executable(bm_Fresh_L2Square
+    bm_Fresh_L2Square.cpp
+    bm_utils.cpp
+    ../commons/paramSetup.cpp
+    ../commons/test_bm_utils.cpp
+    ../commons/dataloaders.cpp
+)
+target_link_libraries(bm_Fresh_L2Square
+    PRIVATE
+    benchmark::benchmark
+    benchmark::benchmark_main
+    dino_lib
+    GTest::gtest
+)
+target_include_directories(bm_Fresh_L2Square
+    PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../lib
+    ${CMAKE_CURRENT_SOURCE_DIR}/../commons
+)
+
 # ////// ODYSSEY (conditional on MPI) //////
 if(BUILD_ODYSSEY_AVAILABLE)
     if(DEBUG_MSG)

benchmark/bm_DumpyOS_L2Square.cpp

@@ -0,0 +1,147 @@
+#include <benchmark/benchmark.h>
+#include <cstdio>
+#include <string>
+#include "bm_utils.hpp"
+#include "../commons/dataloaders.hpp"
+#include "../commons/VectorDataLoader.h"
+#include "../commons/test_bm_utils.hpp"
+#include "../lib/algos/DumpyOS.hpp"
+#include "../lib/algos/DataSource.hpp"
+
+static bool endsWith(const std::string& s, const std::string& suffix) {
+    return s.size() >= suffix.size() &&
+           s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+struct DumpyOSSearchOnlyFixture : public benchmark::Fixture {
+    daisy::DumpyOS* search = nullptr;
+    float* database = nullptr;
+    float* query = nullptr;
+    daisy::idx_t* I = nullptr;
+    float* D = nullptr;
+    daisy::idx_t n_query = 0;
+    size_t k = 0;
+    std::string dataset_name;
+    size_t n_database = 0;
+    int thread_count = 0;
+
+    void SetUp(const benchmark::State& state) override {
+        int config_idx = static_cast<int>(state.range(0));
+        const SSTestConfig& config = test_configs_deep_seismic_astro270m[config_idx];
+
+        const bool use_fvecs = endsWith(config.dataset_path, ".fvecs") || endsWith(config.query_path, ".fvecs");
+        size_t dim_u = 0, n_database_u = 0, n_q_u = 0;
+        database = nullptr;
+
+        if (use_fvecs) {
+            database = fvecs_read(config.dataset_path.c_str(), &dim_u, &n_database_u, 0);
+            if (!database) {
+                std::cerr << "Failed to load dataset (fvecs)" << std::endl;
+                return;
+            }
+            const size_t query_limit = (config.query_limit > 0) ? static_cast<size_t>(config.query_limit) : 0;
+            query = fvecs_read(config.query_path.c_str(), &dim_u, &n_q_u, query_limit);
+            if (!query) {
+                std::cerr << "Failed to load queries (fvecs)" << std::endl;
+                delete[] database;
+                return;
+            }
+        } else {
+            std::string dataset_filename = pathToFilename(config.dataset_path);
+            std::string query_filename = pathToFilename(config.query_path);
+
+            daisy::idx_t dim, n_database, _, __;
+            if (!parseFilenameForConfig(dataset_filename, "bruteForce", dim, n_database, _, __)) {
+                std::cerr << "Failed to parse dataset config from filename: " << dataset_filename << std::endl;
+                return;
+            }
+
+            daisy::idx_t dim_q, n_q, ___, ____;
+            if (!parseFilenameForConfig(query_filename, "bruteForce", dim_q, n_q, ___, ____)) {
+                std::cerr << "Failed to parse query config from filename: " << query_filename << std::endl;
+                return;
+            }
+
+            if (dim != static_cast<daisy::idx_t>(dim_q)) {
+                std::cerr << "Dimension mismatch between dataset and queries" << std::endl;
+                return;
+            }
+
+            dim_u = static_cast<size_t>(dim);
+            n_database_u = static_cast<size_t>(n_database);
+            if (config.query_limit > 0 && static_cast<daisy::idx_t>(config.query_limit) < n_q)
+                n_q = static_cast<daisy::idx_t>(config.query_limit);
+            n_q_u = static_cast<size_t>(n_q);
+
+            database = loadBinData(config.dataset_path.c_str(), n_database, dim, false);
+            if (!database) {
+                std::cerr << "Failed to load dataset" << std::endl;
+                return;
+            }
+
+            query = loadBinData(config.query_path.c_str(), n_q, dim_q, false);
+            if (!query) {
+                std::cerr << "Failed to load queries" << std::endl;
+                delete[] database;
+                return;
+            }
+        }
+
+        search = new daisy::DumpyOS(daisy::DistanceType::L2_SQUARED);
+        search->setNumThreads(config.thread_count);
+
+        fprintf(stderr, "[DUMPYOS] Before buildIndex (n_database=%zu dim=%zu).\n", n_database_u, dim_u);
+        fflush(stderr);
+
+        daisy::InMemoryDataSource data_source(database, static_cast<daisy::idx_t>(n_database_u), static_cast<daisy::idx_t>(dim_u));
+        search->buildIndex(&data_source);
+
+        fprintf(stderr, "[DUMPYOS] Indexing finished (n_database=%zu dim=%zu).\n", n_database_u, dim_u);
+        fflush(stderr);
+
+        k = static_cast<size_t>(config.k_value);
+        n_query = static_cast<daisy::idx_t>(n_q_u);
+        I = new daisy::idx_t[n_query * k];
+        D = new float[n_query * k];
+
+        dataset_name = config.name;
+        n_database = n_database_u;
+        thread_count = config.thread_count;
+
+        fprintf(stderr, "[DUMPYOS] n_database=%zu n_query=%zu dim=%zu k=%zu threads=%d\n",
+                n_database_u, (size_t)n_query, dim_u, k, config.thread_count);
+        fflush(stderr);
+    }
+
+    void TearDown(const benchmark::State&) override {
+        delete search;
+        delete[] database;
+        delete[] query;
+        delete[] I;
+        delete[] D;
+        search = nullptr;
+        database = nullptr;
+        query = nullptr;
+        I = nullptr;
+        D = nullptr;
+    }
+};
+
+BENCHMARK_DEFINE_F(DumpyOSSearchOnlyFixture, BM_DumpyOS_SearchOnly)(benchmark::State& state) {
+    for (auto _ : state) {
+        fprintf(stderr, "[DUMPYOS] --- Query phase ---\n");
+        fprintf(stderr, "[DUMPYOS]   dataset=%s  n_database=%zu\n", dataset_name.c_str(), n_database);
+        fprintf(stderr, "[DUMPYOS]   search_threads=%d  n_query=%zu  k=%zu\n", thread_count, (size_t)n_query, k);
+        fflush(stderr);
+        search->searchIndex(query, n_query, static_cast<daisy::idx_t>(k), I, D);
+        fprintf(stderr, "[DUMPYOS] Querying finished (n_query=%zu k=%zu).\n", (size_t)n_query, k);
+        fflush(stderr);
+    }
+}
+
+BENCHMARK_REGISTER_F(DumpyOSSearchOnlyFixture, BM_DumpyOS_SearchOnly)
+    ->Args({0})->Args({1})->Args({2})->Args({3})->Args({4})->Args({5})->Args({6})->Args({7})
+    ->Iterations(1)
+    ->Unit(benchmark::kMillisecond);
+
+BENCHMARK_MAIN();

benchmark/bm_Fresh_L2Square.cpp

@@ -0,0 +1,153 @@
+#include <benchmark/benchmark.h>
+#include <cstdio>
+#include <string>
+#include "bm_utils.hpp"
+#include "../commons/dataloaders.hpp"
+#include "../commons/VectorDataLoader.h"
+#include "../commons/test_bm_utils.hpp"
+#include "../lib/algos/Fresh.hpp"
+#include "../lib/algos/DataSource.hpp"
+
+static bool endsWith(const std::string& s, const std::string& suffix) {
+    return s.size() >= suffix.size() &&
+           s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+struct FreshSearchOnlyFixture : public benchmark::Fixture {
+    daisy::Fresh* search = nullptr;
+    float* database = nullptr;
+    float* query = nullptr;
+    daisy::idx_t* I = nullptr;
+    float* D = nullptr;
+    daisy::idx_t n_query = 0;
+    size_t k = 0;
+    std::string dataset_name;
+    size_t n_database = 0;
+    int thread_count = 0;
+
+    void SetUp(const benchmark::State& state) override {
+        int config_idx = static_cast<int>(state.range(0));
+        const SSTestConfig& config = test_configs_deep_seismic_astro270m[config_idx];
+
+        const bool use_fvecs = endsWith(config.dataset_path, ".fvecs") || endsWith(config.query_path, ".fvecs");
+        size_t dim_u = 0, n_database_u = 0, n_q_u = 0;
+        database = nullptr;
+
+        if (use_fvecs) {
+            database = fvecs_read(config.dataset_path.c_str(), &dim_u, &n_database_u, 0);
+            if (!database) {
+                std::cerr << "Failed to load dataset (fvecs)" << std::endl;
+                return;
+            }
+            const size_t query_limit = (config.query_limit > 0) ? static_cast<size_t>(config.query_limit) : 0;
+            query = fvecs_read(config.query_path.c_str(), &dim_u, &n_q_u, query_limit);
+            if (!query) {
+                std::cerr << "Failed to load queries (fvecs)" << std::endl;
+                delete[] database;
+                return;
+            }
+        } else {
+            std::string dataset_filename = pathToFilename(config.dataset_path);
+            std::string query_filename = pathToFilename(config.query_path);
+
+            daisy::idx_t dim, n_database, _, __;
+            if (!parseFilenameForConfig(dataset_filename, "bruteForce", dim, n_database, _, __)) {
+                std::cerr << "Failed to parse dataset config from filename: " << dataset_filename << std::endl;
+                return;
+            }
+
+            daisy::idx_t dim_q, n_q, ___, ____;
+            if (!parseFilenameForConfig(query_filename, "bruteForce", dim_q, n_q, ___, ____)) {
+                std::cerr << "Failed to parse query config from filename: " << query_filename << std::endl;
+                return;
+            }
+
+            if (dim != static_cast<daisy::idx_t>(dim_q)) {
+                std::cerr << "Dimension mismatch between dataset and queries" << std::endl;
+                return;
+            }
+
+            dim_u = static_cast<size_t>(dim);
+            n_database_u = static_cast<size_t>(n_database);
+            if (config.query_limit > 0 && static_cast<daisy::idx_t>(config.query_limit) < n_q)
+                n_q = static_cast<daisy::idx_t>(config.query_limit);
+            n_q_u = static_cast<size_t>(n_q);
+
+            database = loadBinData(config.dataset_path.c_str(), n_database, dim, false);
+            if (!database) {
+                std::cerr << "Failed to load dataset" << std::endl;
+                return;
+            }
+
+            query = loadBinData(config.query_path.c_str(), n_q, dim_q, false);
+            if (!query) {
+                std::cerr << "Failed to load queries" << std::endl;
+                delete[] database;
+                return;
+            }
+        }
+
+        search = new daisy::Fresh(daisy::DistanceType::L2_SQUARED);
+        search->setNumThreads(config.thread_count);
+        search->setIndexWorkers(config.thread_count);
+
+        fprintf(stderr, "[FRESH] Before buildIndex (n_database=%zu dim=%zu).\n", n_database_u, dim_u);
+        fflush(stderr);
+
+        daisy::InMemoryDataSource data_source(database, static_cast<daisy::idx_t>(n_database_u), static_cast<daisy::idx_t>(dim_u));
+        search->buildIndex(&data_source);
+
+        fprintf(stderr, "[FRESH] Indexing finished (n_database=%zu dim=%zu).\n", n_database_u, dim_u);
+        fflush(stderr);
+
+        k = static_cast<size_t>(config.k_value);
+        n_query = static_cast<daisy::idx_t>(n_q_u);
+        I = new daisy::idx_t[n_query * k];
+        D = new float[n_query * k];
+
+        dataset_name = config.name;
+        n_database = n_database_u;
+        thread_count = config.thread_count;
+
+        fprintf(stderr, "[FRESH] n_database=%zu n_query=%zu dim=%zu k=%zu threads=%d\n",
+                n_database_u, (size_t)n_query, dim_u, k, config.thread_count);
+        fflush(stderr);
+    }
+
+    void TearDown(const benchmark::State&) override {
+        fprintf(stderr, "[FRESH] TearDown start.\n");
+        fflush(stderr);
+        delete search;
+        delete[] database;
+        delete[] query;
+        delete[] I;
+        delete[] D;
+        search = nullptr;
+        database = nullptr;
+        query = nullptr;
+        I = nullptr;
+        D = nullptr;
+        fprintf(stderr, "[FRESH] TearDown done.\n");
+        fflush(stderr);
+    }
+};
+
+BENCHMARK_DEFINE_F(FreshSearchOnlyFixture, BM_Fresh_SearchOnly)(benchmark::State& state) {
+    for (auto _ : state) {
+        fprintf(stderr, "[FRESH] --- Query phase ---\n");
+        fprintf(stderr, "[FRESH]   dataset=%s  n_database=%zu\n", dataset_name.c_str(), n_database);
+        fprintf(stderr, "[FRESH]   search_threads=%d  n_query=%zu  k=%zu\n", thread_count, (size_t)n_query, k);
+        fflush(stderr);
+        search->searchIndex(query, n_query, static_cast<daisy::idx_t>(k), I, D);
+        fprintf(stderr, "[FRESH] Querying finished (n_query=%zu k=%zu).\n", (size_t)n_query, k);
+        fflush(stderr);
+    }
+}
+
+BENCHMARK_REGISTER_F(FreshSearchOnlyFixture, BM_Fresh_SearchOnly)
+    // q=100, k=1,10,100,1000: DEEP (0-3), Seismic (4-7)
+    ->Args({0})->Args({1})->Args({2})->Args({3})->Args({4})->Args({5})->Args({6})->Args({7})
+    ->Iterations(1)
+    ->Unit(benchmark::kMillisecond);
+
+BENCHMARK_MAIN();