VB-20: Add flexible sweep configs and per-efSearch benchmark output

convert_to_nvidia_format.py

@@ -7,10 +7,14 @@
 from typing import List, Dict, Optional, Tuple
 
 
-def create_index_name(config: Dict) -> str:
-    """Create index name from configuration parameters"""
+def create_index_name(config: Dict, metrics: Dict) -> str:
+    """Create index name from configuration and metrics.
+
+    efSearch is read from metrics (where it's a scalar int per search run)
+    rather than config (where it's now a list of values to sweep).
+    """
     algorithm = config.get('algoToRun', 'UNKNOWN')
-    ef_search = config.get('efSearch', 0)
+    ef_search = metrics.get('efSearch', 0)
 
     if algorithm in ['LUCENE_HNSW', 'hnsw']:
         beam_width = config.get('hnswBeamWidth', 0)
@@ -38,7 +42,7 @@ def convert_results_to_nvidia_format(results_json_path: str, output_dir: str, da
     elif algorithm in ['hnsw', 'LUCENE_HNSW']:
         algorithm = 'LUCENE_HNSW'
 
-    index_name = create_index_name(config)
+    index_name = create_index_name(config, metrics)
 
     recall_key = next((key for key in metrics.keys() if 'recall-accuracy' in key.lower()), None)
     if not recall_key:
@@ -93,16 +97,17 @@ def convert_results_to_nvidia_format(results_json_path: str, output_dir: str, da
         json.dump(data, f, indent=2)
 
     build_filepath = None
-    build_time_key = next((key for key in metrics.keys() if 'indexing-time' in key.lower()), None)
+    build_time_key = next((key for key in metrics.keys() if 'indexing-time-total' in key.lower()), None)
 
     if build_time_key:
         build_time_ms = float(metrics[build_time_key])
+        build_time_s = build_time_ms / 1000.0
 
         build_benchmark = {
             "name": f"{algorithm}/{index_name}",
-            "real_time": build_time_ms,
+            "real_time": build_time_s,
             "iterations": 1,
-            "time_unit": "ms",
+            "time_unit": "s",
             "run_name": "run_1",
             "run_type": "iteration",
             "repetitions": 1,

datasets.json

@@ -1,32 +1,49 @@
 {
   "datasets": {
-    "wiki-10m": {
-      "description": "Wikipedia 768-dimensions, ~87.5 million vectors",
-      "download-url": "https://data.rapids.ai/raft/datasets/wiki_all_10M/wiki_all_10M.tar",
-      "preparation-commands": "tar -xvf wiki_all_10M.tar",
+    "deep1b/deep1b-1M": {
+      "description": "deep1B 1M-slice dataset",
+      "base_file": "base.1M.fbin",
+      "query_file": "query.public.10k.fbin",
+      "ground_truth_file": "groundtruth.neighbors.ibin",
+      "num_docs": 1000000,
+      "vector_dimension": 96,
+      "top_k_ground_truth": 2000
+    },
+    "deep1b/deep1b-10M": {
+      "description": "deep1B 10M-slice dataset",
       "base_file": "base.10M.fbin",
-      "base_checksum": "8a0154c198592dfe096b0a94cfc89424abb1b64720bf9e10b3dad353e09d1fb9",
-      "query_file": "queries.fbin",
-      "query_checksum": "57c88c301a6ba032855af3a54bdb847a972004e1ab671ce1bff790ee9c39c855",
-      "ground_truth_file": "groundtruth.10M.neighbors.ibin",
-      "ground_truth_checksum": "6bf31445010a4f1c3513aa758a221af560ff8e03fcdd5afaa319b72aac6685b8",
+      "query_file": "query.public.10k.fbin",
+      "ground_truth_file": "groundtruth.neighbors.ibin",
       "num_docs": 10000000,
-      "vector_dimension": 768,
-      "top_k_ground_truth": 100
+      "vector_dimension": 96,
+      "top_k_ground_truth": 2000
     },
-    "sift-1m": {
-      "description": "1 million 128-dimensional SIFT vectors",
-      "download-url": "ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz",
-      "preparation-commands": "tar -xvf sift.tar.gz",
-      "base_file": "sift/sift_base.fvecs",
-      "base_checksum": "21f66e2975057b5728ba56de1c825bac4f4d89d596609ae985741c6242631816",
-      "query_file": "sift/sift_query.fvecs",
-      "query_checksum": "f7fc9be140accdfd64116c2fa2365ecdb69b8f084970c6b0532db5ff79ac8fdc",
-      "ground_truth_file": "sift/sift_groundtruth.ivecs",
-      "ground_truth_checksum": "2b71de0a8d5a83e6a84eec3e23fb8b611d8801dd9b3a6cd62f070ab65ea65f4f",
-      "num_docs": 1000000,
-      "vector_dimension": 128,
-      "top_k_ground_truth": 1000
+    "deep1b/deep1b-20M": {
+      "description": "deep1B 20M-slice dataset",
+      "base_file": "base.20M.fbin",
+      "query_file": "query.public.10k.fbin",
+      "ground_truth_file": "groundtruth.neighbors.ibin",
+      "num_docs": 20000000,
+      "vector_dimension": 96,
+      "top_k_ground_truth": 2000
+    },
+    "deep1b/deep1b-40M": {
+      "description": "deep1B 40M-slice dataset",
+      "base_file": "base.40M.fbin",
+      "query_file": "query.public.10k.fbin",
+      "ground_truth_file": "groundtruth.neighbors.ibin",
+      "num_docs": 40000000,
+      "vector_dimension": 96,
+      "top_k_ground_truth": 2000
+    },
+    "deep1b/deep1b-80M": {
+      "description": "deep1B 80M-slice dataset",
+      "base_file": "base.80M.fbin",
+      "query_file": "query.public.10k.fbin",
+      "ground_truth_file": "groundtruth.neighbors.ibin",
+      "num_docs": 80000000,
+      "vector_dimension": 96,
+      "top_k_ground_truth": 2000
     }
   }
 }

generate-combinations.py

@@ -42,158 +42,90 @@
     invariants["vectorDimension"] = dataset_info["vector_dimension"]
     print("sweep: " + sweep)
     for param, value in sweeps[sweep].get("common-params", {}).items():
-        if not isinstance(value, list):
+        # efSearch is always passed through as a list — Java handles the iteration
+        if param == 'efSearch':
+            # Ensure it's always a list
+            invariants[param] = value if isinstance(value, list) else [value]
+        elif not isinstance(value, list):
             invariants[param] = value
         else:
             variants[param] = value
-    for algo in sweeps[sweep].get("algorithms", []):
-        algorithms = sweeps[sweep].get("algorithms", [])
+    # Normalize algorithms to a list of (name, params) tuples.
+    # Supports both the original dict format:
+    #   "algorithms": { "LUCENE_HNSW": {...}, "CAGRA_HNSW": {...} }
+    # and the new list format (allows duplicate algorithm names):
+    #   "algorithms": [ { "name": "LUCENE_HNSW", ... }, { "name": "LUCENE_HNSW", ... } ]
+    raw_algorithms = sweeps[sweep].get("algorithms", [])
+    if isinstance(raw_algorithms, dict):
+        algo_list = [(name, params) for name, params in raw_algorithms.items()]
+    else:
+        algo_list = [(entry.pop("name"), entry) for entry in [dict(e) for e in raw_algorithms]]
+
+    for algo, algo_params in algo_list:
         algo_variants = variants.copy()
         algo_invariants = invariants.copy()
         algo_invariants["algoToRun"] = algo
 
-        for param, value in algorithms[algo].items():
+        for param, value in algo_params.items():
             if param not in ["params"]:
-                if not isinstance(value, list):
+                # efSearch is always passed through as a list — Java handles the iteration
+                if param == 'efSearch':
+                    algo_invariants[param] = value if isinstance(value, list) else [value]
+                elif not isinstance(value, list):
                     algo_invariants[param] = value
                 else:
                     algo_variants[param] = value
 
         # Generate all combination of variants. For each combination, generate a hashed ID, and a file with the
         # name pattern as <sweep>-<algo>-<hash>.json. The file should contain the invariants as is, and the variants as the current combination.
         if algo_variants:
-            # Separate efSearch from other variants if it exists
-            efSearch_values = None
-            other_variant_keys = []
-            other_variant_values = []
-
-            for key, value in algo_variants.items():
-                if key == 'efSearch':
-                    efSearch_values = value
-                else:
-                    other_variant_keys.append(key)
-                    other_variant_values.append(value)
-
-            # Generate combinations with efSearch at the beginning (innermost loop)
-            if efSearch_values and other_variant_keys:
-                # Generate combinations of other parameters first
-                for other_combination in itertools.product(*other_variant_values):
-                    other_variants = dict(zip(other_variant_keys, other_combination))
-                    # Then iterate through efSearch values
-                    for ef_index, ef_value in enumerate(efSearch_values):
-                        current_variants = other_variants.copy()
-                        current_variants['efSearch'] = ef_value
-
-                        # Skip if cagraIntermediateDegree < cagraGraphDegree
-                        if 'cagraIntermediateDegree' in current_variants and 'cagraGraphDegree' in current_variants:
-                            if current_variants['cagraIntermediateDegree'] < current_variants['cagraGraphDegree']:
-                                print(f"\t\tSkipping combination: cagraIntermediateDegree ({current_variants['cagraIntermediateDegree']}) < cagraGraphDegree ({current_variants['cagraGraphDegree']})")
-                                continue
-
-                        # Skip if hnswMaxConn > hnswBeamWidth
-                        if 'hnswMaxConn' in current_variants and 'hnswBeamWidth' in current_variants:
-                            if current_variants['hnswMaxConn'] > current_variants['hnswBeamWidth']:
-                                print(f"\t\tSkipping combination: hnswMaxConn ({current_variants['hnswMaxConn']}) > hnswBeamWidth ({current_variants['hnswBeamWidth']})")
-                                continue
-
-                        # Generate hash only from other_variants (excluding efSearch)
-                        base_hash = hashlib.md5(json.dumps(other_variants, sort_keys=True).encode()).hexdigest()[:8]
-                        hash_id = f"{base_hash}-ef{ef_value}"
-
-                        config = algo_invariants.copy()
-                        config.update(current_variants)
-
-                        # For multiple efSearch combinations: subsequent ones skip indexing
-                        if len(efSearch_values) > 1 and ef_index > 0:
-                            config['skipIndexing'] = True
-
-                        # Set cleanIndexDirectory based on position
-                        if ef_index == 0:
-                            config['cleanIndexDirectory'] = False
-                        elif ef_index == len(efSearch_values) - 1:
-                            config['cleanIndexDirectory'] = True
-                        else:
-                            config['cleanIndexDirectory'] = False
-
-                        # Use base_hash for index directory paths
-                        if 'hnswIndexDirPath' in config:
-                            config['hnswIndexDirPath'] = f"hnswIndex-{base_hash}"
-                        if 'cuvsIndexDirPath' in config:
-                            config['cuvsIndexDirPath'] = f"cuvsIndex-{base_hash}"
-
-                        filename = f"{algo}-{hash_id}.json"
-                        sweep_dir = f"{args.configs_dir}/{sweep}"
-                        filepath = f"{sweep_dir}/{filename}"
-                        os.makedirs(sweep_dir, exist_ok=True)
-                        with open(filepath, 'w') as f:
-                            json.dump(config, f, indent=2)
-                        print(f"\tGenerated config file: {filepath}")
-            elif efSearch_values:
-                # Only efSearch values, no other variants
-                for ef_index, ef_value in enumerate(efSearch_values):
-                    current_variants = {'efSearch': ef_value}
-                    # Generate hash from empty dict since no other variants exist
-                    base_hash = hashlib.md5(json.dumps({}, sort_keys=True).encode()).hexdigest()[:8]
-                    hash_id = f"{base_hash}-ef{ef_value}"
-
-                    config = algo_invariants.copy()
-                    config.update(current_variants)
-
-                    # For multiple efSearch combinations: subsequent ones skip indexing
-                    if len(efSearch_values) > 1 and ef_index > 0:
-                        config['skipIndexing'] = True
-
-                    # Set cleanIndexDirectory based on position
-                    if ef_index == 0:
-                        config['cleanIndexDirectory'] = False
-                    elif ef_index == len(efSearch_values) - 1:
-                        config['cleanIndexDirectory'] = True
-                    else:
-                        config['cleanIndexDirectory'] = False
-
-                    # Use base_hash for index directory paths
-                    if 'hnswIndexDirPath' in config:
-                        config['hnswIndexDirPath'] = f"hnswIndex-{base_hash}"
-                    if 'cuvsIndexDirPath' in config:
-                        config['cuvsIndexDirPath'] = f"cuvsIndex-{base_hash}"
-
-                    filename = f"{algo}-{hash_id}.json"
-                    sweep_dir = f"{args.configs_dir}/{sweep}"
-                    filepath = f"{sweep_dir}/{filename}"
-                    os.makedirs(sweep_dir, exist_ok=True)
-                    with open(filepath, 'w') as f:
-                        json.dump(config, f, indent=2)
-                    print(f"\tGenerated config file: {filepath}")
-            else:
-                # No efSearch, use original logic
-                variant_keys = list(algo_variants.keys())
-                variant_values = list(algo_variants.values())
-                for combination in itertools.product(*variant_values):
-                    current_variants = dict(zip(variant_keys, combination))
-
-                    # Skip if cagraIntermediateDegree < cagraGraphDegree
-                    if 'cagraIntermediateDegree' in current_variants and 'cagraGraphDegree' in current_variants:
-                        if current_variants['cagraIntermediateDegree'] < current_variants['cagraGraphDegree']:
-                            print(f"\t\tSkipping combination: cagraIntermediateDegree ({current_variants['cagraIntermediateDegree']}) < cagraGraphDegree ({current_variants['cagraGraphDegree']})")
-                            continue
-
-                    # Skip if hnswMaxConn > hnswBeamWidth
-                    if 'hnswMaxConn' in current_variants and 'hnswBeamWidth' in current_variants:
-                        if current_variants['hnswMaxConn'] > current_variants['hnswBeamWidth']:
-                            print(f"\t\tSkipping combination: hnswMaxConn ({current_variants['hnswMaxConn']}) > hnswBeamWidth ({current_variants['hnswBeamWidth']})")
-                            continue
-
-                    hash_id = hashlib.md5(json.dumps(current_variants, sort_keys=True).encode()).hexdigest()[:8]
-
-                    config = algo_invariants.copy()
-                    config.update(current_variants)
-                    filename = f"{algo}-{hash_id}.json"
-                    sweep_dir = f"{args.configs_dir}/{sweep}"
-                    filepath = f"{sweep_dir}/{filename}"
-                    os.makedirs(sweep_dir, exist_ok=True)
-                    with open(filepath, 'w') as f:
-                        json.dump(config, f, indent=2)
-                    print(f"\tGenerated config file: {filepath}")
+            variant_keys = list(algo_variants.keys())
+            variant_values = list(algo_variants.values())
+            for combination in itertools.product(*variant_values):
+                current_variants = dict(zip(variant_keys, combination))
+
+                config = algo_invariants.copy()
+                config.update(current_variants)
+
+                # Skip if cagraIntermediateGraphDegree < cagraGraphDegree
+                # (CAGRA silently clamps graphDegree down to intermediateGraphDegree,
+                # which produces duplicate test runs)
+                if config.get('cagraIntermediateGraphDegree', float('inf')) < config.get('cagraGraphDegree', 0):
+                    print(f"\t\tSkipping combination: cagraIntermediateGraphDegree ({config['cagraIntermediateGraphDegree']}) < cagraGraphDegree ({config['cagraGraphDegree']})")
+                    continue
+
+                # Skip if hnswMaxConn > hnswBeamWidth
+                if config.get('hnswMaxConn', 0) > config.get('hnswBeamWidth', float('inf')):
+                    print(f"\t\tSkipping combination: hnswMaxConn ({config['hnswMaxConn']}) > hnswBeamWidth ({config['hnswBeamWidth']})")
+                    continue
+
+                # Set indexDirPath based on hash
+                hash_input = {k: v for k, v in config.items() if k != 'indexDirPath'}
+                hash_id = hashlib.md5(json.dumps(hash_input, sort_keys=True, default=str).encode()).hexdigest()[:8]
+                config['indexDirPath'] = f"index-{hash_id}"
+
+                filename = f"{algo}-{hash_id}.json"
+                sweep_dir = f"{args.configs_dir}/{sweep}"
+                filepath = f"{sweep_dir}/{filename}"
+                os.makedirs(sweep_dir, exist_ok=True)
+                with open(filepath, 'w') as f:
+                    json.dump(config, f, indent=2)
+                print(f"\tGenerated config file: {filepath}")
+        else:
+            # No variants at all, just generate a single config
+            hash_id = hashlib.md5(json.dumps(algo_invariants, sort_keys=True).encode()).hexdigest()[:8]
+            config = algo_invariants.copy()
+
+            # Set indexDirPath based on hash
+            config['indexDirPath'] = f"index-{hash_id}"
+
+            filename = f"{algo}-{hash_id}.json"
+            sweep_dir = f"{args.configs_dir}/{sweep}"
+            filepath = f"{sweep_dir}/{filename}"
+            os.makedirs(sweep_dir, exist_ok=True)
+            with open(filepath, 'w') as f:
+                json.dump(config, f, indent=2)
+            print(f"\tGenerated config file: {filepath}")
 
 
     print("----------------------")

pom.xml

@@ -35,12 +35,12 @@
 	<dependency>
 		  <groupId>com.nvidia.cuvs.lucene</groupId>
 		    <artifactId>cuvs-lucene</artifactId>
-		      <version>26.04.0</version>
+		      <version>26.08.0</version>
 		      </dependency>
         <dependency>
             <groupId>com.nvidia.cuvs</groupId>
             <artifactId>cuvs-java</artifactId>
-            <version>26.04.0</version>
+            <version>26.08.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.lucene</groupId>