Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/arm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
aws-region: ${{ secrets.AWS_REGION }}
- name: Start EC2 runner
id: start-ec2-runner
uses: machulav/ec2-github-runner@v2.4.1
uses: machulav/ec2-github-runner@v2.4.2
with:
mode: start
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
Expand Down Expand Up @@ -50,7 +50,7 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2.4.1
uses: machulav/ec2-github-runner@v2.4.2
with:
mode: stop
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
Expand Down
5 changes: 2 additions & 3 deletions .github/workflows/benchmark-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
aws-region: ${{ secrets.AWS_REGION_BENCHMARK }}
- name: Start EC2 runner
id: start-ec2-runner
uses: machulav/ec2-github-runner@v2
uses: machulav/ec2-github-runner@v2.4.2
with:
mode: start
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
Expand Down Expand Up @@ -102,10 +102,9 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION_BENCHMARK }}
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2
uses: machulav/ec2-github-runner@v2.4.2
with:
mode: stop
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
label: ${{ needs.start-runner.outputs.runner_label }}
ec2-instance-id: ${{ needs.start-runner.outputs.ec2_instance_id }}

4 changes: 2 additions & 2 deletions .github/workflows/coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
aws-region: ${{ secrets.AWS_REGION_BENCHMARK }}
- name: Start EC2 runner
id: start-ec2-runner
uses: machulav/ec2-github-runner@v2
uses: machulav/ec2-github-runner@v2.4.2
with:
mode: start
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
Expand Down Expand Up @@ -103,7 +103,7 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION_BENCHMARK }}
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2
uses: machulav/ec2-github-runner@v2.4.2
with:
mode: stop
github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
Expand Down
100 changes: 88 additions & 12 deletions rust/vecsim-c/src/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1099,9 +1099,20 @@ macro_rules! impl_tiered_wrapper {
};
}

// Implement wrappers for Tiered indices (f32 only for now)
// Implement wrappers for Tiered indices
impl_tiered_wrapper!(TieredSingleF32Wrapper, TieredSingle<f32>, f32, false);
impl_tiered_wrapper!(TieredSingleF64Wrapper, TieredSingle<f64>, f64, false);
impl_tiered_wrapper!(TieredSingleBF16Wrapper, TieredSingle<BFloat16>, BFloat16, false);
impl_tiered_wrapper!(TieredSingleFP16Wrapper, TieredSingle<Float16>, Float16, false);
impl_tiered_wrapper!(TieredSingleI8Wrapper, TieredSingle<Int8>, Int8, false);
impl_tiered_wrapper!(TieredSingleU8Wrapper, TieredSingle<UInt8>, UInt8, false);

impl_tiered_wrapper!(TieredMultiF32Wrapper, TieredMulti<f32>, f32, true);
impl_tiered_wrapper!(TieredMultiF64Wrapper, TieredMulti<f64>, f64, true);
impl_tiered_wrapper!(TieredMultiBF16Wrapper, TieredMulti<BFloat16>, BFloat16, true);
impl_tiered_wrapper!(TieredMultiFP16Wrapper, TieredMulti<Float16>, Float16, true);
impl_tiered_wrapper!(TieredMultiI8Wrapper, TieredMulti<Int8>, Int8, true);
impl_tiered_wrapper!(TieredMultiU8Wrapper, TieredMulti<UInt8>, UInt8, true);

// ============================================================================
// Disk Index Wrappers
Expand Down Expand Up @@ -1247,8 +1258,13 @@ macro_rules! impl_disk_wrapper {
};
}

// Implement wrappers for Disk indices (f32 only for now)
// Implement wrappers for Disk indices
impl_disk_wrapper!(DiskSingleF32Wrapper, f32);
impl_disk_wrapper!(DiskSingleF64Wrapper, f64);
impl_disk_wrapper!(DiskSingleBF16Wrapper, BFloat16);
impl_disk_wrapper!(DiskSingleFP16Wrapper, Float16);
impl_disk_wrapper!(DiskSingleI8Wrapper, Int8);
impl_disk_wrapper!(DiskSingleU8Wrapper, UInt8);

/// Create a new disk-based index.
pub fn create_disk_index(params: &DiskParams) -> Option<Box<IndexHandle>> {
Expand All @@ -1257,14 +1273,34 @@ pub fn create_disk_index(params: &DiskParams) -> Option<Box<IndexHandle>> {
let metric = params.base.metric;
let dim = params.base.dim;

// Only f32 is supported for now
if data_type != VecSimType::VecSimType_FLOAT32 {
return None;
}

let index = DiskIndexSingle::<f32>::new(rust_params).ok()?;
let wrapper: Box<dyn IndexWrapper> =
Box::new(DiskSingleF32Wrapper::new(index, data_type, metric));
let wrapper: Box<dyn IndexWrapper> = match data_type {
VecSimType::VecSimType_FLOAT32 => {
let index = DiskIndexSingle::<f32>::new(rust_params).ok()?;
Box::new(DiskSingleF32Wrapper::new(index, data_type, metric))
}
VecSimType::VecSimType_FLOAT64 => {
let index = DiskIndexSingle::<f64>::new(rust_params).ok()?;
Box::new(DiskSingleF64Wrapper::new(index, data_type, metric))
}
VecSimType::VecSimType_BFLOAT16 => {
let index = DiskIndexSingle::<BFloat16>::new(rust_params).ok()?;
Box::new(DiskSingleBF16Wrapper::new(index, data_type, metric))
}
VecSimType::VecSimType_FLOAT16 => {
let index = DiskIndexSingle::<Float16>::new(rust_params).ok()?;
Box::new(DiskSingleFP16Wrapper::new(index, data_type, metric))
}
VecSimType::VecSimType_INT8 => {
let index = DiskIndexSingle::<Int8>::new(rust_params).ok()?;
Box::new(DiskSingleI8Wrapper::new(index, data_type, metric))
}
VecSimType::VecSimType_UINT8 => {
let index = DiskIndexSingle::<UInt8>::new(rust_params).ok()?;
Box::new(DiskSingleU8Wrapper::new(index, data_type, metric))
}
// INT32 and INT64 types not supported for disk indices
VecSimType::VecSimType_INT32 | VecSimType::VecSimType_INT64 => return None,
};

Some(Box::new(IndexHandle::new(
wrapper,
Expand Down Expand Up @@ -1540,17 +1576,57 @@ pub fn create_tiered_index(params: &TieredParams) -> Option<Box<IndexHandle>> {
let dim = params.base.dim;
let is_multi = params.base.multi;

// Tiered index currently only supports f32
let wrapper: Box<dyn IndexWrapper> = match (data_type, is_multi) {
(VecSimType::VecSimType_FLOAT32, false) => Box::new(TieredSingleF32Wrapper::new(
TieredSingle::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_FLOAT64, false) => Box::new(TieredSingleF64Wrapper::new(
TieredSingle::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_BFLOAT16, false) => Box::new(TieredSingleBF16Wrapper::new(
TieredSingle::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_FLOAT16, false) => Box::new(TieredSingleFP16Wrapper::new(
TieredSingle::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_INT8, false) => Box::new(TieredSingleI8Wrapper::new(
TieredSingle::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_UINT8, false) => Box::new(TieredSingleU8Wrapper::new(
TieredSingle::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_FLOAT32, true) => Box::new(TieredMultiF32Wrapper::new(
TieredMulti::new(rust_params),
data_type,
)),
_ => return None, // Tiered only supports f32 currently
(VecSimType::VecSimType_FLOAT64, true) => Box::new(TieredMultiF64Wrapper::new(
TieredMulti::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_BFLOAT16, true) => Box::new(TieredMultiBF16Wrapper::new(
TieredMulti::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_FLOAT16, true) => Box::new(TieredMultiFP16Wrapper::new(
TieredMulti::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_INT8, true) => Box::new(TieredMultiI8Wrapper::new(
TieredMulti::new(rust_params),
data_type,
)),
(VecSimType::VecSimType_UINT8, true) => Box::new(TieredMultiU8Wrapper::new(
TieredMulti::new(rust_params),
data_type,
)),
// INT32 and INT64 types not yet supported for vector indices
(VecSimType::VecSimType_INT32, _) | (VecSimType::VecSimType_INT64, _) => return None,
};

Some(Box::new(IndexHandle::new(
Expand Down
Loading