diff --git a/vortex-array/src/arrays/filter/execute/simd_compress/tests.rs b/vortex-array/src/arrays/filter/execute/simd_compress/tests.rs index 5ffc6f3c0c7..9800504d9ee 100644 --- a/vortex-array/src/arrays/filter/execute/simd_compress/tests.rs +++ b/vortex-array/src/arrays/filter/execute/simd_compress/tests.rs @@ -85,7 +85,7 @@ fn engages_on_supported_cpus() { let mask = mask_values(&mask).expect("alternating mask is mixed"); #[cfg(target_arch = "x86_64")] - let expected = is_x86_feature_detected!("avx2"); + let expected = is_x86_feature_detected!("avx2") && is_x86_feature_detected!("popcnt"); #[cfg(target_arch = "aarch64")] let expected = true; @@ -109,7 +109,7 @@ fn declines_sparse_and_short_masks() { #[cfg(all(target_arch = "x86_64", not(miri)))] #[test] fn avx2_kernels_match_scalar() { - if !is_x86_feature_detected!("avx2") { + if !(is_x86_feature_detected!("avx2") && is_x86_feature_detected!("popcnt")) { return; } @@ -126,7 +126,7 @@ fn avx2_kernels_match_scalar() { ); let mut out = vec![T::default(); mask.true_count() + SLACK_BYTES / size_of::()]; - // SAFETY: AVX2 was detected above and the output has a vector of slack. + // SAFETY: AVX2 and POPCNT were detected above and the output has a vector of slack. let written = unsafe { kernel_out_of_place(values.as_ptr().cast(), out.as_mut_ptr().cast(), mask) }; assert_eq!(written, mask.true_count()); @@ -134,7 +134,7 @@ fn avx2_kernels_match_scalar() { let mut compacted = values.to_vec(); let ptr = compacted.as_mut_ptr().cast::(); - // SAFETY: AVX2 was detected above; in-place compaction stays within the slice. + // SAFETY: AVX2 and POPCNT were detected above; in-place compaction stays within the slice. let written = unsafe { kernel_in_place(ptr.cast_const(), ptr, mask) }; assert_eq!(written, mask.true_count()); assert_eq!(&compacted[..written], expected.as_slice()); diff --git a/vortex-array/src/arrays/filter/execute/simd_compress/x86.rs b/vortex-array/src/arrays/filter/execute/simd_compress/x86.rs index 90518580e00..1bb02de76e4 100644 --- a/vortex-array/src/arrays/filter/execute/simd_compress/x86.rs +++ b/vortex-array/src/arrays/filter/execute/simd_compress/x86.rs @@ -65,17 +65,18 @@ pub(super) fn select_kernel(mask: &MaskValues) -> Optio } fn avx512f() -> bool { - is_x86_feature_detected!("avx512f") + is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("popcnt") } fn avx512_vbmi2() -> bool { is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512bw") && is_x86_feature_detected!("avx512vbmi2") + && is_x86_feature_detected!("popcnt") } fn avx2() -> bool { - is_x86_feature_detected!("avx2") + is_x86_feature_detected!("avx2") && is_x86_feature_detected!("popcnt") } /// Generate an AVX-512 kernel for one element width. @@ -187,7 +188,7 @@ macro_rules! avx512_compress_kernel { avx512_compress_kernel!( compress_word_avx512_epi8, compress_avx512_epi8, - features: "avx512f,avx512bw,avx512vbmi2", + features: "avx512f,avx512bw,avx512vbmi2,popcnt", elem: i8, lanes: 64, kmask: u64, @@ -200,7 +201,7 @@ avx512_compress_kernel!( avx512_compress_kernel!( compress_word_avx512_epi16, compress_avx512_epi16, - features: "avx512f,avx512bw,avx512vbmi2", + features: "avx512f,avx512bw,avx512vbmi2,popcnt", elem: i16, lanes: 32, kmask: u32, @@ -213,7 +214,7 @@ avx512_compress_kernel!( avx512_compress_kernel!( compress_word_avx512_epi32, compress_avx512_epi32, - features: "avx512f", + features: "avx512f,popcnt", elem: i32, lanes: 16, kmask: u16, @@ -226,7 +227,7 @@ avx512_compress_kernel!( avx512_compress_kernel!( compress_word_avx512_epi64, compress_avx512_epi64, - features: "avx512f", + features: "avx512f,popcnt", elem: i64, lanes: 8, kmask: u8, @@ -329,7 +330,7 @@ macro_rules! avx2_compress_kernel { clippy::cast_possible_truncation, reason = "deliberate submask narrowing" )] - #[target_feature(enable = "avx2")] + #[target_feature(enable = "avx2,popcnt")] #[inline] unsafe fn $word_fn( src: *const u8, @@ -409,7 +410,7 @@ macro_rules! avx2_compress_kernel { /// The CPU must support AVX2 and the pointer contract of /// [`filter_slice_by_bitmap`](super::filter_slice_by_bitmap) / /// [`filter_slice_mut_by_bitmap`](super::filter_slice_mut_by_bitmap) must hold. - #[target_feature(enable = "avx2")] + #[target_feature(enable = "avx2,popcnt")] pub(super) unsafe fn $walk_fn( src: *const u8, dst: *mut u8, diff --git a/vortex-mask/src/intersect_by_rank/bmi2.rs b/vortex-mask/src/intersect_by_rank/bmi2.rs index 97d33ada935..9e84f062e74 100644 --- a/vortex-mask/src/intersect_by_rank/bmi2.rs +++ b/vortex-mask/src/intersect_by_rank/bmi2.rs @@ -45,8 +45,8 @@ unsafe fn pdep_bmi2(source: u64, mask: u64) -> u64 { /// kernel without the feature calls an out-of-line `pdep_bmi2` once per word. /// /// # Safety -/// Requires BMI2. -#[target_feature(enable = "bmi2")] +/// Requires BMI2 and POPCNT. +#[target_feature(enable = "bmi2,popcnt")] pub(super) unsafe fn intersect_bit_buffers_bmi2( self_buffer: &BitBuffer, mask_buffer: &BitBuffer, @@ -58,8 +58,8 @@ pub(super) unsafe fn intersect_bit_buffers_bmi2( /// See [`intersect_bit_buffers_bmi2`]. /// /// # Safety -/// Requires BMI2. -#[target_feature(enable = "bmi2")] +/// Requires BMI2 and POPCNT. +#[target_feature(enable = "bmi2,popcnt")] pub(super) unsafe fn intersect_bit_buffer_by_rank_indices_bmi2( self_buffer: &BitBuffer, mask_indices: &[usize], @@ -70,8 +70,8 @@ pub(super) unsafe fn intersect_bit_buffer_by_rank_indices_bmi2( /// See [`intersect_bit_buffers_bmi2`]. /// /// # Safety -/// Requires BMI2. -#[target_feature(enable = "bmi2")] +/// Requires BMI2 and POPCNT. +#[target_feature(enable = "bmi2,popcnt")] pub(super) unsafe fn intersect_mask_driven_bmi2( self_buffer: &BitBuffer, mask_indices: I, diff --git a/vortex-mask/src/intersect_by_rank/mod.rs b/vortex-mask/src/intersect_by_rank/mod.rs index 5f040f794e2..c2bfcebfd38 100644 --- a/vortex-mask/src/intersect_by_rank/mod.rs +++ b/vortex-mask/src/intersect_by_rank/mod.rs @@ -443,7 +443,9 @@ type IntersectRankIndices = unsafe fn(&BitBuffer, &[usize]) -> Mask; fn select_bit_buffers() -> IntersectBuffers { #[cfg(target_arch = "x86_64")] { - if std::arch::is_x86_feature_detected!("bmi2") { + if std::arch::is_x86_feature_detected!("bmi2") + && std::arch::is_x86_feature_detected!("popcnt") + { return bmi2::intersect_bit_buffers_bmi2; } } @@ -460,7 +462,9 @@ fn select_bit_buffers() -> IntersectBuffers { fn select_rank_indices() -> IntersectRankIndices { #[cfg(target_arch = "x86_64")] { - if std::arch::is_x86_feature_detected!("bmi2") { + if std::arch::is_x86_feature_detected!("bmi2") + && std::arch::is_x86_feature_detected!("popcnt") + { return bmi2::intersect_bit_buffer_by_rank_indices_bmi2; } } @@ -523,8 +527,9 @@ where I: Iterator, { #[cfg(target_arch = "x86_64")] - if std::arch::is_x86_feature_detected!("bmi2") { - // SAFETY: BMI2 was just detected. + if std::arch::is_x86_feature_detected!("bmi2") && std::arch::is_x86_feature_detected!("popcnt") + { + // SAFETY: BMI2 and POPCNT were just detected. return unsafe { bmi2::intersect_mask_driven_bmi2(self_buffer, mask_indices, true_count) }; }