diff --git a/encodings/alp/benches/alp_compress.rs b/encodings/alp/benches/alp_compress.rs index c8745926845..8817daedbbc 100644 --- a/encodings/alp/benches/alp_compress.rs +++ b/encodings/alp/benches/alp_compress.rs @@ -66,9 +66,14 @@ fn compress_alp(bencher: Bencher, args: (usize, f64, let values = values.freeze(); let array = PrimitiveArray::new(values, validity); - bencher - .with_inputs(|| &array) - .bench_values(|array| alp_encode(array.as_view(), None).unwrap()) + bencher.with_inputs(|| &array).bench_values(|array| { + alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + }) } #[divan::bench(types = [f32, f64], args = BENCH_ARGS)] @@ -95,6 +100,7 @@ fn decompress_alp(bencher: Bencher, args: (usize, f64 alp_encode( PrimitiveArray::new(Buffer::copy_from(&values), validity.clone()).as_view(), None, + &mut LEGACY_SESSION.create_execution_ctx(), ) .unwrap(), LEGACY_SESSION.create_execution_ctx(), diff --git a/encodings/alp/public-api.lock b/encodings/alp/public-api.lock index ff4535970be..a2fa68e3cfa 100644 --- a/encodings/alp/public-api.lock +++ b/encodings/alp/public-api.lock @@ -636,7 +636,7 @@ pub fn f64::to_bits(value: Self) -> Self::UINT pub fn f64::to_u16(bits: Self::UINT) -> u16 -pub fn vortex_alp::alp_encode(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, exponents: core::option::Option) -> vortex_error::VortexResult +pub fn vortex_alp::alp_encode(parray: vortex_array::array::view::ArrayView<'_, vortex_array::arrays::primitive::vtable::Primitive>, exponents: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_alp::alp_rd_decode(left_parts: vortex_buffer::buffer_mut::BufferMut, left_parts_dict: &[u16], right_bit_width: u8, right_parts: vortex_buffer::buffer_mut::BufferMut<::UINT>, left_parts_patches: core::option::Option, ctx: &mut vortex_array::executor::ExecutionCtx) -> vortex_error::VortexResult> diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index be0db0f938e..8a86019dcf9 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -552,11 +552,11 @@ mod tests { #[case(2048)] #[case(2049)] fn test_execute_f32(#[case] size: usize) { + let mut ctx = SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter((0..size).map(|i| i as f32)); - let encoded = alp_encode(values.as_view(), None).unwrap(); + let encoded = alp_encode(values.as_view(), None, &mut ctx).unwrap(); let result_canonical = { - let mut ctx = SESSION.create_execution_ctx(); encoded .clone() .into_array() @@ -582,7 +582,12 @@ mod tests { #[case(2049)] fn test_execute_f64(#[case] size: usize) { let values = PrimitiveArray::from_iter((0..size).map(|i| i as f64)); - let encoded = alp_encode(values.as_view(), None).unwrap(); + let encoded = alp_encode( + values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let result_canonical = { let mut ctx = SESSION.create_execution_ctx(); @@ -616,7 +621,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().unwrap().array_len() > 0); let result_canonical = { @@ -650,7 +660,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let result_canonical = { let mut ctx = SESSION.create_execution_ctx(); @@ -685,7 +700,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().unwrap().array_len() > 0); let result_canonical = { @@ -721,7 +741,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values.clone()); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let slice_end = size - slice_start; let slice_len = slice_end - slice_start; @@ -772,7 +797,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values.clone()); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let slice_end = size - slice_start; let slice_len = slice_end - slice_start; @@ -783,7 +813,16 @@ mod tests { for idx in 0..slice_len { let expected_value = values[slice_start + idx]; - let result_valid = result_primitive.validity_mask().unwrap().value(idx); + let result_valid = result_primitive + .as_ref() + .validity() + .unwrap() + .to_mask( + result_primitive.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + .value(idx); assert_eq!( result_valid, expected_value.is_some(), @@ -813,7 +852,12 @@ mod tests { let original = PrimitiveArray::from_iter(values); // First encode normally to get a properly formed ALPArray with patches. - let normally_encoded = alp_encode(original.as_view(), None).unwrap(); + let normally_encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!( normally_encoded.patches().is_some(), "Test requires patches to be present" diff --git a/encodings/alp/src/alp/compress.rs b/encodings/alp/src/alp/compress.rs index f67779996da..059c4b8821c 100644 --- a/encodings/alp/src/alp/compress.rs +++ b/encodings/alp/src/alp/compress.rs @@ -4,6 +4,7 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; use vortex_array::IntoArray; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; @@ -44,10 +45,11 @@ macro_rules! match_each_alp_float_ptype { pub fn alp_encode( parray: ArrayView<'_, Primitive>, exponents: Option, + ctx: &mut ExecutionCtx, ) -> VortexResult { let (exponents, encoded, patches) = match parray.ptype() { - PType::F32 => alp_encode_components_typed::(parray, exponents)?, - PType::F64 => alp_encode_components_typed::(parray, exponents)?, + PType::F32 => alp_encode_components_typed::(parray, exponents, ctx)?, + PType::F64 => alp_encode_components_typed::(parray, exponents, ctx)?, _ => vortex_bail!("ALP can only encode f32 and f64"), }; @@ -62,6 +64,7 @@ pub fn alp_encode( fn alp_encode_components_typed( values: ArrayView<'_, Primitive>, exponents: Option, + ctx: &mut ExecutionCtx, ) -> VortexResult<(Exponents, ArrayRef, Option)> where T: ALPFloat, @@ -73,7 +76,10 @@ where let encoded_array = PrimitiveArray::new(encoded, values.validity()?).into_array(); - let validity = values.array().validity_mask()?; + let validity = values + .array() + .validity()? + .to_mask(values.array().len(), ctx)?; // exceptional_positions may contain exceptions at invalid positions (which contain garbage // data). We remove null exceptions in order to keep the Patches small. let (valid_exceptional_positions, valid_exceptional_values): (Buffer, Buffer) = @@ -146,7 +152,12 @@ mod tests { #[test] fn test_compress() { let array = PrimitiveArray::new(buffer![1.234f32; 1025], Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_iter(vec![1234i32; 1025]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -160,7 +171,12 @@ mod tests { #[test] fn test_nullable_compress() { let array = PrimitiveArray::from_option_iter([None, Some(1.234f32), None]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_option_iter([None, Some(1234i32), None]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -177,7 +193,12 @@ mod tests { fn test_patched_compress() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values.clone(), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); let expected_encoded = PrimitiveArray::from_iter(vec![1234i64, 2718, 1234, 4000]); assert_arrays_eq!(encoded.encoded(), expected_encoded); @@ -194,7 +215,12 @@ mod tests { fn test_compress_ignores_invalid_exceptional_values() { let values = buffer![1.234f64, 2.718, PI, 4.0]; let array = PrimitiveArray::new(values, Validity::from_iter([true, true, false, true])); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let expected_encoded = PrimitiveArray::from_option_iter(buffer![Some(1234i64), Some(2718), None, Some(4000)]); @@ -216,7 +242,12 @@ mod tests { Some(4.0), None, ]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); assert_eq!(encoded.exponents(), Exponents { e: 16, f: 13 }); @@ -230,7 +261,12 @@ mod tests { #[test] fn roundtrips_close_fractional() { let original = PrimitiveArray::from_iter([195.26274f32, 195.27837, -48.815685]); - let alp_arr = alp_encode(original.as_view(), None).unwrap(); + let alp_arr = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert_arrays_eq!(alp_arr, original); } @@ -238,7 +274,12 @@ mod tests { fn roundtrips_all_null() { let original = PrimitiveArray::new(buffer![195.26274f64, PI, -48.815685], Validity::AllInvalid); - let alp_arr = alp_encode(original.as_view(), None).unwrap(); + let alp_arr = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decompressed = alp_arr.into_array().to_primitive(); assert_eq!( @@ -256,7 +297,12 @@ mod tests { buffer![0.0f32, -0.0, f32::NAN, f32::NEG_INFINITY, f32::INFINITY], Validity::NonNullable, ); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decoded = encoded.as_array().to_primitive(); for idx in 0..original.len() { let decoded_val = decoded.as_slice::()[idx]; @@ -277,7 +323,12 @@ mod tests { values[1025] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -300,7 +351,12 @@ mod tests { values[2048] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -322,7 +378,12 @@ mod tests { values[0] = PI; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -345,7 +406,12 @@ mod tests { values[100] = E; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let patches = encoded.patches().unwrap(); let chunk_offsets = patches.chunk_offsets().clone().unwrap().to_primitive(); @@ -366,7 +432,12 @@ mod tests { // Create 1024 elements, encode, slice to first 512, then decode let values = vec![1.234f32; 1024]; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -378,7 +449,12 @@ mod tests { fn test_slice_half_chunk_f64_roundtrip() { let values = vec![5.678f64; 1024]; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -394,7 +470,12 @@ mod tests { values[600] = 42.42; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); @@ -414,7 +495,12 @@ mod tests { values[1023] = 42.42; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(1023..1025).unwrap(); @@ -430,7 +516,12 @@ mod tests { .collect::>(); let original = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let sliced_alp = encoded.slice(512..1024).unwrap(); let decoded = sliced_alp.to_primitive(); @@ -443,7 +534,12 @@ mod tests { fn test_large_f32_array_uniform_values() { let size = 10_000; let array = PrimitiveArray::new(buffer![42.125f32; size], Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let decoded = @@ -455,7 +551,12 @@ mod tests { fn test_large_f64_array_uniform_values() { let size = 50_000; let array = PrimitiveArray::new(buffer![123.456789f64; size], Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); let decoded = @@ -473,7 +574,12 @@ mod tests { values[4500] = f32::INFINITY; let array = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); let decoded = @@ -495,7 +601,12 @@ mod tests { values[7000] = 999.999999999; let array = PrimitiveArray::new(Buffer::from(values.clone()), Validity::NonNullable); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); let decoded = @@ -525,7 +636,12 @@ mod tests { .collect(); let array = PrimitiveArray::from_option_iter(values); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decoded = decompress_into_array(encoded, &mut LEGACY_SESSION.create_execution_ctx()).unwrap(); @@ -546,7 +662,12 @@ mod tests { let validity = Validity::from_iter((0..size).map(|i| !matches!(i, 500 | 2500))); let array = PrimitiveArray::new(Buffer::from(values), validity); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let decoded = decompress_into_array(encoded, &mut LEGACY_SESSION.create_execution_ctx()).unwrap(); @@ -577,7 +698,12 @@ mod tests { values[2900] = PI; let original = PrimitiveArray::new(Buffer::from(values), Validity::NonNullable); - let encoded = alp_encode(original.as_view(), None).unwrap(); + let encoded = alp_encode( + original.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); // Slice ending mid-chunk-2 (element 2500 is inside chunk 2 = 2048..3072). diff --git a/encodings/alp/src/alp/compute/between.rs b/encodings/alp/src/alp/compute/between.rs index b9281140c1d..7f35f67d51c 100644 --- a/encodings/alp/src/alp/compute/between.rs +++ b/encodings/alp/src/alp/compute/between.rs @@ -95,6 +95,8 @@ where #[cfg(test)] mod tests { + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -123,7 +125,12 @@ mod tests { fn comparison_range() { let value = 0.0605_f32; let array = PrimitiveArray::from_iter([value; 1]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_between( diff --git a/encodings/alp/src/alp/compute/cast.rs b/encodings/alp/src/alp/compute/cast.rs index b663b861c79..b6ce03b9eee 100644 --- a/encodings/alp/src/alp/compute/cast.rs +++ b/encodings/alp/src/alp/compute/cast.rs @@ -60,7 +60,9 @@ impl CastReduce for ALP { mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; use vortex_array::builtins::ArrayBuiltins; @@ -78,7 +80,11 @@ mod tests { #[test] fn issue_5766_test_cast_alp_with_patches_to_nullable() -> VortexResult<()> { let values = buffer![1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456].into_array(); - let alp = alp_encode(values.to_primitive().as_view(), None)?; + let alp = alp_encode( + values.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!( alp.patches().is_some(), @@ -98,7 +104,11 @@ mod tests { #[test] fn test_cast_alp_f32_to_f64() -> VortexResult<()> { let values = buffer![1.5f32, 2.5, 3.5, 4.5].into_array(); - let alp = alp_encode(values.to_primitive().as_view(), None)?; + let alp = alp_encode( + values.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let casted = alp .into_array() @@ -120,7 +130,11 @@ mod tests { #[test] fn test_cast_alp_to_int() -> VortexResult<()> { let values = buffer![1.0f32, 2.0, 3.0, 4.0].into_array(); - let alp = alp_encode(values.to_primitive().as_view(), None)?; + let alp = alp_encode( + values.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let casted = alp .into_array() @@ -143,7 +157,12 @@ mod tests { #[case(buffer![42.42f64].into_array())] #[case(buffer![0.0f32, -1.5, 2.5, -3.5, 4.5].into_array())] fn test_cast_alp_conformance(#[case] array: vortex_array::ArrayRef) -> VortexResult<()> { - let alp = alp_encode(array.to_primitive().as_view(), None).vortex_expect("cannot fail"); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("cannot fail"); test_cast_conformance(&alp.into_array()); Ok(()) diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index a6f475a3df2..ebda0a7797c 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -152,7 +152,9 @@ where mod tests { use rstest::rstest; use vortex_array::ArrayRef; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; @@ -183,7 +185,12 @@ mod tests { #[test] fn basic_comparison_test() { let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -206,7 +213,12 @@ mod tests { #[test] fn comparison_with_unencodable_value() { let array = PrimitiveArray::from_iter([1.234f32; 1025]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -229,7 +241,12 @@ mod tests { #[test] fn comparison_range() { let array = PrimitiveArray::from_iter([0.0605_f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -268,7 +285,12 @@ mod tests { #[test] fn comparison_zeroes() { let array = PrimitiveArray::from_iter([0.0_f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_none()); assert_eq!( encoded.encoded().to_primitive().as_slice::(), @@ -310,7 +332,12 @@ mod tests { fn compare_with_patches() { let array = PrimitiveArray::from_iter([1.234f32, 1.5, 19.0, std::f32::consts::E, 1_000_000.9]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(encoded.patches().is_some()); // Not supported! @@ -324,7 +351,12 @@ mod tests { #[test] fn compare_to_null() { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let other = ConstantArray::new( Scalar::null(DType::Primitive(PType::F32, Nullability::Nullable)), @@ -347,7 +379,12 @@ mod tests { #[case(f32::NEG_INFINITY, true)] fn compare_to_non_finite_gt(#[case] value: f32, #[case] result: bool) { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let r = test_alp_compare(encoded.as_view(), value, CompareOperator::Gt).unwrap(); let expected = BoolArray::from_iter([result; 10]); @@ -361,7 +398,12 @@ mod tests { #[case(f32::NEG_INFINITY, false)] fn compare_to_non_finite_lt(#[case] value: f32, #[case] result: bool) { let array = PrimitiveArray::from_iter([1.234f32; 10]); - let encoded = alp_encode(array.as_view(), None).unwrap(); + let encoded = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let r = test_alp_compare(encoded.as_view(), value, CompareOperator::Lt).unwrap(); let expected = BoolArray::from_iter([result; 10]); diff --git a/encodings/alp/src/alp/compute/filter.rs b/encodings/alp/src/alp/compute/filter.rs index 622a8d59b6b..c305ea5e99b 100644 --- a/encodings/alp/src/alp/compute/filter.rs +++ b/encodings/alp/src/alp/compute/filter.rs @@ -44,7 +44,9 @@ mod test { use rstest::rstest; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::filter::test_filter_conformance; use vortex_buffer::buffer; @@ -61,7 +63,12 @@ mod test { 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0 ].into_array())] fn test_filter_alp_conformance(#[case] array: ArrayRef) { - let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); test_filter_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp/compute/mask.rs b/encodings/alp/src/alp/compute/mask.rs index 137aa263f53..7b9757376d9 100644 --- a/encodings/alp/src/alp/compute/mask.rs +++ b/encodings/alp/src/alp/compute/mask.rs @@ -79,7 +79,12 @@ mod test { 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0 ].into_array())] fn test_mask_alp_conformance(#[case] array: vortex_array::ArrayRef) { - let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); test_mask_conformance(&alp.into_array()); } @@ -91,7 +96,12 @@ mod test { .map(|i| if i % 4 == 3 { PI } else { 1.0 }) .collect(); let array = PrimitiveArray::from_iter(values); - let alp = alp_encode(array.as_view(), None).unwrap(); + let alp = alp_encode( + array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(alp.patches().is_some(), "expected patches"); test_mask_conformance(&alp.into_array()); } @@ -99,7 +109,12 @@ mod test { #[test] fn test_mask_alp_with_patches_casts_surviving_patch_values_to_nullable() { let values = PrimitiveArray::from_iter([1.234f32, f32::NAN, 2.345, f32::INFINITY, 3.456]); - let alp = alp_encode(values.as_view(), None).unwrap(); + let alp = alp_encode( + values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(alp.patches().is_some(), "expected patches"); let keep_mask = BoolArray::from_iter([false, true, true, true, true]).into_array(); diff --git a/encodings/alp/src/alp/compute/mod.rs b/encodings/alp/src/alp/compute/mod.rs index 14e118a870a..e5e5b657740 100644 --- a/encodings/alp/src/alp/compute/mod.rs +++ b/encodings/alp/src/alp/compute/mod.rs @@ -14,6 +14,8 @@ mod take; mod tests { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array; use vortex_array::compute::conformance::consistency::test_array_consistency; @@ -23,27 +25,27 @@ mod tests { #[rstest] // Basic float arrays - #[case::f32_array(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None).unwrap())] - #[case::f64_array(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None).unwrap())] + #[case::f32_array(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f64_array(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Nullable arrays - #[case::nullable_f32(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).as_view(), None).unwrap())] - #[case::nullable_f64(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]).as_view(), None).unwrap())] + #[case::nullable_f32(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::nullable_f64(alp_encode(PrimitiveArray::from_option_iter([Some(1.1f64), None, Some(2.2), Some(3.3), None]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Edge cases - #[case::single_element(alp_encode(PrimitiveArray::from_iter([42.42f64]).as_view(), None).unwrap())] + #[case::single_element(alp_encode(PrimitiveArray::from_iter([42.42f64]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Large arrays - #[case::large_f32(alp_encode(PrimitiveArray::from_iter((0..1000).map(|i| i as f32 * 0.1)).as_view(), None).unwrap())] + #[case::large_f32(alp_encode(PrimitiveArray::from_iter((0..1000).map(|i| i as f32 * 0.1)).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] // Arrays with patterns - #[case::repeating_pattern(alp_encode(PrimitiveArray::from_iter([1.1f32, 2.2, 3.3, 1.1, 2.2, 3.3, 1.1, 2.2, 3.3]).as_view(), None).unwrap())] - #[case::close_values(alp_encode(PrimitiveArray::from_iter([100.001f64, 100.002, 100.003, 100.004, 100.005]).as_view(), None).unwrap())] + #[case::repeating_pattern(alp_encode(PrimitiveArray::from_iter([1.1f32, 2.2, 3.3, 1.1, 2.2, 3.3, 1.1, 2.2, 3.3]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::close_values(alp_encode(PrimitiveArray::from_iter([100.001f64, 100.002, 100.003, 100.004, 100.005]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] fn test_alp_consistency(#[case] array: ALPArray) { test_array_consistency(&array.into_array()); } #[rstest] - #[case::f32_basic(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None).unwrap())] - #[case::f64_basic(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None).unwrap())] - #[case::f32_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f32 * 1.5)).as_view(), None).unwrap())] - #[case::f64_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f64 * 2.5)).as_view(), None).unwrap())] + #[case::f32_basic(alp_encode(PrimitiveArray::from_iter([1.23f32, 4.56, 7.89, 10.11, 12.13]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f64_basic(alp_encode(PrimitiveArray::from_iter([100.1f64, 200.2, 300.3, 400.4, 500.5]).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f32_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f32 * 1.5)).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] + #[case::f64_large(alp_encode(PrimitiveArray::from_iter((0..100).map(|i| i as f64 * 2.5)).as_view(), None, &mut LEGACY_SESSION.create_execution_ctx()).unwrap())] fn test_alp_binary_numeric(#[case] array: ALPArray) { test_binary_numeric_array(array.into_array()); } diff --git a/encodings/alp/src/alp/compute/take.rs b/encodings/alp/src/alp/compute/take.rs index f3a7d23fbe4..3fcfa7c9f77 100644 --- a/encodings/alp/src/alp/compute/take.rs +++ b/encodings/alp/src/alp/compute/take.rs @@ -42,7 +42,9 @@ impl TakeExecute for ALP { mod test { use rstest::rstest; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::compute::conformance::take::test_take_conformance; use vortex_buffer::buffer; @@ -55,7 +57,12 @@ mod test { #[case(PrimitiveArray::from_option_iter([Some(1.1f32), None, Some(2.2), Some(3.3), None]).into_array())] #[case(buffer![42.42f64].into_array())] fn test_take_alp_conformance(#[case] array: vortex_array::ArrayRef) { - let alp = alp_encode(array.to_primitive().as_view(), None).unwrap(); + let alp = alp_encode( + array.to_primitive().as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); test_take_conformance(&alp.into_array()); } } diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index 2ac3ac85c54..25266bef821 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -93,6 +93,7 @@ mod tests { use vortex_array::ArrayPlugin; use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::PatchedArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::PatchedArraySlotsExt; @@ -124,7 +125,7 @@ mod tests { .collect(); let parray = PrimitiveArray::from_iter(values); - let alp_encoded = alp_encode(parray.as_view(), None)?; + let alp_encoded = alp_encode(parray.as_view(), None, &mut SESSION.create_execution_ctx())?; assert!( alp_encoded.patches().is_some(), @@ -173,7 +174,7 @@ mod tests { // Values that encode cleanly without patches. let values: Vec = (0..100).map(|i| i as f64).collect(); let parray = PrimitiveArray::from_iter(values); - let alp_encoded = alp_encode(parray.as_view(), None)?; + let alp_encoded = alp_encode(parray.as_view(), None, &mut SESSION.create_execution_ctx())?; assert!( alp_encoded.patches().is_none(), diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index be5f5f9a918..0cdac5c4001 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -261,7 +261,10 @@ impl VTable for ALPRD { // Decode the left_parts using our builtin dictionary. let left_parts_dict = left_parts_dictionary; - let validity = left_parts.validity_mask()?; + let validity = left_parts + .as_ref() + .validity()? + .to_mask(left_parts.as_ref().len(), ctx)?; let decoded_array = if ptype == PType::F32 { PrimitiveArray::new( diff --git a/encodings/bytebool/public-api.lock b/encodings/bytebool/public-api.lock index 514f8742a28..8edd17d2912 100644 --- a/encodings/bytebool/public-api.lock +++ b/encodings/bytebool/public-api.lock @@ -122,12 +122,8 @@ pub trait vortex_bytebool::ByteBoolArrayExt: vortex_array::array::typed::TypedAr pub fn vortex_bytebool::ByteBoolArrayExt::validity(&self) -> vortex_array::validity::Validity -pub fn vortex_bytebool::ByteBoolArrayExt::validity_mask(&self) -> vortex_mask::Mask - impl> vortex_bytebool::ByteBoolArrayExt for T pub fn T::validity(&self) -> vortex_array::validity::Validity -pub fn T::validity_mask(&self) -> vortex_mask::Mask - pub type vortex_bytebool::ByteBoolArray = vortex_array::array::typed::Array diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index e98c5bd656d..42cc48c094e 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -35,7 +35,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; -use vortex_mask::Mask; use vortex_session::VortexSession; use vortex_session::registry::CachedId; @@ -190,10 +189,6 @@ pub trait ByteBoolArrayExt: TypedArrayRef { self.as_ref().dtype().nullability(), ) } - - fn validity_mask(&self) -> Mask { - self.validity().to_mask(self.as_ref().len()) - } } impl> ByteBoolArrayExt for T {} diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index 5f232f61196..49f518a7949 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -11,7 +11,9 @@ use vortex_alp::ALPArraySlotsExt; use vortex_alp::alp_encode; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; use vortex_error::VortexExpect; @@ -49,7 +51,12 @@ fn generate_alp_bit_pack_primitive_array( .map(|_| T::from_usize(rng.random_range(0..10_000)).vortex_expect("")) .collect::(); - let alp = alp_encode(a.as_view(), None).vortex_expect(""); + let alp = alp_encode( + a.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect(""); let encoded = alp.encoded().to_primitive(); diff --git a/encodings/fastlanes/public-api.lock b/encodings/fastlanes/public-api.lock index cf7776cd872..0a84e217c76 100644 --- a/encodings/fastlanes/public-api.lock +++ b/encodings/fastlanes/public-api.lock @@ -578,8 +578,6 @@ pub fn vortex_fastlanes::BitPackedArrayExt::unpacked_chunks vortex_array::validity::Validity -pub fn vortex_fastlanes::BitPackedArrayExt::validity_mask(&self) -> vortex_mask::Mask - impl> vortex_fastlanes::BitPackedArrayExt for T pub fn T::bit_width(&self) -> u8 @@ -596,8 +594,6 @@ pub fn T::unpacked_chunks(&self) -> pub fn T::validity(&self) -> vortex_array::validity::Validity -pub fn T::validity_mask(&self) -> vortex_mask::Mask - pub trait vortex_fastlanes::BitPackedArraySlotsExt: vortex_array::array::typed::TypedArrayRef pub fn vortex_fastlanes::BitPackedArraySlotsExt::patch_chunk_offsets(&self) -> core::option::Option<&vortex_array::array::erased::ArrayRef> diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index b2b6ce8648c..7c7e1bc5d26 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -6,6 +6,8 @@ use itertools::Itertools; use num_traits::PrimInt; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::Primitive; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::PrimitiveArrayExt; @@ -200,7 +202,10 @@ pub fn gather_patches( }; let array_len = parray.len(); - let validity_mask = parray.validity_mask()?; + let validity_mask = parray + .as_ref() + .validity()? + .to_mask(parray.len(), &mut LEGACY_SESSION.create_execution_ctx())?; let patches = if array_len < u8::MAX as usize { match_each_integer_ptype!(parray.ptype(), |T| { @@ -302,7 +307,14 @@ fn bit_width_histogram_typed( |v: T| (8 * size_of::()) - (PrimInt::leading_zeros(v) as usize); let mut bit_widths = vec![0usize; size_of::() * 8 + 1]; - match array.validity_mask().bit_buffer() { + match array + .validity()? + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .bit_buffer() + { AllOr::All => { // All values are valid. for v in array.as_slice::() { @@ -457,7 +469,13 @@ mod test { assert_eq!( (0..(1 << 4)).collect::>(), compressed - .validity_mask() + .as_ref() + .validity() + .unwrap() + .to_mask( + compressed.as_ref().len(), + &mut SESSION.create_execution_ctx() + ) .unwrap() .to_bit_buffer() .set_indices() diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index 5ed8d6b89fa..465c467bcfb 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -58,7 +58,7 @@ pub(crate) fn unpack_into_primitive_builder( // SAFETY: We later initialize the the uninitialized range of values with `copy_from_slice`. unsafe { // Append a dense null Mask. - uninit_range.append_mask(array.validity_mask()); + uninit_range.append_mask(array.validity()?.to_mask(array.as_ref().len(), ctx)?); } // SAFETY: `decode_into` will initialize all values in this range. diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index ca86e78e148..dbca8c0e48b 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -317,11 +317,6 @@ pub trait BitPackedArrayExt: BitPackedArraySlotsExt { ) } - #[inline] - fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.as_ref().len()) - } - #[inline] fn packed_slice(&self) -> &[T] { BitPackedData::packed_slice::(self) diff --git a/encodings/fastlanes/src/bitpacking/compute/take.rs b/encodings/fastlanes/src/bitpacking/compute/take.rs index 5a3177df6c1..5eb03415c88 100644 --- a/encodings/fastlanes/src/bitpacking/compute/take.rs +++ b/encodings/fastlanes/src/bitpacking/compute/take.rs @@ -280,7 +280,13 @@ mod test { taken_primitive, PrimitiveArray::from_option_iter([Some(1i32), Some(2), None, Some(4)]) ); - assert_eq!(taken_primitive.to_primitive().invalid_count().unwrap(), 1); + assert_eq!( + taken_primitive + .to_primitive() + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1 + ); } #[rstest] diff --git a/encodings/fastlanes/src/for/array/for_decompress.rs b/encodings/fastlanes/src/for/array/for_decompress.rs index e19ef8d0ff3..070bef28d32 100644 --- a/encodings/fastlanes/src/for/array/for_decompress.rs +++ b/encodings/fastlanes/src/for/array/for_decompress.rs @@ -109,7 +109,7 @@ pub(crate) fn fused_decompress< let mut uninit_range = builder.uninit_range(bp.len()); unsafe { // Append a dense null Mask. - uninit_range.append_mask(bp.validity_mask()); + uninit_range.append_mask(bp.validity()?.to_mask(bp.as_ref().len(), ctx)?); } // SAFETY: `decode_into` will initialize all values in this range. diff --git a/encodings/fastlanes/src/rle/array/mod.rs b/encodings/fastlanes/src/rle/array/mod.rs index a44ae69ddbd..9857874d040 100644 --- a/encodings/fastlanes/src/rle/array/mod.rs +++ b/encodings/fastlanes/src/rle/array/mod.rs @@ -300,7 +300,14 @@ mod tests { .vortex_expect("RLEData is always valid"); let sliced_array = rle_array.slice(1..4).unwrap(); - let validity_mask = sliced_array.validity_mask().unwrap(); + let validity_mask = sliced_array + .validity() + .unwrap() + .to_mask( + sliced_array.len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let mut ctx = LEGACY_SESSION.create_execution_ctx(); let expected_mask = Validity::from_iter([false, true, false]) diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index d3a51a254e9..ed605f47818 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -288,7 +288,14 @@ impl VTable for FSST { let next_buffer_index = builder.completed_block_count() + u32::from(builder.in_progress()); let (buffers, views) = fsst_decode_views(array, next_buffer_index, ctx)?; - builder.push_buffer_and_adjusted_views(&buffers, &views, array.array().validity_mask()?); + builder.push_buffer_and_adjusted_views( + &buffers, + &views, + array + .array() + .validity()? + .to_mask(array.array().len(), ctx)?, + ); Ok(()) } diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 7c8e74c825c..904c6e4dc13 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -258,7 +258,10 @@ pub(crate) fn number_type_from_ptype(ptype: PType) -> NumberType { } fn collect_valid(parray: ArrayView<'_, Primitive>) -> VortexResult { - let mask = parray.array().validity_mask()?; + let mask = parray.array().validity()?.to_mask( + parray.array().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(parray.array().filter(mask)?.to_primitive()) } diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index d694a0efdbe..b21bf121afc 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -164,12 +164,21 @@ fn test_validity_vtable() { Validity::Array(BoolArray::from_iter(mask_bools.clone()).into_array()), ); let compressed = Pco::from_primitive(array.as_view(), 3, 0).unwrap(); + let arr = compressed.as_array(); assert_eq!( - compressed.as_array().validity_mask().unwrap(), + arr.validity() + .unwrap() + .to_mask(arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(mask_bools) ); + let sliced = compressed.slice(1..4).unwrap(); assert_eq!( - compressed.slice(1..4).unwrap().validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![true, true, false]) ); } diff --git a/encodings/runend/benches/run_end_null_count.rs b/encodings/runend/benches/run_end_null_count.rs index f1473cc47aa..6fa318c7192 100644 --- a/encodings/runend/benches/run_end_null_count.rs +++ b/encodings/runend/benches/run_end_null_count.rs @@ -8,6 +8,8 @@ use rand::RngExt; use rand::SeedableRng; use rand::rngs::StdRng; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_buffer::Buffer; use vortex_runend::RunEnd; @@ -49,9 +51,11 @@ const BENCH_ARGS: &[(usize, usize, f64)] = &[ fn null_count_run_end(bencher: Bencher, (n, run_step, valid_density): (usize, usize, f64)) { let array = fixture(n, run_step, valid_density).into_array(); - bencher - .with_inputs(|| &array) - .bench_refs(|array| array.invalid_count().unwrap()); + bencher.with_inputs(|| &array).bench_refs(|array| { + array + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + }); } fn fixture(n: usize, run_step: usize, valid_density: f64) -> RunEndArray { diff --git a/encodings/runend/src/compress.rs b/encodings/runend/src/compress.rs index df86d6cc5fb..65739507fd9 100644 --- a/encodings/runend/src/compress.rs +++ b/encodings/runend/src/compress.rs @@ -5,7 +5,9 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::ArrayView; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::Primitive; @@ -177,7 +179,10 @@ pub fn runend_decode_primitive( offset: usize, length: usize, ) -> VortexResult { - let validity_mask = values.validity_mask()?; + let validity_mask = values.as_ref().validity()?.to_mask( + values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(match_each_native_ptype!(values.ptype(), |P| { match_each_unsigned_integer_ptype!(ends.ptype(), |E| { runend_decode_typed_primitive( @@ -277,7 +282,10 @@ pub fn runend_decode_varbinview( offset: usize, length: usize, ) -> VortexResult { - let validity_mask = values.validity_mask()?; + let validity_mask = values.as_ref().validity()?.to_mask( + values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let views = values.views(); let (decoded_views, validity) = match_each_unsigned_integer_ptype!(ends.ptype(), |E| { diff --git a/encodings/runend/src/decompress_bool.rs b/encodings/runend/src/decompress_bool.rs index 6733f143a8c..69876b0a273 100644 --- a/encodings/runend/src/decompress_bool.rs +++ b/encodings/runend/src/decompress_bool.rs @@ -9,6 +9,8 @@ use itertools::Itertools; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; @@ -36,7 +38,10 @@ pub fn runend_decode_bools( offset: usize, length: usize, ) -> VortexResult { - let validity = values.validity_mask()?; + let validity = values.as_ref().validity()?.to_mask( + values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let values_buf = values.to_bit_buffer(); let nullability = values.dtype().nullability(); @@ -241,7 +246,9 @@ fn decode_nullable_sequential( #[cfg(test)] mod tests { + use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::bool::BoolArrayExt; @@ -371,12 +378,42 @@ mod tests { // Check length and a few values assert_eq!(decoded.len(), 10000); // First run: valid true - assert!(decoded.validity_mask()?.value(0)); + assert!( + decoded + .as_ref() + .validity()? + .to_mask( + decoded.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .value(0) + ); assert!(decoded.to_bit_buffer().value(0)); // Second run: null (validity false) - assert!(!decoded.validity_mask()?.value(2000)); + assert!( + !decoded + .as_ref() + .validity()? + .to_mask( + decoded.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .value(2000) + ); // Third run: valid true - assert!(decoded.validity_mask()?.value(4000)); + assert!( + decoded + .as_ref() + .validity()? + .to_mask( + decoded.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .value(4000) + ); assert!(decoded.to_bit_buffer().value(4000)); Ok(()) } diff --git a/encodings/sequence/src/compute/take.rs b/encodings/sequence/src/compute/take.rs index ab3a4ce5f97..7610b1dd038 100644 --- a/encodings/sequence/src/compute/take.rs +++ b/encodings/sequence/src/compute/take.rs @@ -77,7 +77,7 @@ impl TakeExecute for Sequence { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let mask = indices.validity_mask()?; + let mask = indices.validity()?.to_mask(indices.len(), ctx)?; let indices = indices.clone().execute::(ctx)?; let result_nullability = array.dtype().nullability() | indices.dtype().nullability(); diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index c53fd8989e8..04ba70398f6 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -148,7 +148,10 @@ fn execute_sparse_lists( let n_filled = array.len() - resolved_patches.num_patches(); let total_canonical_values = values.elements().len() + fill_value.len() * n_filled; - let validity = Validity::from_mask(array.as_array().validity_mask()?, nullability); + let validity = { + let arr = array.as_array(); + Validity::from_mask(arr.validity()?.to_mask(arr.len(), ctx)?, nullability) + }; Ok(match_each_integer_ptype!(indices.ptype(), |I| { match_smallest_offset_type!(total_canonical_values, |O| { @@ -233,7 +236,10 @@ fn execute_sparse_fixed_size_list( .execute::(ctx)?; let fill_value = array.fill_scalar().as_list(); - let validity = Validity::from_mask(array.as_array().validity_mask()?, nullability); + let validity = { + let arr = array.as_array(); + Validity::from_mask(arr.validity()?.to_mask(arr.len(), ctx)?, nullability) + }; Ok(match_each_integer_ptype!(indices.ptype(), |I| { execute_sparse_fixed_size_list_inner::( @@ -420,10 +426,13 @@ fn execute_sparse_struct( unresolved_patches.offset(), unresolved_patches.indices(), &Validity::from_mask( - unresolved_patches - .values() - .validity_mask() - .vortex_expect("validity_mask"), + { + let v = unresolved_patches.values(); + v.validity() + .vortex_expect("validity_mask") + .to_mask(v.len(), ctx) + .vortex_expect("Failed to compute validity mask") + }, Nullability::Nullable, ), ctx, @@ -490,7 +499,13 @@ fn execute_varbin( let patches = array.resolved_patches()?; let indices = patches.indices().clone().execute::(ctx)?; let values = patches.values().clone().execute::(ctx)?; - let validity = Validity::from_mask(array.as_array().validity_mask()?, dtype.nullability()); + let validity = { + let arr = array.as_array(); + Validity::from_mask( + arr.validity()?.to_mask(arr.len(), ctx)?, + dtype.nullability(), + ) + }; let len = array.len(); Ok(match_each_integer_ptype!(indices.ptype(), |I| { diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index 845d918a7a3..736f173e894 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -19,8 +19,10 @@ use vortex_array::ArrayView; use vortex_array::ExecutionCtx; use vortex_array::ExecutionResult; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::Precision; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::buffer::BufferHandle; @@ -402,7 +404,9 @@ impl SparseData { fill_value.dtype() ) } - let mask = array.validity_mask()?; + let mask = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?; if mask.all_false() { // Array is constant NULL @@ -512,6 +516,8 @@ impl ValidityVTable for Sparse { mod test { use itertools::Itertools; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; @@ -592,7 +598,11 @@ mod test { pub fn validity_mask_sliced_null_fill() { let sliced = sparse_array(nullable_fill()).slice(2..7).unwrap(); assert_eq!( - sliced.validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![true, false, false, true, false]) ); } @@ -614,7 +624,11 @@ mod test { .unwrap(); assert_eq!( - sliced.validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![false, true, true, false, true]) ); } @@ -639,7 +653,9 @@ mod test { let array = sparse_array(nullable_fill()); assert_eq!( array - .validity_mask() + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) .unwrap() .to_bit_buffer() .iter() @@ -653,7 +669,14 @@ mod test { #[test] fn sparse_validity_mask_non_null_fill() { let array = sparse_array(non_nullable_fill()); - assert!(array.validity_mask().unwrap().all_true()); + assert!( + array + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -684,7 +707,11 @@ mod test { let sparse = Sparse::encode(&original.clone().into_array(), None) .vortex_expect("Sparse::encode should succeed for test data"); assert_eq!( - sparse.validity_mask().unwrap(), + sparse + .validity() + .unwrap() + .to_mask(sparse.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![ true, true, false, true, false, true, false, true, true, false, true, false, ]) @@ -698,7 +725,11 @@ mod test { let values = PrimitiveArray::from_option_iter([Some(0i16), Some(1), None, None, Some(4)]) .into_array(); let array = Sparse::try_new(indices, values, 10, Scalar::null_native::()).unwrap(); - let actual = array.validity_mask().unwrap(); + let actual = array + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(); let expected = Mask::from_iter([ true, false, true, false, false, false, false, false, true, false, ]); diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 789a0ae91ad..9f3d19e1bd2 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -366,12 +366,18 @@ fn choose_max_dict_size(uncompressed_size: usize) -> usize { } fn collect_valid_primitive(parray: &PrimitiveArray) -> VortexResult { - let mask = parray.validity_mask()?; + let mask = parray.as_ref().validity()?.to_mask( + parray.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(parray.filter(mask)?.to_primitive()) } fn collect_valid_vbv(vbv: &VarBinViewArray) -> VortexResult<(ByteBuffer, Vec)> { - let mask = vbv.validity_mask()?; + let mask = vbv.as_ref().validity()?.to_mask( + vbv.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let buffer_and_value_byte_indices = match mask.bit_buffer() { AllOr::None => (Buffer::empty(), Vec::new()), _ => { diff --git a/encodings/zstd/src/test.rs b/encodings/zstd/src/test.rs index 76a8909645e..662cb434d87 100644 --- a/encodings/zstd/src/test.rs +++ b/encodings/zstd/src/test.rs @@ -144,12 +144,21 @@ fn test_validity_vtable() { Validity::Array(BoolArray::from_iter(mask_bools.clone()).into_array()), ); let compressed = Zstd::from_primitive(&array, 3, 0).unwrap(); + let arr = compressed.as_array(); assert_eq!( - compressed.as_array().validity_mask().unwrap(), + arr.validity() + .unwrap() + .to_mask(arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(mask_bools) ); + let sliced = compressed.slice(1..4).unwrap(); assert_eq!( - compressed.slice(1..4).unwrap().validity_mask().unwrap(), + sliced + .validity() + .unwrap() + .to_mask(sliced.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from_iter(vec![true, true, false]) ); } diff --git a/fuzz/fuzz_targets/file_io.rs b/fuzz/fuzz_targets/file_io.rs index 92e45c5b0c0..f6917a0e9f2 100644 --- a/fuzz/fuzz_targets/file_io.rs +++ b/fuzz/fuzz_targets/file_io.rs @@ -9,6 +9,7 @@ use libfuzzer_sys::fuzz_target; use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::builtins::ArrayBuiltins; @@ -49,7 +50,9 @@ fuzz_target!(|fuzz: FuzzFileAction| -> Corpus { .clone() .apply(&filter_expr.clone().unwrap_or_else(|| lit(true))) .vortex_expect("filter expression evaluation should succeed in fuzz test"); - let mask = bool_mask.to_bool().to_mask_fill_null_false(); + let mask = bool_mask + .to_bool() + .to_mask_fill_null_false(&mut SESSION.create_execution_ctx()); let filtered = array_data .filter(mask) .vortex_expect("filter operation should succeed in fuzz test"); diff --git a/fuzz/src/array/cast.rs b/fuzz/src/array/cast.rs index f9ddba1fd31..503287c2dcf 100644 --- a/fuzz/src/array/cast.rs +++ b/fuzz/src/array/cast.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability::Nullable; @@ -39,7 +41,12 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), - Validity::from_mask(array.validity_mask()?, target.nullability()), + Validity::from_mask( + array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + target.nullability(), + ), ) .into_array() }) @@ -65,7 +72,12 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), - Validity::from_mask(array.validity_mask()?, target.nullability()), + Validity::from_mask( + array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + target.nullability(), + ), ) .into_array(), )), @@ -80,7 +92,12 @@ pub fn cast_canonical_array(array: &ArrayRef, target: &DType) -> VortexResult>(), - Validity::from_mask(array.validity_mask()?, target.nullability()), + Validity::from_mask( + array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())?, + target.nullability(), + ), ) .into_array(), )) diff --git a/fuzz/src/array/compare.rs b/fuzz/src/array/compare.rs index 849a7fdd694..b4c714de335 100644 --- a/fuzz/src/array/compare.rs +++ b/fuzz/src/array/compare.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::bool::BoolArrayExt; @@ -45,8 +47,10 @@ pub fn compare_canonical_array( .iter() .zip( array - .validity_mask() + .validity() .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), ) @@ -70,8 +74,10 @@ pub fn compare_canonical_array( .copied() .zip( array - .validity_mask() + .validity() .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), ) @@ -98,8 +104,10 @@ pub fn compare_canonical_array( .copied() .zip( array - .validity_mask() + .validity() .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter(), ) diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index fbcc2856e61..9e5b90c7c05 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -24,7 +26,10 @@ use crate::array::take_canonical_array_non_nullable_indices; pub fn filter_canonical_array(array: &ArrayRef, filter: &[bool]) -> VortexResult { let validity = if array.dtype().is_nullable() { - let validity_buff = array.validity_mask()?.to_bit_buffer(); + let validity_buff = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .to_bit_buffer(); Validity::from_iter( filter .iter() diff --git a/fuzz/src/array/search_sorted.rs b/fuzz/src/array/search_sorted.rs index d22b583a39b..593be73c598 100644 --- a/fuzz/src/array/search_sorted.rs +++ b/fuzz/src/array/search_sorted.rs @@ -5,7 +5,9 @@ use std::cmp::Ordering; use std::fmt::Debug; use vortex_array::ArrayRef; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::bool::BoolArrayExt; use vortex_array::dtype::DType; @@ -64,7 +66,14 @@ pub fn search_sorted_canonical_array( match array.dtype() { DType::Bool(_) => { let bool_array = array.to_bool(); - let validity = bool_array.validity_mask()?.to_bit_buffer(); + let validity = bool_array + .as_ref() + .validity()? + .to_mask( + bool_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer(); let opt_values = bool_array .to_bit_buffer() .iter() @@ -76,7 +85,14 @@ pub fn search_sorted_canonical_array( } DType::Primitive(p, _) => { let primitive_array = array.to_primitive(); - let validity = primitive_array.validity_mask()?.to_bit_buffer(); + let validity = primitive_array + .as_ref() + .validity()? + .to_mask( + primitive_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer(); match_each_native_ptype!(p, |P| { let opt_values = primitive_array .as_slice::

() @@ -91,7 +107,14 @@ pub fn search_sorted_canonical_array( } DType::Decimal(d, _) => { let decimal_array = array.to_decimal(); - let validity = decimal_array.validity_mask()?.to_bit_buffer(); + let validity = decimal_array + .as_ref() + .validity()? + .to_mask( + decimal_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer(); match_each_decimal_value_type!(decimal_array.values_type(), |D| { let buf = decimal_array.buffer::(); let opt_values = buf diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index eeb1b8adf52..e1e4c388e55 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -28,7 +30,10 @@ pub fn slice_canonical_array( stop: usize, ) -> VortexResult { let validity = if array.dtype().is_nullable() { - let bool_buff = array.validity_mask()?.to_bit_buffer(); + let bool_buff = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .to_bit_buffer(); Validity::from(bool_buff.slice(start..stop)) } else { Validity::NonNullable diff --git a/fuzz/src/array/sort.rs b/fuzz/src/array/sort.rs index 6c07c76c20d..096e5bbdc06 100644 --- a/fuzz/src/array/sort.rs +++ b/fuzz/src/array/sort.rs @@ -5,7 +5,9 @@ use std::cmp::Ordering; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -28,7 +30,17 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { let mut opt_values = bool_array .to_bit_buffer() .iter() - .zip(bool_array.validity_mask()?.to_bit_buffer().iter()) + .zip( + bool_array + .as_ref() + .validity()? + .to_mask( + bool_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer() + .iter(), + ) .map(|(b, v)| v.then_some(b)) .collect::>(); opt_values.sort(); @@ -41,7 +53,17 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { .as_slice::

() .iter() .copied() - .zip(primitive_array.validity_mask()?.to_bit_buffer().iter()) + .zip( + primitive_array + .as_ref() + .validity()? + .to_mask( + primitive_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer() + .iter(), + ) .map(|(p, v)| v.then_some(p)) .collect::>(); sort_primitive_slice(&mut opt_values); @@ -56,7 +78,17 @@ pub fn sort_canonical_array(array: &ArrayRef) -> VortexResult { .as_slice() .iter() .copied() - .zip(decimal_array.validity_mask()?.to_bit_buffer().iter()) + .zip( + decimal_array + .as_ref() + .validity()? + .to_mask( + decimal_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .to_bit_buffer() + .iter(), + ) .map(|(p, v)| v.then_some(p)) .collect::>(); opt_values.sort(); diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index a113829077b..01c7eae4a39 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -3,7 +3,9 @@ use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::BoolArray; use vortex_array::arrays::DecimalArray; @@ -43,7 +45,10 @@ pub fn take_canonical_array(array: &ArrayRef, indices: &[Option]) -> Vort let nullable: Nullability = indices.contains(&None).into(); let validity = if array.dtype().is_nullable() || nullable == Nullability::Nullable { - let validity_idx = array.validity_mask()?.to_bit_buffer(); + let validity_idx = array + .validity()? + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .to_bit_buffer(); Validity::from_iter( indices diff --git a/vortex-array/public-api.lock b/vortex-array/public-api.lock index 25736c6df84..a849acdb20e 100644 --- a/vortex-array/public-api.lock +++ b/vortex-array/public-api.lock @@ -78,7 +78,7 @@ pub fn vortex_array::aggregate_fn::fns::count::Count::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::count::Count::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub mod vortex_array::aggregate_fn::fns::first @@ -126,7 +126,7 @@ pub fn vortex_array::aggregate_fn::fns::first::First::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::first::First::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub struct vortex_array::aggregate_fn::fns::first::FirstPartial @@ -338,7 +338,7 @@ pub fn vortex_array::aggregate_fn::fns::last::Last::serialize(&self, _options: & pub fn vortex_array::aggregate_fn::fns::last::Last::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub struct vortex_array::aggregate_fn::fns::last::LastPartial @@ -442,7 +442,7 @@ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Options = vortex_ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Partial = u64 -pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -506,7 +506,7 @@ pub type vortex_array::aggregate_fn::fns::sum::Sum::Options = vortex_array::aggr pub type vortex_array::aggregate_fn::fns::sum::Sum::Partial = vortex_array::aggregate_fn::fns::sum::SumPartial -pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::sum::Sum::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -816,7 +816,7 @@ pub fn vortex_array::aggregate_fn::fns::count::Count::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::count::Count::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::count::Count::try_accumulate(&self, state: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::aggregate_fn::AggregateFnVTable for vortex_array::aggregate_fn::fns::first::First @@ -852,7 +852,7 @@ pub fn vortex_array::aggregate_fn::fns::first::First::serialize(&self, _options: pub fn vortex_array::aggregate_fn::fns::first::First::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::first::First::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::aggregate_fn::AggregateFnVTable for vortex_array::aggregate_fn::fns::is_constant::IsConstant @@ -960,7 +960,7 @@ pub fn vortex_array::aggregate_fn::fns::last::Last::serialize(&self, _options: & pub fn vortex_array::aggregate_fn::fns::last::Last::to_scalar(&self, partial: &Self::Partial) -> vortex_error::VortexResult -pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult +pub fn vortex_array::aggregate_fn::fns::last::Last::try_accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::ArrayRef, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult impl vortex_array::aggregate_fn::AggregateFnVTable for vortex_array::aggregate_fn::fns::min_max::MinMax @@ -1004,7 +1004,7 @@ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Options = vortex_ pub type vortex_array::aggregate_fn::fns::nan_count::NanCount::Partial = u64 -pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::nan_count::NanCount::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -1040,7 +1040,7 @@ pub type vortex_array::aggregate_fn::fns::sum::Sum::Options = vortex_array::aggr pub type vortex_array::aggregate_fn::fns::sum::Sum::Partial = vortex_array::aggregate_fn::fns::sum::SumPartial -pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> +pub fn vortex_array::aggregate_fn::fns::sum::Sum::accumulate(&self, partial: &mut Self::Partial, batch: &vortex_array::Columnar, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<()> pub fn vortex_array::aggregate_fn::fns::sum::Sum::coerce_args(&self, options: &Self::Options, input_dtype: &vortex_array::dtype::DType) -> vortex_error::VortexResult @@ -1266,8 +1266,6 @@ pub fn vortex_array::arrays::bool::BoolMaskedValidityRule::reduce_parent(&self, pub trait vortex_array::arrays::bool::BoolArrayExt: vortex_array::TypedArrayRef -pub fn vortex_array::arrays::bool::BoolArrayExt::bool_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::bool::BoolArrayExt::maybe_to_mask(&self) -> vortex_error::VortexResult> pub fn vortex_array::arrays::bool::BoolArrayExt::nullability(&self) -> vortex_array::dtype::Nullability @@ -1276,14 +1274,12 @@ pub fn vortex_array::arrays::bool::BoolArrayExt::to_bit_buffer(&self) -> vortex_ pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask(&self) -> vortex_mask::Mask -pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask_fill_null_false(&self) -> vortex_mask::Mask +pub fn vortex_array::arrays::bool::BoolArrayExt::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn vortex_array::arrays::bool::BoolArrayExt::validity(&self) -> vortex_array::validity::Validity impl> vortex_array::arrays::bool::BoolArrayExt for T -pub fn T::bool_validity_mask(&self) -> vortex_mask::Mask - pub fn T::maybe_to_mask(&self) -> vortex_error::VortexResult> pub fn T::nullability(&self) -> vortex_array::dtype::Nullability @@ -1292,7 +1288,7 @@ pub fn T::to_bit_buffer(&self) -> vortex_buffer::bit::buf::BitBuffer pub fn T::to_mask(&self) -> vortex_mask::Mask -pub fn T::to_mask_fill_null_false(&self) -> vortex_mask::Mask +pub fn T::to_mask_fill_null_false(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_mask::Mask pub fn T::validity(&self) -> vortex_array::validity::Validity @@ -2722,8 +2718,6 @@ pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_ pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_list_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::fixed_size_list_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt::list_size(&self) -> u32 impl> vortex_array::arrays::fixed_size_list::FixedSizeListArrayExt for T @@ -2736,8 +2730,6 @@ pub fn T::fixed_size_list_elements_at(&self, index: usize) -> vortex_error::Vort pub fn T::fixed_size_list_validity(&self) -> vortex_array::validity::Validity -pub fn T::fixed_size_list_validity_mask(&self) -> vortex_mask::Mask - pub fn T::list_size(&self) -> u32 pub type vortex_array::arrays::fixed_size_list::FixedSizeListArray = vortex_array::Array @@ -2876,8 +2868,6 @@ pub fn vortex_array::arrays::list::ListArrayExt::list_elements_at(&self, index: pub fn vortex_array::arrays::list::ListArrayExt::list_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::list::ListArrayExt::list_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::list::ListArrayExt::nullability(&self) -> vortex_array::dtype::Nullability pub fn vortex_array::arrays::list::ListArrayExt::offset_at(&self, index: usize) -> vortex_error::VortexResult @@ -2898,8 +2888,6 @@ pub fn T::list_elements_at(&self, index: usize) -> vortex_error::VortexResult vortex_array::validity::Validity -pub fn T::list_validity_mask(&self) -> vortex_mask::Mask - pub fn T::nullability(&self) -> vortex_array::dtype::Nullability pub fn T::offset_at(&self, index: usize) -> vortex_error::VortexResult @@ -3058,8 +3046,6 @@ pub fn vortex_array::arrays::listview::ListViewArrayExt::list_elements_at(&self, pub fn vortex_array::arrays::listview::ListViewArrayExt::listview_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::listview::ListViewArrayExt::listview_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::listview::ListViewArrayExt::nullability(&self) -> vortex_array::dtype::Nullability pub fn vortex_array::arrays::listview::ListViewArrayExt::offset_at(&self, index: usize) -> usize @@ -3080,8 +3066,6 @@ pub fn T::list_elements_at(&self, index: usize) -> vortex_error::VortexResult vortex_array::validity::Validity -pub fn T::listview_validity_mask(&self) -> vortex_mask::Mask - pub fn T::nullability(&self) -> vortex_array::dtype::Nullability pub fn T::offset_at(&self, index: usize) -> usize @@ -3206,8 +3190,6 @@ pub fn vortex_array::arrays::masked::MaskedArrayExt::child(&self) -> &vortex_arr pub fn vortex_array::arrays::masked::MaskedArrayExt::masked_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::masked::MaskedArrayExt::masked_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::masked::MaskedArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> impl> vortex_array::arrays::masked::MaskedArrayExt for T @@ -3216,8 +3198,6 @@ pub fn T::child(&self) -> &vortex_array::ArrayRef pub fn T::masked_validity(&self) -> vortex_array::validity::Validity -pub fn T::masked_validity_mask(&self) -> vortex_mask::Mask - pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> pub fn vortex_array::arrays::masked::mask_validity_canonical(canonical: vortex_array::Canonical, validity_mask: &vortex_mask::Mask, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -3794,8 +3774,6 @@ pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity(&self) -> vo pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> -pub fn vortex_array::arrays::primitive::PrimitiveArrayExt::validity_mask(&self) -> vortex_mask::Mask - impl> vortex_array::arrays::primitive::PrimitiveArrayExt for T pub fn T::buffer_handle(&self) -> &vortex_array::buffer::BufferHandle @@ -3812,8 +3790,6 @@ pub fn T::validity(&self) -> vortex_array::validity::Validity pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> -pub fn T::validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::primitive::chunk_range(chunk_idx: usize, offset: usize, array_len: usize) -> core::ops::range::Range pub fn vortex_array::arrays::primitive::patch_chunk(decoded_values: &mut [T], patches_indices: &[I], patches_values: &[T], patches_offset: usize, chunk_offsets_slice: &[C], chunk_idx: usize, offset_within_chunk: usize) where T: vortex_array::dtype::NativePType, I: vortex_array::dtype::UnsignedPType, C: vortex_array::dtype::UnsignedPType @@ -4658,8 +4634,6 @@ pub fn vortex_array::arrays::varbin::VarBinArrayExt::validity_child(&self) -> co pub fn vortex_array::arrays::varbin::VarBinArrayExt::varbin_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::varbin::VarBinArrayExt::varbin_validity_mask(&self) -> vortex_mask::Mask - impl> vortex_array::arrays::varbin::VarBinArrayExt for T pub fn T::bytes_at(&self, index: usize) -> vortex_buffer::ByteBuffer @@ -4680,8 +4654,6 @@ pub fn T::validity_child(&self) -> core::option::Option<&vortex_array::ArrayRef> pub fn T::varbin_validity(&self) -> vortex_array::validity::Validity -pub fn T::varbin_validity_mask(&self) -> vortex_mask::Mask - pub fn vortex_array::arrays::varbin::varbin_scalar(value: vortex_buffer::ByteBuffer, dtype: &vortex_array::dtype::DType) -> vortex_array::scalar::Scalar pub type vortex_array::arrays::varbin::VarBinArray = vortex_array::Array @@ -5042,16 +5014,12 @@ pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::dtype_parts(&self) pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::varbinview_validity(&self) -> vortex_array::validity::Validity -pub fn vortex_array::arrays::varbinview::VarBinViewArrayExt::varbinview_validity_mask(&self) -> vortex_mask::Mask - impl> vortex_array::arrays::varbinview::VarBinViewArrayExt for T pub fn T::dtype_parts(&self) -> (bool, vortex_array::dtype::Nullability) pub fn T::varbinview_validity(&self) -> vortex_array::validity::Validity -pub fn T::varbinview_validity_mask(&self) -> vortex_mask::Mask - pub type vortex_array::arrays::varbinview::VarBinViewArray = vortex_array::Array pub mod vortex_array::arrays::variant @@ -6780,11 +6748,11 @@ pub mod vortex_array::arrow pub mod vortex_array::arrow::bool -pub fn vortex_array::arrow::bool::canonical_bool_to_arrow(array: &vortex_array::arrays::BoolArray) -> vortex_error::VortexResult +pub fn vortex_array::arrow::bool::canonical_bool_to_arrow(array: &vortex_array::arrays::BoolArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub mod vortex_array::arrow::byte_view -pub fn vortex_array::arrow::byte_view::canonical_varbinview_to_arrow(array: &vortex_array::arrays::VarBinViewArray) -> vortex_error::VortexResult +pub fn vortex_array::arrow::byte_view::canonical_varbinview_to_arrow(array: &vortex_array::arrays::VarBinViewArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::arrow::byte_view::execute_varbinview_to_arrow(array: &vortex_array::arrays::VarBinViewArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult @@ -6794,7 +6762,7 @@ pub fn vortex_array::arrow::null::canonical_null_to_arrow(array: &vortex_array:: pub mod vortex_array::arrow::primitive -pub fn vortex_array::arrow::primitive::canonical_primitive_to_arrow(array: vortex_array::arrays::PrimitiveArray) -> vortex_error::VortexResult where ::Native: vortex_array::dtype::NativePType +pub fn vortex_array::arrow::primitive::canonical_primitive_to_arrow(array: vortex_array::arrays::PrimitiveArray, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult where ::Native: vortex_array::dtype::NativePType pub struct vortex_array::arrow::ArrowArrayStreamAdapter @@ -19082,7 +19050,7 @@ pub fn vortex_array::validity::Validity::take(&self, indices: &vortex_array::Arr pub fn vortex_array::validity::Validity::to_array(&self, len: usize) -> vortex_array::ArrayRef -pub fn vortex_array::validity::Validity::to_mask(&self, length: usize) -> vortex_mask::Mask +pub fn vortex_array::validity::Validity::to_mask(&self, length: usize, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::validity::Validity::uncompressed_size(&self) -> usize @@ -21788,7 +21756,7 @@ pub fn vortex_array::Array::as_constant(&self) -> core::option::Option::filter(&self, mask: vortex_mask::Mask) -> vortex_error::VortexResult -pub fn vortex_array::Array::invalid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::invalid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::Array::is_invalid(&self, index: usize) -> vortex_error::VortexResult @@ -21806,12 +21774,10 @@ pub fn vortex_array::Array::take(&self, indices: vortex_array::ArrayRef) -> v pub fn vortex_array::Array::to_canonical(&self) -> vortex_error::VortexResult -pub fn vortex_array::Array::valid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::Array::valid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::Array::validity(&self) -> vortex_error::VortexResult -pub fn vortex_array::Array::validity_mask(&self) -> vortex_error::VortexResult - impl vortex_array::Array pub fn vortex_array::Array::as_array(&self) -> &vortex_array::ArrayRef @@ -22006,7 +21972,7 @@ pub fn vortex_array::ArrayRef::filter(&self, mask: vortex_mask::Mask) -> vortex_ pub fn vortex_array::ArrayRef::into_canonical(self) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::invalid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::invalid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::is(&self) -> bool @@ -22060,12 +22026,10 @@ pub fn vortex_array::ArrayRef::to_canonical(&self) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::try_downcast(self) -> core::result::Result, vortex_array::ArrayRef> -pub fn vortex_array::ArrayRef::valid_count(&self) -> vortex_error::VortexResult +pub fn vortex_array::ArrayRef::valid_count(&self, ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::validity(&self) -> vortex_error::VortexResult -pub fn vortex_array::ArrayRef::validity_mask(&self) -> vortex_error::VortexResult - pub fn vortex_array::ArrayRef::with_slot(self, slot_idx: usize, replacement: vortex_array::ArrayRef) -> vortex_error::VortexResult pub fn vortex_array::ArrayRef::with_slots(self, slots: alloc::vec::Vec>) -> vortex_error::VortexResult diff --git a/vortex-array/src/aggregate_fn/fns/count/mod.rs b/vortex-array/src/aggregate_fn/fns/count/mod.rs index da12b85b0ee..e25c42e0845 100644 --- a/vortex-array/src/aggregate_fn/fns/count/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/count/mod.rs @@ -76,9 +76,9 @@ impl AggregateFnVTable for Count { &self, state: &mut Self::Partial, batch: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - *state += batch.valid_count()? as u64; + *state += batch.valid_count(ctx)? as u64; Ok(true) } diff --git a/vortex-array/src/aggregate_fn/fns/first/mod.rs b/vortex-array/src/aggregate_fn/fns/first/mod.rs index 67affbd4fdc..93e1fea6043 100644 --- a/vortex-array/src/aggregate_fn/fns/first/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/first/mod.rs @@ -94,12 +94,12 @@ impl AggregateFnVTable for First { &self, partial: &mut Self::Partial, batch: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { if partial.value.is_some() { return Ok(true); } - if let Some(idx) = batch.validity_mask()?.first() { + if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.first() { let scalar = batch.scalar_at(idx)?; partial.value = Some(scalar.into_nullable()); } diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index 3edebb91333..b684ea98b17 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -59,8 +59,8 @@ fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> Vor } // Check validity masks match (null positions must be identical). - let a_mask = a.validity_mask()?; - let b_mask = b.validity_mask()?; + let a_mask = a.validity()?.to_mask(a.len(), ctx)?; + let b_mask = b.validity()?.to_mask(b.len(), ctx)?; if a_mask != b_mask { return Ok(false); } diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs index c1efdc91563..408fa9b7806 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/bool.rs @@ -5,11 +5,20 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use super::IsSortedIteratorExt; +use crate::ExecutionCtx; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; -pub(super) fn check_bool_sorted(array: &BoolArray, strict: bool) -> VortexResult { - match array.validity_mask()? { +pub(super) fn check_bool_sorted( + array: &BoolArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { let values = array.to_bit_buffer(); diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs index e56858fd015..a5e2d597485 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/decimal.rs @@ -6,21 +6,34 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use super::IsSortedIteratorExt; +use crate::ExecutionCtx; use crate::arrays::DecimalArray; use crate::dtype::NativeDecimalType; use crate::match_each_decimal_value_type; -pub(super) fn check_decimal_sorted(array: &DecimalArray, strict: bool) -> VortexResult { +pub(super) fn check_decimal_sorted( + array: &DecimalArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { match_each_decimal_value_type!(array.values_type(), |S| { - compute_is_sorted::(array, strict) + compute_is_sorted::(array, strict, ctx) }) } -fn compute_is_sorted(array: &DecimalArray, strict: bool) -> VortexResult +fn compute_is_sorted( + array: &DecimalArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult where dyn Iterator: IsSortedIteratorExt, { - match array.validity_mask()? { + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { let buf = array.buffer::(); diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs index e68e74da63d..f0a016796c9 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/mod.rs @@ -108,7 +108,7 @@ fn is_sorted_impl(array: &ArrayRef, strict: bool, ctx: &mut ExecutionCtx) -> Vor // Enforce strictness before we even try to check if the array is sorted. if strict { - let invalid_count = array.invalid_count()?; + let invalid_count = array.invalid_count(ctx)?; match invalid_count { // We can keep going 0 => {} @@ -462,10 +462,10 @@ impl AggregateFnVTable for IsSorted { // Check within-batch sortedness. let batch_is_sorted = match c { - Canonical::Primitive(p) => check_primitive_sorted(p, partial.strict)?, - Canonical::Bool(b) => check_bool_sorted(b, partial.strict)?, + Canonical::Primitive(p) => check_primitive_sorted(p, partial.strict, ctx)?, + Canonical::Bool(b) => check_bool_sorted(b, partial.strict, ctx)?, Canonical::VarBinView(v) => check_varbinview_sorted(v, partial.strict)?, - Canonical::Decimal(d) => check_decimal_sorted(d, partial.strict)?, + Canonical::Decimal(d) => check_decimal_sorted(d, partial.strict, ctx)?, Canonical::Extension(e) => check_extension_sorted(e, partial.strict, ctx)?, Canonical::Null(_) => !partial.strict, // Struct, List, FixedSizeList should have been filtered out by return_dtype diff --git a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs index 21c80e7bd45..e5433b6b176 100644 --- a/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/is_sorted/primitive.rs @@ -6,17 +6,32 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use super::IsSortedIteratorExt; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::NativeValue; use crate::dtype::NativePType; use crate::match_each_native_ptype; -pub(super) fn check_primitive_sorted(array: &PrimitiveArray, strict: bool) -> VortexResult { - match_each_native_ptype!(array.ptype(), |P| { compute_is_sorted::

(array, strict) }) +pub(super) fn check_primitive_sorted( + array: &PrimitiveArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match_each_native_ptype!(array.ptype(), |P| { + compute_is_sorted::

(array, strict, ctx) + }) } -fn compute_is_sorted(array: &PrimitiveArray, strict: bool) -> VortexResult { - match array.validity_mask()? { +fn compute_is_sorted( + array: &PrimitiveArray, + strict: bool, + ctx: &mut ExecutionCtx, +) -> VortexResult { + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { Mask::AllFalse(_) => Ok(!strict), Mask::AllTrue(_) => { let slice = array.as_slice::(); diff --git a/vortex-array/src/aggregate_fn/fns/last/mod.rs b/vortex-array/src/aggregate_fn/fns/last/mod.rs index a1cbf032b88..9edfce66437 100644 --- a/vortex-array/src/aggregate_fn/fns/last/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/last/mod.rs @@ -95,9 +95,9 @@ impl AggregateFnVTable for Last { &self, partial: &mut Self::Partial, batch: &ArrayRef, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult { - if let Some(idx) = batch.validity_mask()?.last() { + if let Some(idx) = batch.validity()?.to_mask(batch.len(), ctx)?.last() { let scalar = batch.scalar_at(idx)?; partial.value = Some(scalar.into_nullable()); } diff --git a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs index 557598489e6..a4ac8d4f4c0 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/bool.rs @@ -8,17 +8,25 @@ use vortex_mask::Mask; use super::MinMaxPartial; use super::MinMaxResult; +use crate::ExecutionCtx; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::dtype::Nullability::NonNullable; use crate::scalar::Scalar; -pub(super) fn accumulate_bool(partial: &mut MinMaxPartial, array: &BoolArray) -> VortexResult<()> { +pub(super) fn accumulate_bool( + partial: &mut MinMaxPartial, + array: &BoolArray, + ctx: &mut ExecutionCtx, +) -> VortexResult<()> { if array.is_empty() { return Ok(()); } - let mask = array.validity_mask()?; + let mask = array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?; let true_non_null = match &mask { Mask::AllTrue(_) => array.to_bit_buffer(), Mask::AllFalse(_) => return Ok(()), diff --git a/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs b/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs index 93400f37275..8fa6b6b8dcd 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/decimal.rs @@ -7,6 +7,7 @@ use vortex_mask::Mask; use super::MinMaxPartial; use super::MinMaxResult; +use crate::ExecutionCtx; use crate::arrays::DecimalArray; use crate::dtype::DecimalDType; use crate::dtype::NativeDecimalType; @@ -18,30 +19,40 @@ use crate::scalar::Scalar; pub(super) fn accumulate_decimal( partial: &mut MinMaxPartial, array: &DecimalArray, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { match_each_decimal_value_type!(array.values_type(), |T| { - let local = compute_min_max_with_validity::(array)?; + let local = compute_min_max_with_validity::(array, ctx)?; partial.merge(local); Ok(()) }) } -fn compute_min_max_with_validity(array: &DecimalArray) -> VortexResult> +fn compute_min_max_with_validity( + array: &DecimalArray, + ctx: &mut ExecutionCtx, +) -> VortexResult> where D: Into + NativeDecimalType, { - Ok(match array.validity_mask()? { - Mask::AllTrue(_) => compute_min_max(array.buffer::().iter(), array.decimal_dtype()), - Mask::AllFalse(_) => None, - Mask::Values(v) => compute_min_max( - array - .buffer::() - .iter() - .zip(v.bit_buffer().iter()) - .filter_map(|(v, m)| m.then_some(v)), - array.decimal_dtype(), - ), - }) + Ok( + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { + Mask::AllTrue(_) => compute_min_max(array.buffer::().iter(), array.decimal_dtype()), + Mask::AllFalse(_) => None, + Mask::Values(v) => compute_min_max( + array + .buffer::() + .iter() + .zip(v.bit_buffer().iter()) + .filter_map(|(v, m)| m.then_some(v)), + array.decimal_dtype(), + ), + }, + ) } fn compute_min_max<'a, T>( diff --git a/vortex-array/src/aggregate_fn/fns/min_max/mod.rs b/vortex-array/src/aggregate_fn/fns/min_max/mod.rs index 17e886b1efb..0b845e3276a 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/mod.rs @@ -63,7 +63,7 @@ pub fn min_max(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult match c { - Canonical::Primitive(p) => accumulate_primitive(partial, p), - Canonical::Bool(b) => accumulate_bool(partial, b), + Canonical::Primitive(p) => accumulate_primitive(partial, p, ctx), + Canonical::Bool(b) => accumulate_bool(partial, b, ctx), Canonical::VarBinView(v) => accumulate_varbinview(partial, v), - Canonical::Decimal(d) => accumulate_decimal(partial, d), + Canonical::Decimal(d) => accumulate_decimal(partial, d, ctx), Canonical::Extension(e) => accumulate_extension(partial, e, ctx), Canonical::Null(_) => Ok(()), Canonical::Struct(_) diff --git a/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs b/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs index 1332fa7dd88..7bc46c90a1f 100644 --- a/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/min_max/primitive.rs @@ -7,6 +7,7 @@ use vortex_mask::Mask; use super::MinMaxPartial; use super::MinMaxResult; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::dtype::NativePType; use crate::dtype::Nullability::NonNullable; @@ -17,30 +18,40 @@ use crate::scalar::Scalar; pub(super) fn accumulate_primitive( partial: &mut MinMaxPartial, p: &PrimitiveArray, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { match_each_native_ptype!(p.ptype(), |T| { - let local = compute_min_max_with_validity::(p)?; + let local = compute_min_max_with_validity::(p, ctx)?; partial.merge(local); Ok(()) }) } -fn compute_min_max_with_validity(array: &PrimitiveArray) -> VortexResult> +fn compute_min_max_with_validity( + array: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult> where T: NativePType, PValue: From, { - Ok(match array.validity_mask()? { - Mask::AllTrue(_) => compute_min_max(array.as_slice::().iter()), - Mask::AllFalse(_) => None, - Mask::Values(v) => compute_min_max( - array - .as_slice::() - .iter() - .zip(v.bit_buffer().iter()) - .filter_map(|(v, m)| m.then_some(v)), - ), - }) + Ok( + match array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)? + { + Mask::AllTrue(_) => compute_min_max(array.as_slice::().iter()), + Mask::AllFalse(_) => None, + Mask::Values(v) => compute_min_max( + array + .as_slice::() + .iter() + .zip(v.bit_buffer().iter()) + .filter_map(|(v, m)| m.then_some(v)), + ), + }, + ) } fn compute_min_max<'a, T>(iter: impl Iterator) -> Option diff --git a/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs b/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs index 602963cc385..0e229ed7982 100644 --- a/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/nan_count/mod.rs @@ -46,7 +46,7 @@ pub fn nan_count(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult VortexResult<()> { match batch { Columnar::Constant(c) => { @@ -150,7 +150,7 @@ impl AggregateFnVTable for NanCount { Ok(()) } Columnar::Canonical(c) => match c { - Canonical::Primitive(p) => accumulate_primitive(partial, p), + Canonical::Primitive(p) => accumulate_primitive(partial, p, ctx), _ => vortex_bail!( "Unsupported canonical type for nan_count: {}", batch.dtype() diff --git a/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs b/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs index 34781bafbd7..77ad78df1c4 100644 --- a/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/nan_count/primitive.rs @@ -4,13 +4,21 @@ use vortex_error::VortexResult; use vortex_mask::Mask; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::dtype::NativePType; use crate::match_each_float_ptype; -pub(super) fn accumulate_primitive(count: &mut u64, p: &PrimitiveArray) -> VortexResult<()> { +pub(super) fn accumulate_primitive( + count: &mut u64, + p: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult<()> { match_each_float_ptype!(p.ptype(), |F| { - *count += compute_nan_count_with_validity(p.as_slice::(), p.validity_mask()?) as u64; + *count += compute_nan_count_with_validity( + p.as_slice::(), + p.as_ref().validity()?.to_mask(p.as_ref().len(), ctx)?, + ) as u64; }); Ok(()) } diff --git a/vortex-array/src/aggregate_fn/fns/sum/bool.rs b/vortex-array/src/aggregate_fn/fns/sum/bool.rs index f80023e0b57..f5a02299278 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/bool.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/bool.rs @@ -9,15 +9,20 @@ use vortex_mask::AllOr; use super::SumState; use super::checked_add_u64; +use crate::ExecutionCtx; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; -pub(super) fn accumulate_bool(inner: &mut SumState, b: &BoolArray) -> VortexResult { +pub(super) fn accumulate_bool( + inner: &mut SumState, + b: &BoolArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { let SumState::Unsigned(acc) = inner else { vortex_panic!("expected unsigned sum state for bool input"); }; - let mask = b.validity_mask()?; + let mask = b.as_ref().validity()?.to_mask(b.as_ref().len(), ctx)?; let true_count = match mask.bit_buffer() { AllOr::None => return Ok(false), AllOr::All => b.to_bit_buffer().true_count() as u64, diff --git a/vortex-array/src/aggregate_fn/fns/sum/decimal.rs b/vortex-array/src/aggregate_fn/fns/sum/decimal.rs index 91fd48e73ea..f5c801d8528 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/decimal.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/decimal.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_panic; use vortex_mask::Mask; use super::SumState; +use crate::ExecutionCtx; use crate::arrays::DecimalArray; use crate::dtype::DecimalDType; use crate::dtype::DecimalType; @@ -22,8 +23,12 @@ use crate::scalar::DecimalValue; /// Accumulate a decimal array into the sum state. /// Returns Ok(true) if saturated (overflow), Ok(false) if not. -pub(super) fn accumulate_decimal(inner: &mut SumState, d: &DecimalArray) -> VortexResult { - let mask = d.validity_mask()?; +pub(super) fn accumulate_decimal( + inner: &mut SumState, + d: &DecimalArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let mask = d.as_ref().validity()?.to_mask(d.as_ref().len(), ctx)?; let validity = match &mask { Mask::AllTrue(_) => None, Mask::Values(mask_values) => Some(mask_values.bit_buffer()), diff --git a/vortex-array/src/aggregate_fn/fns/sum/mod.rs b/vortex-array/src/aggregate_fn/fns/sum/mod.rs index 7b45aa65fa9..cad8cedf626 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/mod.rs @@ -217,7 +217,7 @@ impl AggregateFnVTable for Sum { &self, partial: &mut Self::Partial, batch: &Columnar, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult<()> { // Constants compute scalar * len and combine via combine_partials. if let Columnar::Constant(c) = batch { @@ -234,9 +234,9 @@ impl AggregateFnVTable for Sum { let result = match batch { Columnar::Canonical(c) => match c { - Canonical::Primitive(p) => accumulate_primitive(&mut inner, p), - Canonical::Bool(b) => accumulate_bool(&mut inner, b), - Canonical::Decimal(d) => accumulate_decimal(&mut inner, d), + Canonical::Primitive(p) => accumulate_primitive(&mut inner, p, ctx), + Canonical::Bool(b) => accumulate_bool(&mut inner, b, ctx), + Canonical::Decimal(d) => accumulate_decimal(&mut inner, d, ctx), _ => vortex_bail!("Unsupported canonical type for sum: {}", batch.dtype()), }, Columnar::Constant(_) => unreachable!(), diff --git a/vortex-array/src/aggregate_fn/fns/sum/primitive.rs b/vortex-array/src/aggregate_fn/fns/sum/primitive.rs index ed8ecd17a6a..e236c624ad4 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/primitive.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/primitive.rs @@ -11,11 +11,16 @@ use vortex_mask::AllOr; use super::SumState; use super::checked_add_i64; use super::checked_add_u64; +use crate::ExecutionCtx; use crate::arrays::PrimitiveArray; use crate::match_each_native_ptype; -pub(super) fn accumulate_primitive(inner: &mut SumState, p: &PrimitiveArray) -> VortexResult { - let mask = p.validity_mask()?; +pub(super) fn accumulate_primitive( + inner: &mut SumState, + p: &PrimitiveArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { + let mask = p.as_ref().validity()?.to_mask(p.as_ref().len(), ctx)?; match mask.bit_buffer() { AllOr::None => Ok(false), AllOr::All => accumulate_primitive_all(inner, p), diff --git a/vortex-array/src/array/erased.rs b/vortex-array/src/array/erased.rs index f951f9423ae..b8ebee660bc 100644 --- a/vortex-array/src/array/erased.rs +++ b/vortex-array/src/array/erased.rs @@ -25,7 +25,6 @@ use crate::Canonical; use crate::ExecutionCtx; use crate::IntoArray; use crate::LEGACY_SESSION; -use crate::ToCanonical; use crate::VTable; use crate::VortexSessionExecute; use crate::aggregate_fn::fns::sum::sum; @@ -41,7 +40,6 @@ use crate::arrays::Primitive; use crate::arrays::SliceArray; use crate::arrays::VarBin; use crate::arrays::VarBinView; -use crate::arrays::bool::BoolArrayExt; use crate::buffer::BufferHandle; use crate::builders::ArrayBuilder; use crate::dtype::DType; @@ -256,7 +254,7 @@ impl ArrayRef { } /// Returns the number of valid elements in the array. - pub fn valid_count(&self) -> VortexResult { + pub fn valid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult { let len = self.len(); if let Some(Precision::Exact(invalid_count)) = self.statistics().get_as::(Stat::NullCount) @@ -268,8 +266,7 @@ impl ArrayRef { Validity::NonNullable | Validity::AllValid => len, Validity::AllInvalid => 0, Validity::Array(a) => { - let mut ctx = LEGACY_SESSION.create_execution_ctx(); - let array_sum = sum(&a, &mut ctx)?; + let array_sum = sum(&a, ctx)?; array_sum .as_primitive() .as_::() @@ -285,8 +282,8 @@ impl ArrayRef { } /// Returns the number of invalid elements in the array. - pub fn invalid_count(&self) -> VortexResult { - Ok(self.len() - self.valid_count()?) + pub fn invalid_count(&self, ctx: &mut ExecutionCtx) -> VortexResult { + Ok(self.len() - self.valid_count(ctx)?) } /// Returns the [`Validity`] of the array. @@ -294,15 +291,6 @@ impl ArrayRef { self.0.validity(self) } - /// Returns the canonical validity mask for the array. - pub fn validity_mask(&self) -> VortexResult { - match self.validity()? { - Validity::NonNullable | Validity::AllValid => Ok(Mask::new_true(self.len())), - Validity::AllInvalid => Ok(Mask::new_false(self.len())), - Validity::Array(a) => Ok(a.to_bool().to_mask()), - } - } - /// Returns the canonical representation of the array. pub fn into_canonical(self) -> VortexResult { self.execute(&mut LEGACY_SESSION.create_execution_ctx()) diff --git a/vortex-array/src/array/typed.rs b/vortex-array/src/array/typed.rs index 55656409128..3cba3cb527a 100644 --- a/vortex-array/src/array/typed.rs +++ b/vortex-array/src/array/typed.rs @@ -392,12 +392,12 @@ impl Array { self.inner.as_constant() } - pub fn valid_count(&self) -> VortexResult { - self.inner.valid_count() + pub fn valid_count(&self, ctx: &mut crate::ExecutionCtx) -> VortexResult { + self.inner.valid_count(ctx) } - pub fn invalid_count(&self) -> VortexResult { - self.inner.invalid_count() + pub fn invalid_count(&self, ctx: &mut crate::ExecutionCtx) -> VortexResult { + self.inner.invalid_count(ctx) } pub fn append_to_builder( @@ -407,10 +407,6 @@ impl Array { ) -> VortexResult<()> { self.inner.append_to_builder(builder, ctx) } - - pub fn validity_mask(&self) -> VortexResult { - self.inner.validity_mask() - } } impl Deref for Array { diff --git a/vortex-array/src/arrays/bool/array.rs b/vortex-array/src/arrays/bool/array.rs index 8a7eed44037..5ed85d7eab4 100644 --- a/vortex-array/src/arrays/bool/array.rs +++ b/vortex-array/src/arrays/bool/array.rs @@ -13,6 +13,7 @@ use vortex_error::vortex_ensure; use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::array::Array; use crate::array::ArrayParts; @@ -92,10 +93,6 @@ pub trait BoolArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn bool_validity_mask(&self) -> Mask { - self.validity().to_mask(self.as_ref().len()) - } - fn to_bit_buffer(&self) -> BitBuffer { let buffer = self.bits.as_host().clone(); BitBuffer::new_with_offset(buffer, self.as_ref().len(), self.offset) @@ -116,8 +113,11 @@ pub trait BoolArrayExt: TypedArrayRef { .vortex_expect("cannot convert nullable boolean array to mask") } - fn to_mask_fill_null_false(&self) -> Mask { - let validity_mask = self.bool_validity_mask(); + fn to_mask_fill_null_false(&self, ctx: &mut ExecutionCtx) -> Mask { + let validity_mask = self + .validity() + .to_mask(self.as_ref().len(), ctx) + .vortex_expect("Failed to compute validity mask"); let buffer = match validity_mask { Mask::AllTrue(_) => self.to_bit_buffer(), Mask::AllFalse(_) => return Mask::new_false(self.as_ref().len()), diff --git a/vortex-array/src/arrays/bool/compute/take.rs b/vortex-array/src/arrays/bool/compute/take.rs index 19b1508fd84..b3b3a269795 100644 --- a/vortex-array/src/arrays/bool/compute/take.rs +++ b/vortex-array/src/arrays/bool/compute/take.rs @@ -28,7 +28,7 @@ impl TakeExecute for Bool { indices: &ArrayRef, ctx: &mut ExecutionCtx, ) -> VortexResult> { - let indices_nulls_zeroed = match indices.validity_mask()? { + let indices_nulls_zeroed = match indices.validity()?.to_mask(indices.len(), ctx)? { Mask::AllTrue(_) => indices.clone(), Mask::AllFalse(_) => { return Ok(Some( diff --git a/vortex-array/src/arrays/bool/test_harness.rs b/vortex-array/src/arrays/bool/test_harness.rs index 52cffafdbd9..056d122b3dc 100644 --- a/vortex-array/src/arrays/bool/test_harness.rs +++ b/vortex-array/src/arrays/bool/test_harness.rs @@ -4,13 +4,20 @@ use vortex_error::VortexExpect; use vortex_error::vortex_panic; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; impl BoolArray { pub fn opt_bool_vec(&self) -> Vec> { - self.validity_mask() - .vortex_expect("validity_mask") + self.validity() + .vortex_expect("failed to get validity") + .to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter() .zip(self.to_bit_buffer().iter()) @@ -19,8 +26,13 @@ impl BoolArray { } pub fn bool_vec(&self) -> Vec { - self.validity_mask() - .vortex_expect("validity_mask") + self.validity() + .vortex_expect("failed to get validity") + .to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask") .to_bit_buffer() .iter() .zip(self.to_bit_buffer().iter()) diff --git a/vortex-array/src/arrays/chunked/compute/take.rs b/vortex-array/src/arrays/chunked/compute/take.rs index 1cd48c57319..38b0f4313ea 100644 --- a/vortex-array/src/arrays/chunked/compute/take.rs +++ b/vortex-array/src/arrays/chunked/compute/take.rs @@ -31,7 +31,10 @@ fn take_chunked( .cast(DType::Primitive(PType::U64, indices.dtype().nullability()))? .execute::(ctx)?; - let indices_mask = indices.validity_mask()?; + let indices_mask = indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?; let indices_values = indices.as_slice::(); let n = indices_values.len(); @@ -96,8 +99,13 @@ fn take_chunked( // 4. Single take to restore original order and expand duplicates. // Carry the original index validity so null indices produce null outputs. - let take_validity = - Validity::from_mask(indices.validity_mask()?, indices.dtype().nullability()); + let take_validity = Validity::from_mask( + indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?, + indices.dtype().nullability(), + ); flat.take(PrimitiveArray::new(final_take.freeze(), take_validity).into_array()) } diff --git a/vortex-array/src/arrays/constant/compute/take.rs b/vortex-array/src/arrays/constant/compute/take.rs index 96aa71fd9ac..e01a37f7caa 100644 --- a/vortex-array/src/arrays/constant/compute/take.rs +++ b/vortex-array/src/arrays/constant/compute/take.rs @@ -6,6 +6,8 @@ use vortex_mask::AllOr; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::array::ArrayView; use crate::arrays::Constant; use crate::arrays::ConstantArray; @@ -18,7 +20,12 @@ use crate::validity::Validity; impl TakeReduce for Constant { fn take(array: ArrayView<'_, Constant>, indices: &ArrayRef) -> VortexResult> { - let result = match indices.validity_mask()?.bit_buffer() { + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + let result = match indices + .validity()? + .to_mask(indices.len(), &mut ctx)? + .bit_buffer() + { AllOr::All => { let scalar = Scalar::try_new( array @@ -64,7 +71,9 @@ mod tests { use vortex_mask::AllOr; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; @@ -98,7 +107,12 @@ mod tests { ) ); assert_eq!( - taken.validity_mask().unwrap().indices(), + taken + .validity() + .unwrap() + .to_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .indices(), AllOr::Some(valid_indices) ); } @@ -117,7 +131,15 @@ mod tests { taken.to_primitive(), PrimitiveArray::new(buffer![42i32, 42, 42], Validity::AllValid) ); - assert_eq!(taken.validity_mask().unwrap().indices(), AllOr::All); + assert_eq!( + taken + .validity() + .unwrap() + .to_mask(taken.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .indices(), + AllOr::All + ); } #[rstest] diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index 137f48fd8eb..9adc39212cc 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -323,6 +323,8 @@ mod tests { use vortex_error::VortexResult; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; @@ -475,7 +477,12 @@ mod tests { let struct_array = array.as_array().to_struct(); assert_eq!(struct_array.len(), 3); - assert_eq!(struct_array.valid_count().unwrap(), 0); + assert_eq!( + struct_array + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 0 + ); let field = struct_array .unmasked_field_by_name("non_null_field") diff --git a/vortex-array/src/arrays/decimal/compute/cast.rs b/vortex-array/src/arrays/decimal/compute/cast.rs index 80377102b7d..f602b827c03 100644 --- a/vortex-array/src/arrays/decimal/compute/cast.rs +++ b/vortex-array/src/arrays/decimal/compute/cast.rs @@ -146,6 +146,8 @@ mod tests { use super::upcast_decimal_values; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builtins::ArrayBuiltins; use crate::canonical::ToCanonical; @@ -383,7 +385,15 @@ mod tests { assert_eq!(casted.len(), 3); // Check validity is preserved - let mask = casted.validity_mask().unwrap(); + let mask = casted + .as_ref() + .validity() + .unwrap() + .to_mask( + casted.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); assert!(mask.value(0)); assert!(!mask.value(1)); assert!(mask.value(2)); diff --git a/vortex-array/src/arrays/decimal/compute/fill_null.rs b/vortex-array/src/arrays/decimal/compute/fill_null.rs index e697a277cca..de01076b231 100644 --- a/vortex-array/src/arrays/decimal/compute/fill_null.rs +++ b/vortex-array/src/arrays/decimal/compute/fill_null.rs @@ -89,6 +89,8 @@ mod tests { use vortex_buffer::buffer; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; @@ -123,7 +125,14 @@ mod tests { p.buffer::().as_slice(), vec![4200, 800, 4200, 1000, 4200] ); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] diff --git a/vortex-array/src/arrays/dict/array.rs b/vortex-array/src/arrays/dict/array.rs index b5f007a05d3..d3ef863990c 100644 --- a/vortex-array/src/arrays/dict/array.rs +++ b/vortex-array/src/arrays/dict/array.rs @@ -12,7 +12,9 @@ use vortex_error::vortex_ensure; use vortex_mask::AllOr; use crate::ArrayRef; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::array::Array; use crate::array::ArrayParts; use crate::array::TypedArrayRef; @@ -139,7 +141,10 @@ pub trait DictArrayExt: TypedArrayRef + DictArraySlotsExt { } fn compute_referenced_values_mask(&self, referenced: bool) -> VortexResult { - let codes_validity = self.codes().validity_mask()?; + let codes = self.codes(); + let codes_validity = codes + .validity()? + .to_mask(codes.len(), &mut LEGACY_SESSION.create_execution_ctx())?; let codes_primitive = self.codes().to_primitive(); let values_len = self.values().len(); @@ -290,7 +295,15 @@ mod test { PrimitiveArray::new(buffer![3, 6, 9], Validity::AllValid).into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; @@ -308,7 +321,15 @@ mod test { .into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; @@ -330,7 +351,15 @@ mod test { .into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; @@ -348,7 +377,15 @@ mod test { PrimitiveArray::new(buffer![3, 6, 9], Validity::NonNullable).into_array(), ) .unwrap(); - let mask = dict.validity_mask().unwrap(); + let mask = dict + .as_ref() + .validity() + .unwrap() + .to_mask( + dict.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); let AllOr::Some(indices) = mask.indices() else { vortex_panic!("Expected indices from mask") }; diff --git a/vortex-array/src/arrays/filter/execute/mod.rs b/vortex-array/src/arrays/filter/execute/mod.rs index bcbf28c0d96..fbe840f9bee 100644 --- a/vortex-array/src/arrays/filter/execute/mod.rs +++ b/vortex-array/src/arrays/filter/execute/mod.rs @@ -54,7 +54,7 @@ fn filter_validity(validity: Validity, mask: &Arc) -> Validity { /// Check for some fast-path execution conditions before calling [`execute_filter`]. pub(super) fn execute_filter_fast_paths( array: ArrayView<'_, Filter>, - _ctx: &mut ExecutionCtx, + ctx: &mut ExecutionCtx, ) -> VortexResult> { let true_count = array.mask.true_count(); @@ -70,7 +70,13 @@ pub(super) fn execute_filter_fast_paths( // Also check if the array itself is completely null, in which case we only care about the total // number of nulls, not the values. - if array.array().validity_mask()?.true_count() == 0 { + let child_arr = array.array(); + if child_arr + .validity()? + .to_mask(child_arr.len(), ctx)? + .true_count() + == 0 + { return Ok(Some( ConstantArray::new(Scalar::null(array.dtype().clone()), true_count).into_array(), )); diff --git a/vortex-array/src/arrays/fixed_size_list/array.rs b/vortex-array/src/arrays/fixed_size_list/array.rs index dcbee9f8f6f..22dd34b46bb 100644 --- a/vortex-array/src/arrays/fixed_size_list/array.rs +++ b/vortex-array/src/arrays/fixed_size_list/array.rs @@ -231,10 +231,6 @@ pub trait FixedSizeListArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], nullability) } - fn fixed_size_list_validity_mask(&self) -> vortex_mask::Mask { - self.fixed_size_list_validity().to_mask(self.as_ref().len()) - } - fn fixed_size_list_elements_at(&self, index: usize) -> VortexResult { debug_assert!( index < self.as_ref().len(), diff --git a/vortex-array/src/arrays/fixed_size_list/compute/take.rs b/vortex-array/src/arrays/fixed_size_list/compute/take.rs index 259eb382013..95cb101e9da 100644 --- a/vortex-array/src/arrays/fixed_size_list/compute/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/compute/take.rs @@ -6,17 +6,19 @@ use vortex_buffer::BufferMut; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_panic; +use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::ToCanonical; use crate::array::ArrayView; use crate::arrays::FixedSizeList; use crate::arrays::FixedSizeListArray; use crate::arrays::Primitive; use crate::arrays::PrimitiveArray; +use crate::arrays::bool::BoolArrayExt; use crate::arrays::dict::TakeExecute; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; -use crate::arrays::primitive::PrimitiveArrayExt; use crate::dtype::IntegerPType; use crate::executor::ExecutionCtx; use crate::match_each_integer_ptype; @@ -82,7 +84,7 @@ fn take_with_indices( // The result's nullability is the union of the input nullabilities. if array.dtype().is_nullable() || indices_array.dtype().is_nullable() { let indices_array = indices_array.as_view(); - take_nullable_fsl::(array, indices_array) + take_nullable_fsl::(array, indices_array, ctx) } else { let indices_array = indices_array.as_view(); take_non_nullable_fsl::(array, indices_array) @@ -143,13 +145,25 @@ fn take_non_nullable_fsl( fn take_nullable_fsl( array: ArrayView<'_, FixedSizeList>, indices_array: ArrayView<'_, Primitive>, + ctx: &mut ExecutionCtx, ) -> VortexResult { let list_size = array.list_size() as usize; let indices: &[I] = indices_array.as_slice::(); let new_len = indices.len(); - let array_validity = array.fixed_size_list_validity_mask(); - let indices_validity = indices_array.validity_mask(); + let array_validity = array + .fixed_size_list_validity() + .to_mask(array.as_ref().len(), ctx) + .vortex_expect("Failed to compute validity mask"); + let indices_len = indices_array.as_ref().len(); + let indices_validity = match indices_array + .validity() + .vortex_expect("Failed to compute validity mask") + { + Validity::NonNullable | Validity::AllValid => Mask::new_true(indices_len), + Validity::AllInvalid => Mask::new_false(indices_len), + Validity::Array(a) => a.to_bool().to_mask(), + }; // We must use placeholder zeros for null lists to maintain the array length without // propagating nullability to the element array's take operation. diff --git a/vortex-array/src/arrays/list/array.rs b/vortex-array/src/arrays/list/array.rs index 75db7d0f08c..c941060a42d 100644 --- a/vortex-array/src/arrays/list/array.rs +++ b/vortex-array/src/arrays/list/array.rs @@ -288,10 +288,6 @@ pub trait ListArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn list_validity_mask(&self) -> vortex_mask::Mask { - self.list_validity().to_mask(self.as_ref().len()) - } - fn offset_at(&self, index: usize) -> VortexResult { vortex_ensure!( index <= self.as_ref().len(), diff --git a/vortex-array/src/arrays/list/compute/take.rs b/vortex-array/src/arrays/list/compute/take.rs index 841fd4ac45b..22cfe224580 100644 --- a/vortex-array/src/arrays/list/compute/take.rs +++ b/vortex-array/src/arrays/list/compute/take.rs @@ -59,8 +59,11 @@ fn _take( indices_array: ArrayView<'_, Primitive>, ctx: &mut ExecutionCtx, ) -> VortexResult { - let data_validity = array.list_validity_mask(); - let indices_validity = indices_array.validity_mask(); + let data_validity = array.list_validity().to_mask(array.as_ref().len(), ctx)?; + let indices_validity = indices_array + .validity() + .vortex_expect("Failed to compute validity mask") + .to_mask(indices_array.as_ref().len(), ctx)?; if !indices_validity.all_true() || !data_validity.all_true() { return _take_nullable::(array, indices_array, ctx); @@ -124,8 +127,11 @@ fn _take_nullable(ctx)?; let offsets: &[O] = offsets_array.as_slice(); let indices: &[I] = indices_array.as_slice(); - let data_validity = array.list_validity_mask(); - let indices_validity = indices_array.validity_mask(); + let data_validity = array.list_validity().to_mask(array.as_ref().len(), ctx)?; + let indices_validity = indices_array + .validity() + .vortex_expect("Failed to compute validity mask") + .to_mask(indices_array.as_ref().len(), ctx)?; let mut new_offsets = PrimitiveBuilder::::with_capacity( Nullability::NonNullable, diff --git a/vortex-array/src/arrays/listview/array.rs b/vortex-array/src/arrays/listview/array.rs index 72c376b2f44..8f9aee6406e 100644 --- a/vortex-array/src/arrays/listview/array.rs +++ b/vortex-array/src/arrays/listview/array.rs @@ -334,10 +334,6 @@ pub trait ListViewArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn listview_validity_mask(&self) -> vortex_mask::Mask { - self.listview_validity().to_mask(self.as_ref().len()) - } - fn offset_at(&self, index: usize) -> usize { assert!( index < self.as_ref().len(), diff --git a/vortex-array/src/arrays/masked/array.rs b/vortex-array/src/arrays/masked/array.rs index 4f6d53e5b35..e902fa56a47 100644 --- a/vortex-array/src/arrays/masked/array.rs +++ b/vortex-array/src/arrays/masked/array.rs @@ -50,10 +50,6 @@ pub trait MaskedArrayExt: TypedArrayRef { self.as_ref().dtype().nullability(), ) } - - fn masked_validity_mask(&self) -> vortex_mask::Mask { - self.masked_validity().to_mask(self.as_ref().len()) - } } impl> MaskedArrayExt for T {} diff --git a/vortex-array/src/arrays/masked/tests.rs b/vortex-array/src/arrays/masked/tests.rs index 760383bcce0..551ead31448 100644 --- a/vortex-array/src/arrays/masked/tests.rs +++ b/vortex-array/src/arrays/masked/tests.rs @@ -61,7 +61,8 @@ fn test_masked_child_with_validity() { let prim = array.as_array().to_primitive(); // Positions where validity is false should be null in masked_child. - assert_eq!(prim.valid_count().unwrap(), 3); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); assert!(prim.is_valid(0).unwrap()); assert!(!prim.is_valid(1).unwrap()); assert!(prim.is_valid(2).unwrap()); @@ -76,7 +77,12 @@ fn test_masked_child_all_valid() { let array = MaskedArray::try_new(child, Validity::AllValid).unwrap(); assert_eq!(array.len(), 3); - assert_eq!(array.valid_count().unwrap(), 3); + assert_eq!( + array + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3 + ); assert_arrays_eq!( PrimitiveArray::from_option_iter([10i32, 20, 30].map(Some)), array diff --git a/vortex-array/src/arrays/masked/vtable/canonical.rs b/vortex-array/src/arrays/masked/vtable/canonical.rs index c612f522ef8..397cabf768a 100644 --- a/vortex-array/src/arrays/masked/vtable/canonical.rs +++ b/vortex-array/src/arrays/masked/vtable/canonical.rs @@ -7,6 +7,8 @@ mod tests { use vortex_error::VortexResult; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::MaskedArray; use crate::arrays::PrimitiveArray; use crate::dtype::Nullability; @@ -49,7 +51,8 @@ mod tests { let prim = canonical.into_primitive(); // Check that null positions match validity. - assert_eq!(prim.valid_count().unwrap(), 3); + let mut ctx = LEGACY_SESSION.create_execution_ctx(); + assert_eq!(prim.valid_count(&mut ctx).unwrap(), 3); assert!(prim.is_valid(0).unwrap()); assert!(!prim.is_valid(1).unwrap()); assert!(prim.is_valid(2).unwrap()); @@ -68,7 +71,13 @@ mod tests { let canonical = array.to_canonical()?; assert_eq!(canonical.dtype().nullability(), Nullability::Nullable); - assert_eq!(canonical.into_array().valid_count().unwrap(), 3); + assert_eq!( + canonical + .into_array() + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 3 + ); Ok(()) } } diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 5f1f09e0644..2bfd91c61a8 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -155,7 +155,7 @@ impl VTable for Masked { } fn execute(array: Array, ctx: &mut ExecutionCtx) -> VortexResult { - let validity_mask = array.masked_validity_mask(); + let validity_mask = array.masked_validity().to_mask(array.len(), ctx)?; // Fast path: all masked means result is all nulls. if validity_mask.all_false() { diff --git a/vortex-array/src/arrays/null/compute/mod.rs b/vortex-array/src/arrays/null/compute/mod.rs index ad5558a8807..e6c69beee54 100644 --- a/vortex-array/src/arrays/null/compute/mod.rs +++ b/vortex-array/src/arrays/null/compute/mod.rs @@ -15,7 +15,9 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::NullArray; use crate::compute::conformance::consistency::test_array_consistency; use crate::compute::conformance::filter::test_filter_conformance; @@ -29,8 +31,13 @@ mod test { let sliced = nulls.slice(0..4).unwrap().to_null(); assert_eq!(sliced.len(), 4); + let sliced_arr = sliced.as_array(); assert!(matches!( - sliced.as_array().validity_mask().unwrap(), + sliced_arr + .validity() + .unwrap() + .to_mask(sliced_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::AllFalse(4) )); } @@ -44,8 +51,13 @@ mod test { .to_null(); assert_eq!(taken.len(), 5); + let taken_arr = taken.as_array(); assert!(matches!( - taken.as_array().validity_mask().unwrap(), + taken_arr + .validity() + .unwrap() + .to_mask(taken_arr.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::AllFalse(5) )); } diff --git a/vortex-array/src/arrays/primitive/array/mod.rs b/vortex-array/src/arrays/primitive/array/mod.rs index e64cc891d4e..cb1f132b70f 100644 --- a/vortex-array/src/arrays/primitive/array/mod.rs +++ b/vortex-array/src/arrays/primitive/array/mod.rs @@ -123,10 +123,6 @@ pub trait PrimitiveArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn validity_mask(&self) -> vortex_mask::Mask { - self.validity().to_mask(self.as_ref().len()) - } - fn buffer_handle(&self) -> &BufferHandle { &self.buffer } diff --git a/vortex-array/src/arrays/primitive/array/top_value.rs b/vortex-array/src/arrays/primitive/array/top_value.rs index dae36a5a523..8638ea9af97 100644 --- a/vortex-array/src/arrays/primitive/array/top_value.rs +++ b/vortex-array/src/arrays/primitive/array/top_value.rs @@ -10,6 +10,8 @@ use vortex_mask::AllOr; use vortex_mask::Mask; use vortex_utils::aliases::hash_map::HashMap; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::NativeValue; use crate::dtype::NativePType; @@ -29,7 +31,13 @@ impl PrimitiveArray { } match_each_native_ptype!(self.ptype(), |P| { - let (top, count) = typed_top_value(self.as_slice::

(), self.validity_mask()?); + let (top, count) = typed_top_value( + self.as_slice::

(), + self.as_ref().validity()?.to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?, + ); Ok(Some((top.into(), count))) }) } diff --git a/vortex-array/src/arrays/primitive/compute/cast.rs b/vortex-array/src/arrays/primitive/compute/cast.rs index 13cba60bfa6..e5258bccf65 100644 --- a/vortex-array/src/arrays/primitive/compute/cast.rs +++ b/vortex-array/src/arrays/primitive/compute/cast.rs @@ -78,7 +78,7 @@ impl CastKernel for Primitive { })); } - // Otherwise, cast the values element-wise. + // Otherwise, we need to cast the values one-by-one. Ok(Some(match_each_native_ptype!(new_ptype, |T| { match_each_native_ptype!(array.ptype(), |F| { PrimitiveArray::new(cast::(array.as_slice()), new_validity).into_array() @@ -116,6 +116,8 @@ mod test { use vortex_mask::Mask; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; @@ -228,7 +230,11 @@ mod test { PrimitiveArray::from_option_iter([None, Some(0u32), Some(10)]) ); assert_eq!( - p.validity_mask().unwrap(), + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), Mask::from(BitBuffer::from(vec![false, true, true])) ); } diff --git a/vortex-array/src/arrays/primitive/compute/fill_null.rs b/vortex-array/src/arrays/primitive/compute/fill_null.rs index d0de9a81a4f..3ee48ce4c05 100644 --- a/vortex-array/src/arrays/primitive/compute/fill_null.rs +++ b/vortex-array/src/arrays/primitive/compute/fill_null.rs @@ -51,6 +51,8 @@ mod test { use vortex_buffer::buffer; use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::primitive::compute::fill_null::BoolArray; use crate::assert_arrays_eq; @@ -68,7 +70,14 @@ mod test { .unwrap() .to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([42u8, 8, 42, 10, 42])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -81,7 +90,14 @@ mod test { .unwrap() .to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([255u8, 255, 255, 255, 255])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -96,7 +112,14 @@ mod test { .unwrap() .to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([8u8, 10, 12, 14, 16])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } #[test] @@ -104,6 +127,13 @@ mod test { let arr = buffer![8u8, 10, 12, 14, 16].into_array(); let p = arr.fill_null(Scalar::from(255u8)).unwrap().to_primitive(); assert_arrays_eq!(p, PrimitiveArray::from_iter([8u8, 10, 12, 14, 16])); - assert!(p.validity_mask().unwrap().all_true()); + assert!( + p.as_ref() + .validity() + .unwrap() + .to_mask(p.as_ref().len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .all_true() + ); } } diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index fb35ee503bc..a09bd7495af 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -10,7 +10,6 @@ use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; -use vortex_mask::Mask; use crate::ArrayRef; use crate::ToCanonical; @@ -317,10 +316,6 @@ pub trait VarBinArrayExt: TypedArrayRef { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.nullability()) } - fn varbin_validity_mask(&self) -> Mask { - self.varbin_validity().to_mask(self.as_ref().len()) - } - fn offset_at(&self, index: usize) -> usize { assert!( index <= self.as_ref().len(), diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index b4ad470760e..776f77a8796 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -145,7 +145,9 @@ mod test { use vortex_buffer::ByteBuffer; use crate::IntoArray; + use crate::LEGACY_SESSION; use crate::ToCanonical; + use crate::VortexSessionExecute; use crate::arrays::ConstantArray; use crate::arrays::VarBinArray; use crate::arrays::VarBinViewArray; @@ -176,7 +178,16 @@ mod test { .to_bool(); assert_eq!( - &result.validity_mask().unwrap().to_bit_buffer(), + &result + .as_ref() + .validity() + .unwrap() + .to_mask( + result.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .to_bit_buffer(), &BitBuffer::from_iter([true, false, true]) ); assert_eq!( @@ -202,7 +213,16 @@ mod test { .to_bool(); assert_eq!( - result.validity_mask().unwrap().to_bit_buffer(), + result + .as_ref() + .validity() + .unwrap() + .to_mask( + result.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap() + .to_bit_buffer(), BitBuffer::from_iter([false, false, true]) ); assert_eq!( diff --git a/vortex-array/src/arrays/varbin/compute/filter.rs b/vortex-array/src/arrays/varbin/compute/filter.rs index 8f4b6793a72..911bc9c5e00 100644 --- a/vortex-array/src/arrays/varbin/compute/filter.rs +++ b/vortex-array/src/arrays/varbin/compute/filter.rs @@ -68,7 +68,10 @@ fn filter_select_var_bin_by_slice( offsets.as_slice::(), values.bytes().as_slice(), mask_slices, - values.varbin_validity_mask(), + values + .varbin_validity() + .to_mask(values.as_ref().len(), ctx) + .vortex_expect("Failed to compute validity mask"), selection_count, ) }) diff --git a/vortex-array/src/arrays/varbin/compute/take.rs b/vortex-array/src/arrays/varbin/compute/take.rs index de8b1ffff77..0a6d5624229 100644 --- a/vortex-array/src/arrays/varbin/compute/take.rs +++ b/vortex-array/src/arrays/varbin/compute/take.rs @@ -37,8 +37,11 @@ impl TakeExecute for VarBin { .dtype() .clone() .union_nullability(indices.dtype().nullability()); - let array_validity = array.varbin_validity_mask(); - let indices_validity = indices.validity_mask()?; + let array_validity = array.varbin_validity().to_mask(array.as_ref().len(), ctx)?; + let indices_validity = indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?; let array = match_each_integer_ptype!(indices.ptype(), |I| { // On take, offsets get widened to either 32- or 64-bit based on the original type, diff --git a/vortex-array/src/arrays/varbinview/array.rs b/vortex-array/src/arrays/varbinview/array.rs index 37d055107cf..7db90077628 100644 --- a/vortex-array/src/arrays/varbinview/array.rs +++ b/vortex-array/src/arrays/varbinview/array.rs @@ -15,7 +15,6 @@ use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_error::vortex_panic; -use vortex_mask::Mask; use crate::ArrayRef; use crate::array::Array; @@ -544,10 +543,6 @@ pub trait VarBinViewArrayExt: TypedArrayRef { fn varbinview_validity(&self) -> Validity { child_to_validity(&self.as_ref().slots()[VALIDITY_SLOT], self.dtype_parts().1) } - - fn varbinview_validity_mask(&self) -> Mask { - self.varbinview_validity().to_mask(self.as_ref().len()) - } } impl> VarBinViewArrayExt for T {} diff --git a/vortex-array/src/arrays/varbinview/compact.rs b/vortex-array/src/arrays/varbinview/compact.rs index 37bae3d9822..9667acbd5dd 100644 --- a/vortex-array/src/arrays/varbinview/compact.rs +++ b/vortex-array/src/arrays/varbinview/compact.rs @@ -11,6 +11,8 @@ use vortex_error::VortexResult; use vortex_mask::Mask; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::Ref; use crate::builders::ArrayBuilder; @@ -63,7 +65,10 @@ impl VarBinViewArray { where F: FnMut(&Ref), { - match self.validity_mask()? { + match self.as_ref().validity()?.to_mask( + self.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )? { Mask::AllTrue(_) => { for &view in self.views().iter() { if !view.is_inlined() { diff --git a/vortex-array/src/arrays/varbinview/compute/take.rs b/vortex-array/src/arrays/varbinview/compute/take.rs index 644465ecac3..e7b93bc79d6 100644 --- a/vortex-array/src/arrays/varbinview/compute/take.rs +++ b/vortex-array/src/arrays/varbinview/compute/take.rs @@ -32,7 +32,10 @@ impl TakeExecute for VarBinView { let validity = array.validity()?.take(indices)?; let indices = indices.clone().execute::(ctx)?; - let indices_mask = indices.validity_mask()?; + let indices_mask = indices + .as_ref() + .validity()? + .to_mask(indices.as_ref().len(), ctx)?; let views_buffer = match_each_integer_ptype!(indices.ptype(), |I| { take_views(array.views(), indices.as_slice::(), &indices_mask) }); diff --git a/vortex-array/src/arrays/varbinview/compute/zip.rs b/vortex-array/src/arrays/varbinview/compute/zip.rs index dfa2ae8e71c..57f8aab8067 100644 --- a/vortex-array/src/arrays/varbinview/compute/zip.rs +++ b/vortex-array/src/arrays/varbinview/compute/zip.rs @@ -54,8 +54,8 @@ impl ZipKernel for VarBinView { let mut views_builder = BufferMut::::with_capacity(len); let mut validity_builder = LazyBitBufferBuilder::new(len); - let true_validity = if_true.varbinview_validity_mask(); - let false_validity = if_false.varbinview_validity_mask(); + let true_validity = if_true.varbinview_validity().to_mask(len, ctx)?; + let false_validity = if_false.varbinview_validity().to_mask(len, ctx)?; let mask = mask.try_to_mask_fill_null_false(ctx)?; let if_false_view = if_false; diff --git a/vortex-array/src/arrow/executor/bool.rs b/vortex-array/src/arrow/executor/bool.rs index 28862a150d0..44fb9d4f683 100644 --- a/vortex-array/src/arrow/executor/bool.rs +++ b/vortex-array/src/arrow/executor/bool.rs @@ -14,10 +14,18 @@ use crate::arrays::bool::BoolArrayExt; use crate::arrow::null_buffer::to_null_buffer; /// Convert a canonical BoolArray directly to Arrow. -pub fn canonical_bool_to_arrow(array: &BoolArray) -> VortexResult { +pub fn canonical_bool_to_arrow( + array: &BoolArray, + ctx: &mut ExecutionCtx, +) -> VortexResult { Ok(Arc::new(ArrowBooleanArray::new( array.to_bit_buffer().into(), - to_null_buffer(array.validity_mask()?), + to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ), ))) } @@ -26,5 +34,5 @@ pub(super) fn to_arrow_bool( ctx: &mut ExecutionCtx, ) -> VortexResult { let bool_array = array.execute::(ctx)?; - canonical_bool_to_arrow(&bool_array) + canonical_bool_to_arrow(&bool_array, ctx) } diff --git a/vortex-array/src/arrow/executor/byte_view.rs b/vortex-array/src/arrow/executor/byte_view.rs index 9b19ddcc722..653817a982e 100644 --- a/vortex-array/src/arrow/executor/byte_view.rs +++ b/vortex-array/src/arrow/executor/byte_view.rs @@ -22,6 +22,7 @@ use crate::dtype::arrow::FromArrowType; /// Convert a canonical VarBinViewArray directly to Arrow. pub fn canonical_varbinview_to_arrow( array: &VarBinViewArray, + ctx: &mut ExecutionCtx, ) -> VortexResult { let views = ScalarBuffer::::from(array.views_handle().as_host().clone().into_arrow_buffer()); @@ -30,7 +31,12 @@ pub fn canonical_varbinview_to_arrow( .iter() .map(|buffer| buffer.as_host().clone().into_arrow_buffer()) .collect(); - let nulls = to_null_buffer(array.validity_mask()?); + let nulls = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); // SAFETY: our own VarBinView array is considered safe. Ok(Arc::new(unsafe { @@ -68,5 +74,5 @@ pub(super) fn to_arrow_byte_view( let array = array.cast(DType::from_arrow((&T::DATA_TYPE, Nullability::Nullable)))?; let varbinview = array.execute::(ctx)?; - canonical_varbinview_to_arrow::(&varbinview) + canonical_varbinview_to_arrow::(&varbinview, ctx) } diff --git a/vortex-array/src/arrow/executor/decimal.rs b/vortex-array/src/arrow/executor/decimal.rs index 234b253f903..bb4e12ef1fb 100644 --- a/vortex-array/src/arrow/executor/decimal.rs +++ b/vortex-array/src/arrow/executor/decimal.rs @@ -32,16 +32,21 @@ pub(super) fn to_arrow_decimal( let decimal_array = array.execute::(ctx)?; match data_type { - DataType::Decimal32(..) => to_arrow_decimal32(decimal_array), - DataType::Decimal64(..) => to_arrow_decimal64(decimal_array), - DataType::Decimal128(..) => to_arrow_decimal128(decimal_array), - DataType::Decimal256(..) => to_arrow_decimal256(decimal_array), + DataType::Decimal32(..) => to_arrow_decimal32(decimal_array, ctx), + DataType::Decimal64(..) => to_arrow_decimal64(decimal_array, ctx), + DataType::Decimal128(..) => to_arrow_decimal128(decimal_array, ctx), + DataType::Decimal256(..) => to_arrow_decimal256(decimal_array, ctx), _ => unreachable!("to_arrow_decimal called with non-decimal type"), } } -fn to_arrow_decimal32(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal32(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) @@ -84,8 +89,13 @@ fn to_arrow_decimal32(array: DecimalArray) -> VortexResult { )) } -fn to_arrow_decimal64(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal64(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) @@ -123,8 +133,13 @@ fn to_arrow_decimal64(array: DecimalArray) -> VortexResult { )) } -fn to_arrow_decimal128(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal128(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) @@ -157,8 +172,13 @@ fn to_arrow_decimal128(array: DecimalArray) -> VortexResult { )) } -fn to_arrow_decimal256(array: DecimalArray) -> VortexResult { - let null_buffer = to_null_buffer(array.validity_mask()?); +fn to_arrow_decimal256(array: DecimalArray, ctx: &mut ExecutionCtx) -> VortexResult { + let null_buffer = to_null_buffer( + array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?, + ); let buffer: Buffer = match array.values_type() { DecimalType::I8 => { Buffer::from_trusted_len_iter(array.buffer::().into_iter().map(|x| x.as_())) diff --git a/vortex-array/src/arrow/executor/primitive.rs b/vortex-array/src/arrow/executor/primitive.rs index 7910ed5fa31..2e7d8a9716c 100644 --- a/vortex-array/src/arrow/executor/primitive.rs +++ b/vortex-array/src/arrow/executor/primitive.rs @@ -20,11 +20,15 @@ use crate::dtype::Nullability; /// Convert a canonical PrimitiveArray directly to Arrow. pub fn canonical_primitive_to_arrow( array: PrimitiveArray, + ctx: &mut ExecutionCtx, ) -> VortexResult where T::Native: NativePType, { - let validity = array.validity_mask()?; + let validity = array + .as_ref() + .validity()? + .to_mask(array.as_ref().len(), ctx)?; let null_buffer = to_null_buffer(validity); let buffer = array.into_buffer::().into_arrow_scalar_buffer(); Ok(Arc::new(ArrowPrimitiveArray::::new(buffer, null_buffer))) @@ -40,5 +44,5 @@ where // We use nullable here so we can essentially ignore nullability during the cast. let array = array.cast(DType::Primitive(T::Native::PTYPE, Nullability::Nullable))?; let primitive = array.execute::(ctx)?; - canonical_primitive_to_arrow::(primitive) + canonical_primitive_to_arrow::(primitive, ctx) } diff --git a/vortex-array/src/arrow/executor/temporal.rs b/vortex-array/src/arrow/executor/temporal.rs index 29bcc15763e..b3941b69916 100644 --- a/vortex-array/src/arrow/executor/temporal.rs +++ b/vortex-array/src/arrow/executor/temporal.rs @@ -157,7 +157,10 @@ where primitive.ptype() ); - let validity = primitive.validity_mask()?; + let validity = primitive + .as_ref() + .validity()? + .to_mask(primitive.as_ref().len(), ctx)?; let buffer = primitive.to_buffer::(); let values = buffer.into_arrow_scalar_buffer(); diff --git a/vortex-array/src/builders/bool.rs b/vortex-array/src/builders/bool.rs index 4c5eb5d9a61..5cf2f0fa9fe 100644 --- a/vortex-array/src/builders/bool.rs +++ b/vortex-array/src/builders/bool.rs @@ -12,6 +12,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builders::ArrayBuilder; @@ -116,8 +118,17 @@ impl ArrayBuilder for BoolBuilder { let bool_array = array.to_bool(); self.inner.append_buffer(&bool_array.to_bit_buffer()); - self.nulls - .append_validity_mask(bool_array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + bool_array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + bool_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, additional: usize) { diff --git a/vortex-array/src/builders/decimal.rs b/vortex-array/src/builders/decimal.rs index d3811314a04..9f33722dcb7 100644 --- a/vortex-array/src/builders/decimal.rs +++ b/vortex-array/src/builders/decimal.rs @@ -13,7 +13,9 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::arrays::DecimalArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -202,8 +204,17 @@ impl ArrayBuilder for DecimalBuilder { .extend(decimal_array.buffer::().iter().copied()); }); - self.nulls - .append_validity_mask(decimal_array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + decimal_array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + decimal_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, additional: usize) { diff --git a/vortex-array/src/builders/fixed_size_list.rs b/vortex-array/src/builders/fixed_size_list.rs index f26281d5363..87b623f5f21 100644 --- a/vortex-array/src/builders/fixed_size_list.rs +++ b/vortex-array/src/builders/fixed_size_list.rs @@ -13,6 +13,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::FixedSizeListArray; use crate::arrays::fixed_size_list::FixedSizeListArrayExt; use crate::builders::ArrayBuilder; @@ -243,8 +245,10 @@ impl ArrayBuilder for FixedSizeListBuilder { self.elements_builder.extend_from_array(fsl.elements()); self.nulls.append_validity_mask( array - .validity_mask() - .vortex_expect("validity_mask in extend_from_array_unchecked"), + .validity() + .vortex_expect("validity_mask in extend_from_array_unchecked") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), ); } diff --git a/vortex-array/src/builders/list.rs b/vortex-array/src/builders/list.rs index 5eb6d12223b..4c49185d380 100644 --- a/vortex-array/src/builders/list.rs +++ b/vortex-array/src/builders/list.rs @@ -14,6 +14,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::ListArray; use crate::arrays::listview::ListViewArrayExt; use crate::builders::ArrayBuilder; @@ -223,8 +225,10 @@ impl ArrayBuilder for ListBuilder { // Append validity information. self.nulls.append_validity_mask( array - .validity_mask() - .vortex_expect("validity_mask in extend_from_array_unchecked"), + .validity() + .vortex_expect("validity_mask in extend_from_array_unchecked") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), ); // Note that `ListViewArray` has `n` offsets and sizes, not `n+1` offsets like `ListArray`. diff --git a/vortex-array/src/builders/listview.rs b/vortex-array/src/builders/listview.rs index a694bbbf035..c2f5c1471ee 100644 --- a/vortex-array/src/builders/listview.rs +++ b/vortex-array/src/builders/listview.rs @@ -20,7 +20,9 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::Canonical; +use crate::LEGACY_SESSION; use crate::ToCanonical; +use crate::VortexSessionExecute; use crate::array::IntoArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; @@ -303,8 +305,13 @@ impl ArrayBuilder for ListViewBuilder { .vortex_expect("ListViewArray::rebuild(MakeExact) failed in extend_from_array"); debug_assert!(listview.is_zero_copy_to_list()); - self.nulls - .append_validity_mask(listview.listview_validity_mask()); + self.nulls.append_validity_mask( + array + .validity() + .vortex_expect("validity_mask in extend_from_array_unchecked") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), + ); // Bulk append the new elements (which should have no gaps or overlaps). let old_elements_len = self.elements_builder.len(); diff --git a/vortex-array/src/builders/primitive.rs b/vortex-array/src/builders/primitive.rs index 8321c03500c..6952db2cfd2 100644 --- a/vortex-array/src/builders/primitive.rs +++ b/vortex-array/src/builders/primitive.rs @@ -12,6 +12,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::builders::ArrayBuilder; use crate::builders::DEFAULT_BUILDER_CAPACITY; @@ -185,8 +187,17 @@ impl ArrayBuilder for PrimitiveBuilder { ); self.values.extend_from_slice(array.as_slice::()); - self.nulls - .append_validity_mask(array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, additional: usize) { diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index 87f7ecb5e88..f37b88d7c80 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -13,6 +13,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::StructArray; use crate::arrays::struct_::StructArrayExt; use crate::builders::ArrayBuilder; @@ -175,8 +177,13 @@ impl ArrayBuilder for StructBuilder { builder.extend_from_array(a); } - self.nulls - .append_validity_mask(array.validity_mask().vortex_expect("validity_mask")); + self.nulls.append_validity_mask( + array + .validity() + .vortex_expect("validity_mask") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), + ); } fn reserve_exact(&mut self, capacity: usize) { @@ -203,6 +210,8 @@ impl ArrayBuilder for StructBuilder { #[cfg(test)] mod tests { use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; use crate::arrays::PrimitiveArray; use crate::arrays::VarBinArray; use crate::assert_arrays_eq; @@ -246,7 +255,12 @@ mod tests { let struct_ = builder.finish(); assert_eq!(struct_.len(), 3); assert_eq!(struct_.dtype(), &dtype); - assert_eq!(struct_.valid_count().unwrap(), 1); + assert_eq!( + struct_ + .valid_count(&mut LEGACY_SESSION.create_execution_ctx()) + .unwrap(), + 1 + ); } #[test] diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index e3adcb8f630..6d802696c74 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -20,6 +20,8 @@ use vortex_utils::aliases::hash_map::HashMap; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::VarBinViewArray; use crate::arrays::varbinview::build_views::BinaryView; use crate::arrays::varbinview::compact::BufferUtilization; @@ -295,7 +297,17 @@ impl ArrayBuilder for VarBinViewBuilder { let array = array.to_varbinview(); self.flush_in_progress(); - self.push_only_validity_mask(array.validity_mask().vortex_expect("validity_mask")); + self.push_only_validity_mask( + array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask"), + ); let view_adjustment = self.completed @@ -312,7 +324,16 @@ impl ArrayBuilder for VarBinViewBuilder { .map(|view| adjustment.adjust_view(view)), ), ViewAdjustment::Rewriting(adjustment) => { - match array.validity_mask().vortex_expect("validity_mask") { + match array + .as_ref() + .validity() + .vortex_expect("validity_mask") + .to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("Failed to compute validity mask") + { Mask::AllTrue(_) => { for (idx, &view) in array.views().iter().enumerate() { let new_view = self.push_view(view, &adjustment, &array, idx); diff --git a/vortex-array/src/compute/conformance/cast.rs b/vortex-array/src/compute/conformance/cast.rs index cab73acc521..b46a14d634b 100644 --- a/vortex-array/src/compute/conformance/cast.rs +++ b/vortex-array/src/compute/conformance/cast.rs @@ -129,7 +129,7 @@ fn test_cast_from_null(array: &ArrayRef) { fn test_cast_to_non_nullable(array: &ArrayRef) { if array - .invalid_count() + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) .vortex_expect("invalid_count should succeed in conformance test") == 0 { @@ -286,11 +286,15 @@ fn test_cast_to_primitive(array: &ArrayRef, target_ptype: PType, test_round_trip }); assert_eq!( array - .validity_mask() - .vortex_expect("validity_mask should succeed in conformance test"), + .validity() + .vortex_expect("validity_mask should succeed in conformance test") + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask"), casted - .validity_mask() + .validity() .vortex_expect("validity_mask should succeed in conformance test") + .to_mask(casted.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .vortex_expect("Failed to compute validity mask") ); for i in 0..array.len().min(10) { let original = array diff --git a/vortex-array/src/compute/conformance/consistency.rs b/vortex-array/src/compute/conformance/consistency.rs index dec1f8d1d61..9579454d24e 100644 --- a/vortex-array/src/compute/conformance/consistency.rs +++ b/vortex-array/src/compute/conformance/consistency.rs @@ -1037,10 +1037,10 @@ fn test_slice_aggregate_consistency(array: &ArrayRef) { // Test null count through invalid_count let sliced_invalid_count = sliced - .invalid_count() + .invalid_count(&mut ctx) .vortex_expect("invalid_count should succeed in conformance test"); let canonical_invalid_count = canonical_sliced - .invalid_count() + .invalid_count(&mut ctx) .vortex_expect("invalid_count should succeed in conformance test"); assert_eq!( sliced_invalid_count, canonical_invalid_count, diff --git a/vortex-array/src/compute/conformance/mask.rs b/vortex-array/src/compute/conformance/mask.rs index debdff9a806..5c448db126f 100644 --- a/vortex-array/src/compute/conformance/mask.rs +++ b/vortex-array/src/compute/conformance/mask.rs @@ -6,6 +6,8 @@ use vortex_mask::Mask; use crate::ArrayRef; use crate::IntoArray; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; use crate::builtins::ArrayBuiltins; @@ -279,7 +281,8 @@ fn test_nullable_mask_input(array: &ArrayRef) { let validity = crate::validity::Validity::from_iter(validity_values.clone()); let nullable_mask = BoolArray::new(bool_array.to_bit_buffer(), validity); - let mask_array = nullable_mask.to_mask_fill_null_false(); + let mask_array = + nullable_mask.to_mask_fill_null_false(&mut LEGACY_SESSION.create_execution_ctx()); let masked = array .clone() .mask((!&mask_array).into_array()) diff --git a/vortex-array/src/mask.rs b/vortex-array/src/mask.rs index c23a6f79652..43fee47c486 100644 --- a/vortex-array/src/mask.rs +++ b/vortex-array/src/mask.rs @@ -34,7 +34,10 @@ impl Executable for Mask { } Columnar::Canonical(a) => { let bool = a.into_array().execute::(ctx)?; - let mask = bool.validity_mask()?; + let mask = bool + .as_ref() + .validity()? + .to_mask(bool.as_ref().len(), ctx)?; let bits = bool.into_bit_buffer(); // To handle nullable boolean arrays, we treat nulls as false in the mask. // TODO(ngates): is this correct? Feels like we should just force the caller to diff --git a/vortex-array/src/patches.rs b/vortex-array/src/patches.rs index d25d6446ce6..1d37d82fb75 100644 --- a/vortex-array/src/patches.rs +++ b/vortex-array/src/patches.rs @@ -763,7 +763,10 @@ impl Patches { take_indices_with_search_fn( patch_indices_slice, take_slice, - take_indices.validity_mask()?, + take_indices + .as_ref() + .validity()? + .to_mask(take_indices.as_ref().len(), ctx)?, include_nulls, |take_idx| { self.search_index_chunked_batch( @@ -778,7 +781,10 @@ impl Patches { take_indices_with_search_fn( patch_indices_slice, take_slice, - take_indices.validity_mask()?, + take_indices + .as_ref() + .validity()? + .to_mask(take_indices.as_ref().len(), ctx)?, include_nulls, |take_idx| { let Some(offset) = ::from(self.offset) else { @@ -1157,7 +1163,15 @@ mod test { ); assert_arrays_eq!(primitive_indices, PrimitiveArray::from_iter([0u64])); assert_eq!( - primitive_values.validity_mask().unwrap(), + primitive_values + .as_ref() + .validity() + .unwrap() + .to_mask( + primitive_values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_iter(vec![true]) ); } @@ -1191,7 +1205,15 @@ mod test { assert_arrays_eq!(taken.indices(), PrimitiveArray::from_iter([0u64, 1])); assert_eq!( - primitive_values.validity_mask().unwrap(), + primitive_values + .as_ref() + .validity() + .unwrap() + .to_mask( + primitive_values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_iter([true, false]) ); } diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-array/src/scalar_fn/fns/case_when.rs index f88dbdaf356..539614874a9 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-array/src/scalar_fn/fns/case_when.rs @@ -223,7 +223,7 @@ impl ScalarFnVTable for CaseWhen { let condition = args.get(i * 2)?; let cond_bool = condition.execute::(ctx)?; - let cond_mask = cond_bool.to_mask_fill_null_false(); + let cond_mask = cond_bool.to_mask_fill_null_false(ctx); let effective_mask = &remaining & &cond_mask; if effective_mask.all_false() { diff --git a/vortex-array/src/scalar_fn/fns/zip/mod.rs b/vortex-array/src/scalar_fn/fns/zip/mod.rs index 54949ef5d70..5277e4bf437 100644 --- a/vortex-array/src/scalar_fn/fns/zip/mod.rs +++ b/vortex-array/src/scalar_fn/fns/zip/mod.rs @@ -118,7 +118,7 @@ impl ScalarFnVTable for Zip { let mask = mask_array .execute::(ctx)? - .to_mask_fill_null_false(); + .to_mask_fill_null_false(ctx); let return_dtype = if_true .dtype() diff --git a/vortex-array/src/stats/array.rs b/vortex-array/src/stats/array.rs index bd2e3cf7dea..b9d7adc902b 100644 --- a/vortex-array/src/stats/array.rs +++ b/vortex-array/src/stats/array.rs @@ -179,7 +179,11 @@ impl StatsSetRef<'_> { }) .transpose()? } - Stat::NullCount => self.dyn_array_ref.invalid_count().ok().map(Into::into), + Stat::NullCount => self + .dyn_array_ref + .invalid_count(&mut ctx) + .ok() + .map(Into::into), Stat::IsConstant => { if self.dyn_array_ref.is_empty() { None diff --git a/vortex-array/src/test_harness.rs b/vortex-array/src/test_harness.rs index 6486a53735f..fe1fc88bb15 100644 --- a/vortex-array/src/test_harness.rs +++ b/vortex-array/src/test_harness.rs @@ -8,6 +8,8 @@ use goldenfile::differs::binary_diff; use itertools::Itertools; use vortex_error::VortexResult; +use crate::LEGACY_SESSION; +use crate::VortexSessionExecute; use crate::arrays::BoolArray; use crate::arrays::bool::BoolArrayExt; @@ -26,7 +28,10 @@ pub fn check_metadata(name: &str, metadata: &[u8]) { /// Outputs the indices of the true values in a BoolArray pub fn to_int_indices(indices_bits: BoolArray) -> VortexResult> { let buffer = indices_bits.to_bit_buffer(); - let mask = indices_bits.validity_mask()?; + let mask = indices_bits.as_ref().validity()?.to_mask( + indices_bits.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; Ok(buffer .iter() .enumerate() diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index a5f6903e712..f86256128fd 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -151,22 +151,38 @@ impl Validity { pub fn take(&self, indices: &ArrayRef) -> VortexResult { match self { - Self::NonNullable => match indices.validity_mask()?.bit_buffer() { - AllOr::All => { - if indices.dtype().is_nullable() { - Ok(Self::AllValid) - } else { - Ok(Self::NonNullable) + Self::NonNullable => { + let len = indices.len(); + let indices_mask = match indices.validity()? { + Validity::NonNullable | Validity::AllValid => Mask::new_true(len), + Validity::AllInvalid => Mask::new_false(len), + Validity::Array(a) => a.to_bool().to_mask(), + }; + match indices_mask.bit_buffer() { + AllOr::All => { + if indices.dtype().is_nullable() { + Ok(Self::AllValid) + } else { + Ok(Self::NonNullable) + } } + AllOr::None => Ok(Self::AllInvalid), + AllOr::Some(buf) => Ok(Validity::from(buf.clone())), } - AllOr::None => Ok(Self::AllInvalid), - AllOr::Some(buf) => Ok(Validity::from(buf.clone())), - }, - Self::AllValid => match indices.validity_mask()?.bit_buffer() { - AllOr::All => Ok(Self::AllValid), - AllOr::None => Ok(Self::AllInvalid), - AllOr::Some(buf) => Ok(Validity::from(buf.clone())), - }, + } + Self::AllValid => { + let len = indices.len(); + let indices_mask = match indices.validity()? { + Validity::NonNullable | Validity::AllValid => Mask::new_true(len), + Validity::AllInvalid => Mask::new_false(len), + Validity::Array(a) => a.to_bool().to_mask(), + }; + match indices_mask.bit_buffer() { + AllOr::All => Ok(Self::AllValid), + AllOr::None => Ok(Self::AllInvalid), + AllOr::Some(buf) => Ok(Validity::from(buf.clone())), + } + } Self::AllInvalid => Ok(Self::AllInvalid), Self::Array(is_valid) => { let maybe_is_valid = is_valid.take(indices.clone())?; @@ -208,11 +224,11 @@ impl Validity { /// Converts this validity into a [`Mask`] of the given length. /// /// Valid elements are `true` and invalid elements are `false`. - pub fn to_mask(&self, length: usize) -> Mask { + pub fn to_mask(&self, length: usize, ctx: &mut ExecutionCtx) -> VortexResult { match self { - Self::NonNullable | Self::AllValid => Mask::new_true(length), - Self::AllInvalid => Mask::new_false(length), - Self::Array(a) => a.to_bool().to_mask(), + Self::NonNullable | Self::AllValid => Ok(Mask::new_true(length)), + Self::AllInvalid => Ok(Mask::new_false(length)), + Self::Array(arr) => arr.clone().execute::(ctx), } } @@ -376,10 +392,13 @@ impl Validity { /// Create Validity by copying the given array's validity. #[inline] pub fn copy_from_array(array: &ArrayRef) -> VortexResult { - Ok(Validity::from_mask( - array.validity_mask()?, - array.dtype().nullability(), - )) + let len = array.len(); + let mask = match array.validity()? { + Validity::NonNullable | Validity::AllValid => Mask::new_true(len), + Validity::AllInvalid => Mask::new_false(len), + Validity::Array(a) => a.to_bool().to_mask(), + }; + Ok(Validity::from_mask(mask, array.dtype().nullability())) } /// Create Validity from boolean array with given nullability of the array. diff --git a/vortex-array/src/variants.rs b/vortex-array/src/variants.rs index f60eea316a9..257d6287b0c 100644 --- a/vortex-array/src/variants.rs +++ b/vortex-array/src/variants.rs @@ -100,7 +100,9 @@ impl ArrayRef { .clone() .fill_null(Scalar::bool(false, self.dtype().nullability()))?; - Ok(array.execute::(ctx)?.to_mask_fill_null_false()) + Ok(array + .execute::(ctx)? + .to_mask_fill_null_false(ctx)) } } diff --git a/vortex-btrblocks/src/schemes/float.rs b/vortex-btrblocks/src/schemes/float.rs index 3d2557c63c7..6c204c7263a 100644 --- a/vortex-btrblocks/src/schemes/float.rs +++ b/vortex-btrblocks/src/schemes/float.rs @@ -107,7 +107,11 @@ impl Scheme for ALPScheme { data: &mut ArrayAndStats, ctx: CompressorContext, ) -> VortexResult { - let alp_encoded = alp_encode(data.array_as_primitive(), None)?; + let alp_encoded = alp_encode( + data.array_as_primitive(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; // Compress the ALP ints. let compressed_alp_ints = diff --git a/vortex-compressor/src/stats/bool.rs b/vortex-compressor/src/stats/bool.rs index 4f357f048aa..d393db58cb1 100644 --- a/vortex-compressor/src/stats/bool.rs +++ b/vortex-compressor/src/stats/bool.rs @@ -3,6 +3,8 @@ //! Bool compression statistics. +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::bool::BoolArrayExt; use vortex_error::VortexResult; @@ -42,7 +44,10 @@ impl BoolStats { }); } - let validity = input.validity_mask()?; + let validity = input.as_ref().validity()?.to_mask( + input.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let null_count = validity.false_count(); let value_count = validity.true_count(); diff --git a/vortex-compressor/src/stats/float.rs b/vortex-compressor/src/stats/float.rs index 37b06c52eca..8a1a85075ab 100644 --- a/vortex-compressor/src/stats/float.rs +++ b/vortex-compressor/src/stats/float.rs @@ -8,6 +8,8 @@ use std::hash::Hash; use itertools::Itertools; use num_traits::Float; use rustc_hash::FxBuildHasher; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::NativePType; @@ -203,7 +205,10 @@ where HashSet::with_hasher(FxBuildHasher) }; - let validity = array.validity_mask()?; + let validity = array.as_ref().validity()?.to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let mut runs = 1; let head_idx = validity diff --git a/vortex-compressor/src/stats/integer.rs b/vortex-compressor/src/stats/integer.rs index 2203cad30ef..af99efe3c0a 100644 --- a/vortex-compressor/src/stats/integer.rs +++ b/vortex-compressor/src/stats/integer.rs @@ -7,6 +7,8 @@ use std::hash::Hash; use num_traits::PrimInt; use rustc_hash::FxBuildHasher; +use vortex_array::LEGACY_SESSION; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::primitive::NativeValue; use vortex_array::dtype::IntegerPType; @@ -349,7 +351,10 @@ where }); } - let validity = array.validity_mask()?; + let validity = array.as_ref().validity()?.to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let null_count = validity.false_count(); let value_count = validity.true_count(); diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index 2ae0434b243..917fd1b7cd4 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -18,7 +18,9 @@ use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; use cudarc::driver::sys::CUevent_flags; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::scalar::Scalar; @@ -373,7 +375,12 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { let float_prim = PrimitiveArray::new(Buffer::from(floats), NonNullable); // Encode: ALP → FoR → BitPacked - let alp = alp_encode(float_prim.as_view(), Some(exponents)).vortex_expect("alp_encode"); + let alp = alp_encode( + float_prim.as_view(), + Some(exponents), + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .vortex_expect("alp_encode"); assert!(alp.patches().is_none()); let for_arr = FoRData::encode(alp.encoded().to_primitive()).vortex_expect("for encode"); let bp = diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 2bc12a536d0..67167b57ba4 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -519,6 +519,8 @@ mod tests { use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::session::VortexSession; + use vortex_array::LEGACY_SESSION; + use vortex_array::VortexSessionExecute; use super::*; use crate::CanonicalCudaExt; @@ -874,7 +876,11 @@ mod tests { .collect(); let float_prim = PrimitiveArray::new(Buffer::from(floats.clone()), NonNullable); - let alp = alp_encode(float_prim.as_view(), Some(exponents))?; + let alp = alp_encode( + float_prim.as_view(), + Some(exponents), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; assert!(alp.patches().is_none()); let for_arr = FoR::encode(alp.encoded().to_primitive())?; let bp = BitPacked::encode(for_arr.encoded(), 6)?; diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index 2f6e2a74901..fae81d76ae8 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -121,6 +121,8 @@ where #[cfg(test)] mod tests { use vortex::array::IntoArray; + use vortex::array::LEGACY_SESSION; + use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::assert_arrays_eq; use vortex::array::patches::Patches; @@ -205,7 +207,11 @@ mod tests { Some(5.0), ]; let prim = PrimitiveArray::from_option_iter(values); - let alp_array = alp_encode(prim.as_view(), None)?; + let alp_array = alp_encode( + prim.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let cpu_result = alp_array.to_canonical()?.into_array(); @@ -232,7 +238,11 @@ mod tests { Buffer::from(vec![1.0f32, 2.0, 3.0, 4.0, 5.0]), Validity::AllValid, ); - let alp_array = alp_encode(values.as_view(), None)?; + let alp_array = alp_encode( + values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let cpu_result = alp_array.to_canonical()?.into_array(); diff --git a/vortex-duckdb/src/convert/vector.rs b/vortex-duckdb/src/convert/vector.rs index 029a5ae0638..c63687037a4 100644 --- a/vortex-duckdb/src/convert/vector.rs +++ b/vortex-duckdb/src/convert/vector.rs @@ -375,7 +375,9 @@ pub fn data_chunk_to_vortex( mod tests { use std::ffi::CString; + use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; + use vortex::array::VortexSessionExecute; use vortex::array::arrays::BoolArray; use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; use vortex::array::arrays::listview::ListViewArrayExt; @@ -506,7 +508,15 @@ mod tests { assert_eq!(values_slice, values); assert_eq!( - vortex_values.validity_mask().unwrap(), + vortex_values + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_values.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(3, vec![0, 2]) ); } @@ -611,7 +621,15 @@ mod tests { assert_eq!(vortex_slice, values); assert_eq!( - vortex_array.validity_mask().unwrap(), + vortex_array + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(3, vec![0, 2]) ); } @@ -778,7 +796,15 @@ mod tests { PrimitiveArray::from_option_iter([Some(1i32), Some(2), Some(3), Some(4)]) ); assert_eq!( - vortex_array.validity_mask().unwrap(), + vortex_array + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(2, vec![0]) ); } @@ -891,7 +917,15 @@ mod tests { assert_eq!(sizes.as_slice::()[1], 0); assert_eq!( - vortex_array.validity_mask().unwrap(), + vortex_array + .as_ref() + .validity() + .unwrap() + .to_mask( + vortex_array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx() + ) + .unwrap(), Mask::from_indices(3, vec![0, 2]) ); } diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index 5b52f6d6c53..66c96e3a2d0 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -135,6 +135,7 @@ pub fn precision_to_duckdb_storage_size(decimal_dtype: &DecimalDType) -> VortexR #[cfg(test)] mod tests { + use vortex::array::LEGACY_SESSION; use vortex::array::VortexSessionExecute; use vortex::array::arrays::DecimalArray; use vortex::dtype::DecimalDType; @@ -148,7 +149,10 @@ mod tests { pub(crate) fn new_zero_copy_exporter( array: &DecimalArray, ) -> VortexResult> { - let validity = array.validity_mask()?; + let validity = array.as_ref().validity()?.to_mask( + array.as_ref().len(), + &mut LEGACY_SESSION.create_execution_ctx(), + )?; let dest_values_type = precision_to_duckdb_storage_size(&array.decimal_dtype())?; assert_eq!(array.values_type(), dest_values_type); diff --git a/vortex-duckdb/src/exporter/dict.rs b/vortex-duckdb/src/exporter/dict.rs index 6377906940c..d576a85c148 100644 --- a/vortex-duckdb/src/exporter/dict.rs +++ b/vortex-duckdb/src/exporter/dict.rs @@ -42,16 +42,19 @@ pub(crate) fn new_exporter_with_flatten( ) -> VortexResult> { // Grab the cache dictionary values. let values = array.values(); + let codes = array.codes(); + let codes_len = codes.len(); + if let Some(constant) = values.as_opt::() { return constant::new_exporter_with_mask( - ConstantArray::new(constant.scalar().clone(), array.codes().len()), - array.codes().validity_mask()?, + ConstantArray::new(constant.scalar().clone(), codes_len), + codes.validity()?.to_mask(codes_len, ctx)?, cache, ctx, ); } - let codes_mask = array.codes().validity_mask()?; + let codes_mask = codes.validity()?.to_mask(codes_len, ctx)?; match codes_mask { Mask::AllTrue(_) => {} diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 5f684622ee6..0a196598171 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -10,7 +10,9 @@ use std::sync::Arc; use paste::paste; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::NullArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::struct_::StructArrayExt; @@ -251,7 +253,7 @@ pub unsafe extern "C-unwind" fn vx_array_invalid_count( try_or_default(error_out, || { vortex_ensure!(!array.is_null()); let array = vx_array::as_ref(array); - array.invalid_count() + array.invalid_count(&mut LEGACY_SESSION.create_execution_ctx()) }) } diff --git a/vortex-jni/src/array.rs b/vortex-jni/src/array.rs index c45f232b920..88c7286b3b4 100644 --- a/vortex-jni/src/array.rs +++ b/vortex-jni/src/array.rs @@ -28,7 +28,9 @@ use jni::sys::jshort; use jni::sys::jstring; use vortex::array::ArrayRef; use vortex::array::ArrayView; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::VarBin; use vortex::array::arrays::VarBinView; use vortex::array::arrays::extension::ExtensionArrayExt; @@ -280,7 +282,9 @@ pub extern "system" fn Java_dev_vortex_jni_NativeArrayMethods_getNullCount( ) -> jint { let array_ref = unsafe { NativeArray::from_ptr(array_ptr) }; try_or_throw(&mut env, |_| { - let count = array_ref.inner.invalid_count()?; + let count = array_ref + .inner + .invalid_count(&mut LEGACY_SESSION.create_execution_ctx())?; Ok(jint::try_from(count).unwrap_or(-1)) }) } diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 1ffb669d08e..c301a696784 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -260,7 +260,9 @@ mod tests { use rstest::rstest; use vortex_array::ArrayContext; use vortex_array::IntoArray as _; + use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinArray; @@ -516,7 +518,12 @@ mod tests { .unwrap() .await .unwrap(); - let expected = array.validity_mask().unwrap().into_array(); + let expected = array + .validity() + .unwrap() + .to_mask(array.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .into_array(); assert_arrays_eq!( actual .to_canonical() diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index 0d804ee6fb8..e8c78c2e2a7 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -199,8 +199,10 @@ mod tests { use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; + use vortex_array::LEGACY_SESSION; use vortex_array::MaskFuture; use vortex_array::ToCanonical; + use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::Dict; use vortex_array::arrays::DictArray; @@ -393,7 +395,12 @@ mod tests { .unwrap(); assert_eq!( - result.validity_mask().unwrap().bit_buffer(), + result + .validity() + .unwrap() + .to_mask(result.len(), &mut LEGACY_SESSION.create_execution_ctx()) + .unwrap() + .bit_buffer(), AllOr::Some(&validity_boolean_buffer) ); assert_eq!( diff --git a/vortex-layout/src/layouts/table.rs b/vortex-layout/src/layouts/table.rs index f8a0cee3581..0c602cd78fa 100644 --- a/vortex-layout/src/layouts/table.rs +++ b/vortex-layout/src/layouts/table.rs @@ -16,7 +16,9 @@ use itertools::Itertools; use vortex_array::ArrayContext; use vortex_array::ArrayRef; use vortex_array::IntoArray; +use vortex_array::LEGACY_SESSION; use vortex_array::ToCanonical; +use vortex_array::VortexSessionExecute; use vortex_array::arrays::struct_::StructArrayExt; use vortex_array::dtype::DType; use vortex_array::dtype::Field; @@ -239,7 +241,10 @@ impl LayoutStrategy for TableStrategy { if is_nullable { columns.push(( sequence_pointer.advance(), - chunk.validity_mask()?.into_array(), + chunk + .validity()? + .to_mask(chunk.len(), &mut LEGACY_SESSION.create_execution_ctx())? + .into_array(), )); } diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index f0cc68e2f34..6f6a0666d8c 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -24,7 +24,9 @@ use pyo3::types::PyRangeMethods; use pyo3_bytes::PyBytes; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::Chunked; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::chunked::ChunkedArrayExt; @@ -525,7 +527,9 @@ impl PyArray { /// ``` fn filter(slf: Bound, mask: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); - let mask = (&*mask as &ArrayRef).to_bool().to_mask_fill_null_false(); + let mask = (&*mask as &ArrayRef) + .to_bool() + .to_mask_fill_null_false(&mut LEGACY_SESSION.create_execution_ctx()); let inner = slf.filter(mask)?.to_canonical()?.into_array(); Ok(PyArrayRef::from(inner)) } diff --git a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs index 6cf60cf4270..4c4175b773d 100644 --- a/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs +++ b/vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs @@ -5,6 +5,8 @@ use vortex::array::ArrayId; use vortex::array::ArrayRef; use vortex::array::ArrayVTable; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::StructArray; use vortex::array::dtype::FieldNames; @@ -130,17 +132,72 @@ impl FlatLayoutFixture for AlpFixture { "f64_boundary_specials", ]), vec![ - alp_encode(f64_prices.as_view(), None)?.into_array(), - alp_encode(f32_near_int.as_view(), None)?.into_array(), - alp_encode(f64_negative_near_int.as_view(), None)?.into_array(), - alp_encode(f64_currency.as_view(), None)?.into_array(), - alp_encode(f64_nullable.as_view(), None)?.into_array(), - alp_encode(f64_patched.as_view(), None)?.into_array(), - alp_encode(f64_patch_heavy.as_view(), None)?.into_array(), - alp_encode(f64_special_values.as_view(), None)?.into_array(), - alp_encode(f32_special_values.as_view(), None)?.into_array(), - alp_encode(f64_extremes.as_view(), None)?.into_array(), - alp_encode(f64_boundary_specials.as_view(), None)?.into_array(), + alp_encode( + f64_prices.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f32_near_int.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_negative_near_int.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_currency.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_nullable.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_patched.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_patch_heavy.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_special_values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f32_special_values.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_extremes.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), + alp_encode( + f64_boundary_specials.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + )? + .into_array(), ], N, Validity::NonNullable, diff --git a/vortex/benches/common_encoding_tree_throughput.rs b/vortex/benches/common_encoding_tree_throughput.rs index 54fe1b61926..4a0edf99389 100644 --- a/vortex/benches/common_encoding_tree_throughput.rs +++ b/vortex/benches/common_encoding_tree_throughput.rs @@ -14,7 +14,9 @@ use rand::RngExt; use rand::SeedableRng; use vortex::array::ArrayRef; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; +use vortex::array::VortexSessionExecute; use vortex::array::arrays::DictArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::TemporalArray; @@ -102,7 +104,12 @@ mod setup { /// Create ALP <- FoR <- BitPacked encoding tree for f64 pub fn alp_for_bp_f64() -> ArrayRef { let (_, _, float_array) = setup_primitive_arrays(); - let alp_compressed = alp_encode(float_array.as_view(), None).unwrap(); + let alp_compressed = alp_encode( + float_array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); // Manually construct ALP <- FoR <- BitPacked tree let for_array = FoR::encode(alp_compressed.encoded().to_primitive()).unwrap(); diff --git a/vortex/benches/single_encoding_throughput.rs b/vortex/benches/single_encoding_throughput.rs index 4cedf23c07d..3c8777bc3d3 100644 --- a/vortex/benches/single_encoding_throughput.rs +++ b/vortex/benches/single_encoding_throughput.rs @@ -15,6 +15,7 @@ use rand::SeedableRng; use rand::prelude::IndexedRandom; use rand::rngs::StdRng; use vortex::array::IntoArray; +use vortex::array::LEGACY_SESSION; use vortex::array::ToCanonical; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::VarBinViewArray; @@ -257,13 +258,25 @@ fn bench_alp_compress_f64(bencher: Bencher) { with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &float_array) - .bench_refs(|a| alp_encode(a.as_view(), None).unwrap()); + .bench_refs(|a| { + alp_encode( + a.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap() + }); } #[divan::bench(name = "alp_decompress_f64")] fn bench_alp_decompress_f64(bencher: Bencher) { let (_, _, float_array) = setup_primitive_arrays(); - let compressed = alp_encode(float_array.as_view(), None).unwrap(); + let compressed = alp_encode( + float_array.as_view(), + None, + &mut LEGACY_SESSION.create_execution_ctx(), + ) + .unwrap(); with_byte_counter(bencher, NUM_VALUES * 8) .with_inputs(|| &compressed)