Skip to content

Commit 24736e1

Browse files
gatesngatesn
authored
Move Array::serialize -> ArrayPlugin::Serialize (#7348)
The cheeky thing here is that now we require a VortexSession to serialize metadata: 1. Makes it hard to serialize arrays you didn't intend to. All arrays must be registered at least. 2. Allows ScalarFnArray to serialize metadata using a dedicated ArrayPlugin for that ScalarFn. So not all scalar function vtables need to define array serde semantics, they just register an optional plugin. This will allow us to port existing scalar arrays to scalar functions without a wire break --------- Signed-off-by: Nicholas Gates <nick@nickgates.com>
1 parent 1748d17 commit 24736e1

131 files changed

Lines changed: 1500 additions & 1051 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎AGENTS.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@
1313
* you can try running
1414
`cargo fix --lib --allow-dirty --allow-staged && cargo clippy --fix --lib --allow-dirty --allow-staged` to
1515
automatically many fix minor errors.
16+
* when iterating on CI failures, fetch only failed job logs first (`gh run view <run-id> --job <job-id> --log-failed`)
17+
and run narrow local repro commands for the affected crate/tests before running workspace-wide checks.
18+
* if `gh` commands fail with `error connecting to api.github.com` in sandbox, immediately rerun with escalated network
19+
permissions instead of retrying in sandbox.
20+
* if cargo fails with `sccache: error: Operation not permitted`, rerun the command with `RUSTC_WRAPPER=` so rustc runs
21+
directly.
22+
* run docs doctests from the docs directory (`make -C docs doctest`) so the correct Sphinx Makefile target is used.
1623

1724
## Architecture
1825

@@ -64,4 +71,4 @@
6471

6572
## Commits
6673

67-
* All commits must be signed of by the committers in the form `Signed-off-by: "COMMITTER" <COMMITTER_EMAIL>`.
74+
* All commits must be signed of by the committers in the form `Signed-off-by: "COMMITTER" <COMMITTER_EMAIL>`.

‎encodings/alp/public-api.lock‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ pub fn vortex_alp::ALP::nbuffers(_array: vortex_array::array::view::ArrayView<'_
4848

4949
pub fn vortex_alp::ALP::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
5050

51-
pub fn vortex_alp::ALP::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
51+
pub fn vortex_alp::ALP::serialize(array: vortex_array::array::view::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
5252

5353
pub fn vortex_alp::ALP::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
5454

@@ -112,6 +112,10 @@ impl core::fmt::Debug for vortex_alp::ALPData
112112

113113
pub fn vortex_alp::ALPData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
114114

115+
impl core::fmt::Display for vortex_alp::ALPData
116+
117+
pub fn vortex_alp::ALPData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
118+
115119
impl vortex_array::hash::ArrayEq for vortex_alp::ALPData
116120

117121
pub fn vortex_alp::ALPData::array_eq(&self, other: &Self, _precision: vortex_array::hash::Precision) -> bool
@@ -182,7 +186,7 @@ pub fn vortex_alp::ALPRD::nbuffers(_array: vortex_array::array::view::ArrayView<
182186

183187
pub fn vortex_alp::ALPRD::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
184188

185-
pub fn vortex_alp::ALPRD::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
189+
pub fn vortex_alp::ALPRD::serialize(array: vortex_array::array::view::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
186190

187191
pub fn vortex_alp::ALPRD::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
188192

@@ -236,6 +240,10 @@ impl core::fmt::Debug for vortex_alp::ALPRDData
236240

237241
pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
238242

243+
impl core::fmt::Display for vortex_alp::ALPRDData
244+
245+
pub fn vortex_alp::ALPRDData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
246+
239247
impl vortex_array::hash::ArrayEq for vortex_alp::ALPRDData
240248

241249
pub fn vortex_alp::ALPRDData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool

‎encodings/alp/src/alp/array.rs‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::fmt::Debug;
5+
use std::fmt::Display;
6+
use std::fmt::Formatter;
57
use std::hash::Hash;
68
use std::hash::Hasher;
79

@@ -107,7 +109,10 @@ impl VTable for ALP {
107109
None
108110
}
109111

110-
fn serialize(array: ArrayView<'_, Self>) -> VortexResult<Option<Vec<u8>>> {
112+
fn serialize(
113+
array: ArrayView<'_, Self>,
114+
_session: &VortexSession,
115+
) -> VortexResult<Option<Vec<u8>>> {
111116
let exponents = array.exponents();
112117
Ok(Some(
113118
ALPMetadata {
@@ -223,6 +228,16 @@ pub struct ALPData {
223228
exponents: Exponents,
224229
}
225230

231+
impl Display for ALPData {
232+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
233+
write!(f, "exponents: {}", self.exponents)?;
234+
if let Some(offset) = self.patch_offset {
235+
write!(f, ", patch_offset: {offset}")?;
236+
}
237+
Ok(())
238+
}
239+
}
240+
226241
#[derive(Clone, Debug)]
227242
pub struct ALP;
228243

‎encodings/alp/src/alp_rd/array.rs‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::fmt::Debug;
5+
use std::fmt::Display;
6+
use std::fmt::Formatter;
57
use std::hash::Hash;
68
use std::hash::Hasher;
79

@@ -127,7 +129,10 @@ impl VTable for ALPRD {
127129
None
128130
}
129131

130-
fn serialize(array: ArrayView<'_, Self>) -> VortexResult<Option<Vec<u8>>> {
132+
fn serialize(
133+
array: ArrayView<'_, Self>,
134+
_session: &VortexSession,
135+
) -> VortexResult<Option<Vec<u8>>> {
131136
let dict = array
132137
.left_parts_dictionary()
133138
.iter()
@@ -330,6 +335,16 @@ pub struct ALPRDData {
330335
right_bit_width: u8,
331336
}
332337

338+
impl Display for ALPRDData {
339+
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
340+
write!(f, "right_bit_width: {}", self.right_bit_width)?;
341+
if let Some(offset) = self.patch_offset {
342+
write!(f, ", patch_offset: {offset}")?;
343+
}
344+
Ok(())
345+
}
346+
}
347+
333348
#[derive(Clone, Debug)]
334349
pub struct ALPRDDataParts {
335350
pub left_parts: ArrayRef,

‎encodings/bytebool/public-api.lock‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pub fn vortex_bytebool::ByteBool::nbuffers(_array: vortex_array::array::view::Ar
4444

4545
pub fn vortex_bytebool::ByteBool::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
4646

47-
pub fn vortex_bytebool::ByteBool::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
47+
pub fn vortex_bytebool::ByteBool::serialize(_array: vortex_array::array::view::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
4848

4949
pub fn vortex_bytebool::ByteBool::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
5050

@@ -108,6 +108,10 @@ impl core::fmt::Debug for vortex_bytebool::ByteBoolData
108108

109109
pub fn vortex_bytebool::ByteBoolData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
110110

111+
impl core::fmt::Display for vortex_bytebool::ByteBoolData
112+
113+
pub fn vortex_bytebool::ByteBoolData::fmt(&self, _f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
114+
111115
impl vortex_array::hash::ArrayEq for vortex_bytebool::ByteBoolData
112116

113117
pub fn vortex_bytebool::ByteBoolData::array_eq(&self, other: &Self, precision: vortex_array::hash::Precision) -> bool

‎encodings/bytebool/src/array.rs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::fmt::Debug;
5+
use std::fmt::Display;
6+
use std::fmt::Formatter;
57
use std::hash::Hasher;
68

79
use vortex_array::Array;
@@ -92,7 +94,10 @@ impl VTable for ByteBool {
9294
}
9395
}
9496

95-
fn serialize(_array: ArrayView<'_, Self>) -> VortexResult<Option<Vec<u8>>> {
97+
fn serialize(
98+
_array: ArrayView<'_, Self>,
99+
_session: &VortexSession,
100+
) -> VortexResult<Option<Vec<u8>>> {
96101
Ok(Some(vec![]))
97102
}
98103

@@ -170,6 +175,12 @@ pub struct ByteBoolData {
170175
buffer: BufferHandle,
171176
}
172177

178+
impl Display for ByteBoolData {
179+
fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
180+
Ok(())
181+
}
182+
}
183+
173184
pub trait ByteBoolArrayExt: TypedArrayRef<ByteBool> {
174185
fn validity(&self) -> Validity {
175186
child_to_validity(
@@ -341,6 +352,7 @@ impl From<Vec<Option<bool>>> for ByteBoolData {
341352
mod tests {
342353
use vortex_array::ArrayContext;
343354
use vortex_array::IntoArray;
355+
use vortex_array::LEGACY_SESSION;
344356
use vortex_array::assert_arrays_eq;
345357
use vortex_array::serde::SerializeOptions;
346358
use vortex_array::serde::SerializedArray;
@@ -395,7 +407,7 @@ mod tests {
395407
let serialized = array
396408
.clone()
397409
.into_array()
398-
.serialize(&ctx, &SerializeOptions::default())
410+
.serialize(&ctx, &LEGACY_SESSION, &SerializeOptions::default())
399411
.unwrap();
400412

401413
let mut concat = ByteBufferMut::empty();

‎encodings/datetime-parts/public-api.lock‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ pub fn vortex_datetime_parts::DateTimeParts::nbuffers(_array: vortex_array::arra
4242

4343
pub fn vortex_datetime_parts::DateTimeParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
4444

45-
pub fn vortex_datetime_parts::DateTimeParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
45+
pub fn vortex_datetime_parts::DateTimeParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
4646

4747
pub fn vortex_datetime_parts::DateTimeParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
4848

@@ -100,6 +100,10 @@ impl core::fmt::Debug for vortex_datetime_parts::DateTimePartsData
100100

101101
pub fn vortex_datetime_parts::DateTimePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
102102

103+
impl core::fmt::Display for vortex_datetime_parts::DateTimePartsData
104+
105+
pub fn vortex_datetime_parts::DateTimePartsData::fmt(&self, _f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
106+
103107
impl vortex_array::hash::ArrayEq for vortex_datetime_parts::DateTimePartsData
104108

105109
pub fn vortex_datetime_parts::DateTimePartsData::array_eq(&self, _other: &Self, _precision: vortex_array::hash::Precision) -> bool

‎encodings/datetime-parts/src/array.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use std::fmt::Debug;
5+
use std::fmt::Display;
6+
use std::fmt::Formatter;
57
use std::hash::Hasher;
68

79
use prost::Message;
@@ -124,7 +126,10 @@ impl VTable for DateTimeParts {
124126
vortex_panic!("DateTimePartsArray buffer_name index {idx} out of bounds")
125127
}
126128

127-
fn serialize(array: ArrayView<'_, Self>) -> VortexResult<Option<Vec<u8>>> {
129+
fn serialize(
130+
array: ArrayView<'_, Self>,
131+
_session: &VortexSession,
132+
) -> VortexResult<Option<Vec<u8>>> {
128133
Ok(Some(
129134
DateTimePartsMetadata {
130135
days_ptype: PType::try_from(array.days().dtype())? as i32,
@@ -213,6 +218,12 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["days", "seconds", "subseconds
213218
#[derive(Clone, Debug)]
214219
pub struct DateTimePartsData {}
215220

221+
impl Display for DateTimePartsData {
222+
fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
223+
Ok(())
224+
}
225+
}
226+
216227
pub trait DateTimePartsArrayExt: TypedArrayRef<DateTimeParts> {
217228
fn days(&self) -> &ArrayRef {
218229
self.as_ref().slots()[DAYS_SLOT]

‎encodings/decimal-byte-parts/public-api.lock‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ pub fn vortex_decimal_byte_parts::DecimalByteParts::nbuffers(_array: vortex_arra
4040

4141
pub fn vortex_decimal_byte_parts::DecimalByteParts::reduce_parent(array: vortex_array::array::view::ArrayView<'_, Self>, parent: &vortex_array::array::erased::ArrayRef, child_idx: usize) -> vortex_error::VortexResult<core::option::Option<vortex_array::array::erased::ArrayRef>>
4242

43-
pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
43+
pub fn vortex_decimal_byte_parts::DecimalByteParts::serialize(array: vortex_array::array::view::ArrayView<'_, Self>, _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
4444

4545
pub fn vortex_decimal_byte_parts::DecimalByteParts::slot_name(_array: vortex_array::array::view::ArrayView<'_, Self>, idx: usize) -> alloc::string::String
4646

@@ -92,6 +92,10 @@ impl core::fmt::Debug for vortex_decimal_byte_parts::DecimalBytePartsData
9292

9393
pub fn vortex_decimal_byte_parts::DecimalBytePartsData::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
9494

95+
impl core::fmt::Display for vortex_decimal_byte_parts::DecimalBytePartsData
96+
97+
pub fn vortex_decimal_byte_parts::DecimalBytePartsData::fmt(&self, _f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
98+
9599
impl vortex_array::hash::ArrayEq for vortex_decimal_byte_parts::DecimalBytePartsData
96100

97101
pub fn vortex_decimal_byte_parts::DecimalBytePartsData::array_eq(&self, _other: &Self, _precision: vortex_array::hash::Precision) -> bool

‎encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// SPDX-License-Identifier: Apache-2.0
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

4+
use std::fmt::Display;
5+
use std::fmt::Formatter;
46
use std::hash::Hasher;
57

68
use vortex_array::Array;
@@ -104,7 +106,10 @@ impl VTable for DecimalByteParts {
104106
vortex_panic!("DecimalBytePartsArray buffer_name index {idx} out of bounds")
105107
}
106108

107-
fn serialize(array: ArrayView<'_, Self>) -> VortexResult<Option<Vec<u8>>> {
109+
fn serialize(
110+
array: ArrayView<'_, Self>,
111+
_session: &VortexSession,
112+
) -> VortexResult<Option<Vec<u8>>> {
108113
Ok(Some(
109114
DecimalBytesPartsMetadata {
110115
zeroth_child_ptype: PType::try_from(array.msp().dtype())? as i32,
@@ -186,6 +191,12 @@ pub struct DecimalBytePartsData {
186191
_lower_parts: Vec<ArrayRef>,
187192
}
188193

194+
impl Display for DecimalBytePartsData {
195+
fn fmt(&self, _f: &mut Formatter<'_>) -> std::fmt::Result {
196+
Ok(())
197+
}
198+
}
199+
189200
pub struct DecimalBytePartsDataParts {
190201
pub msp: ArrayRef,
191202
}

0 commit comments

Comments
 (0)