Skip to content

Commit 4187c1a

Browse files
gatesngatesn
authored
Use aggregate descriptors for zoned stats (#7938)
This PR adds a new layout type: vortex.zoned During deserialization, we read the old zone map layout `vortex.stats` into a `vortex.zoned` in memory. The new zoned layout now stores aggregate descriptors instead of legacy `Stat` bits, enabling arbitrary aggregate functions to be used as zone statistics. FLUPs: * Move FooterStats over to using AggregateFns. Or even just a root-level ZonedLayout with a single zone? --------- Signed-off-by: Nicholas Gates <nick@nickgates.com>
1 parent 5d55fbc commit 4187c1a

34 files changed

Lines changed: 1762 additions & 645 deletions

File tree

‎vortex-array/src/aggregate_fn/fns/all_nan/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex_error::VortexResult;
5+
use vortex_session::VortexSession;
56

67
use crate::ArrayRef;
78
use crate::Columnar;
@@ -37,7 +38,15 @@ impl AggregateFnVTable for AllNan {
3738
}
3839

3940
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
40-
Ok(None)
41+
Ok(Some(vec![]))
42+
}
43+
44+
fn deserialize(
45+
&self,
46+
_metadata: &[u8],
47+
_session: &VortexSession,
48+
) -> VortexResult<Self::Options> {
49+
Ok(EmptyOptions)
4150
}
4251

4352
fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/all_non_nan/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex_error::VortexResult;
5+
use vortex_session::VortexSession;
56

67
use crate::ArrayRef;
78
use crate::Columnar;
@@ -37,7 +38,15 @@ impl AggregateFnVTable for AllNonNan {
3738
}
3839

3940
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
40-
Ok(None)
41+
Ok(Some(vec![]))
42+
}
43+
44+
fn deserialize(
45+
&self,
46+
_metadata: &[u8],
47+
_session: &VortexSession,
48+
) -> VortexResult<Self::Options> {
49+
Ok(EmptyOptions)
4150
}
4251

4352
fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/all_non_null/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex_error::VortexResult;
5+
use vortex_session::VortexSession;
56

67
use crate::ArrayRef;
78
use crate::Columnar;
@@ -29,7 +30,15 @@ impl AggregateFnVTable for AllNonNull {
2930
}
3031

3132
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
32-
Ok(None)
33+
Ok(Some(vec![]))
34+
}
35+
36+
fn deserialize(
37+
&self,
38+
_metadata: &[u8],
39+
_session: &VortexSession,
40+
) -> VortexResult<Self::Options> {
41+
Ok(EmptyOptions)
3342
}
3443

3544
fn return_dtype(&self, _options: &Self::Options, _input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/all_null/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// SPDX-FileCopyrightText: Copyright the Vortex contributors
33

44
use vortex_error::VortexResult;
5+
use vortex_session::VortexSession;
56

67
use crate::ArrayRef;
78
use crate::Columnar;
@@ -29,7 +30,15 @@ impl AggregateFnVTable for AllNull {
2930
}
3031

3132
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
32-
Ok(None)
33+
Ok(Some(vec![]))
34+
}
35+
36+
fn deserialize(
37+
&self,
38+
_metadata: &[u8],
39+
_session: &VortexSession,
40+
) -> VortexResult<Self::Options> {
41+
Ok(EmptyOptions)
3342
}
3443

3544
fn return_dtype(&self, _options: &Self::Options, _input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/max/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use vortex_error::VortexExpect;
55
use vortex_error::VortexResult;
6+
use vortex_session::VortexSession;
67

78
use crate::ArrayRef;
89
use crate::Columnar;
@@ -52,7 +53,15 @@ impl AggregateFnVTable for Max {
5253
}
5354

5455
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
55-
Ok(None)
56+
Ok(Some(vec![]))
57+
}
58+
59+
fn deserialize(
60+
&self,
61+
_metadata: &[u8],
62+
_session: &VortexSession,
63+
) -> VortexResult<Self::Options> {
64+
Ok(EmptyOptions)
5665
}
5766

5867
fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/min/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
use vortex_error::VortexExpect;
55
use vortex_error::VortexResult;
6+
use vortex_session::VortexSession;
67

78
use crate::ArrayRef;
89
use crate::Columnar;
@@ -52,7 +53,15 @@ impl AggregateFnVTable for Min {
5253
}
5354

5455
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
55-
Ok(None)
56+
Ok(Some(vec![]))
57+
}
58+
59+
fn deserialize(
60+
&self,
61+
_metadata: &[u8],
62+
_session: &VortexSession,
63+
) -> VortexResult<Self::Options> {
64+
Ok(EmptyOptions)
5665
}
5766

5867
fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/nan_count/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ use vortex_error::VortexExpect;
77
use vortex_error::VortexResult;
88
use vortex_error::vortex_bail;
99
use vortex_error::vortex_err;
10+
use vortex_session::VortexSession;
1011

1112
use self::primitive::accumulate_primitive;
1213
use crate::ArrayRef;
@@ -87,7 +88,15 @@ impl AggregateFnVTable for NanCount {
8788
}
8889

8990
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
90-
unimplemented!("NanCount is not yet serializable");
91+
Ok(Some(vec![]))
92+
}
93+
94+
fn deserialize(
95+
&self,
96+
_metadata: &[u8],
97+
_session: &VortexSession,
98+
) -> VortexResult<Self::Options> {
99+
Ok(EmptyOptions)
91100
}
92101

93102
fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/null_count/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
use vortex_error::VortexExpect;
55
use vortex_error::VortexResult;
66
use vortex_error::vortex_err;
7+
use vortex_session::VortexSession;
78

89
use crate::ArrayRef;
910
use crate::Columnar;
@@ -62,7 +63,15 @@ impl AggregateFnVTable for NullCount {
6263
}
6364

6465
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
65-
Ok(None)
66+
Ok(Some(vec![]))
67+
}
68+
69+
fn deserialize(
70+
&self,
71+
_metadata: &[u8],
72+
_session: &VortexSession,
73+
) -> VortexResult<Self::Options> {
74+
Ok(EmptyOptions)
6675
}
6776

6877
fn return_dtype(&self, _options: &Self::Options, _input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/sum/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use vortex_error::VortexResult;
1212
use vortex_error::vortex_bail;
1313
use vortex_error::vortex_err;
1414
use vortex_error::vortex_panic;
15+
use vortex_session::VortexSession;
1516

1617
use self::bool::accumulate_bool;
1718
use self::constant::multiply_constant;
@@ -76,7 +77,15 @@ impl AggregateFnVTable for Sum {
7677
}
7778

7879
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
79-
unimplemented!("Sum is not yet serializable");
80+
Ok(Some(vec![]))
81+
}
82+
83+
fn deserialize(
84+
&self,
85+
_metadata: &[u8],
86+
_session: &VortexSession,
87+
) -> VortexResult<Self::Options> {
88+
Ok(EmptyOptions)
8089
}
8190

8291
fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {

‎vortex-array/src/aggregate_fn/fns/uncompressed_size_in_bytes/mod.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use vortex_error::VortexResult;
2727
use vortex_error::vortex_bail;
2828
use vortex_error::vortex_err;
2929
use vortex_mask::Mask;
30+
use vortex_session::VortexSession;
3031

3132
use crate::ArrayRef;
3233
use crate::Canonical;
@@ -104,7 +105,15 @@ impl AggregateFnVTable for UncompressedSizeInBytes {
104105
}
105106

106107
fn serialize(&self, _options: &Self::Options) -> VortexResult<Option<Vec<u8>>> {
107-
unimplemented!("UncompressedSizeInBytes is not yet serializable");
108+
Ok(Some(vec![]))
109+
}
110+
111+
fn deserialize(
112+
&self,
113+
_metadata: &[u8],
114+
_session: &VortexSession,
115+
) -> VortexResult<Self::Options> {
116+
Ok(EmptyOptions)
108117
}
109118

110119
fn return_dtype(&self, _options: &Self::Options, input_dtype: &DType) -> Option<DType> {

0 commit comments

Comments
 (0)