Skip to content

Commit f5da6a1

Browse files
ktfktf
committed
DPL Analysis: add initial support for boolean columns
For the moment we use 1 byte per bool.
1 parent bae3141 commit f5da6a1

4 files changed

Lines changed: 22 additions & 10 deletions

File tree

‎Framework/Core/include/Framework/ASoA.h‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,10 @@ template <typename T>
8989
struct arrow_array_for {
9090
};
9191
template <>
92+
struct arrow_array_for<bool> {
93+
using type = arrow::Int8Array;
94+
};
95+
template <>
9296
struct arrow_array_for<int8_t> {
9397
using type = arrow::Int8Array;
9498
};
@@ -165,7 +169,7 @@ class ColumnIterator : ChunkingPolicy
165169
{
166170
auto chunks = mColumn->data();
167171
auto array = std::static_pointer_cast<arrow_array_for_t<T>>(chunks->chunk(mCurrentChunk));
168-
mCurrent = array->raw_values();
172+
mCurrent = reinterpret_cast<T const*>(array->raw_values());
169173
mLast = mCurrent + array->length();
170174
}
171175

@@ -184,7 +188,7 @@ class ColumnIterator : ChunkingPolicy
184188
mFirstIndex += previousArray->length();
185189
mCurrentChunk++;
186190
auto array = std::static_pointer_cast<arrow_array_for_t<T>>(chunks->chunk(mCurrentChunk));
187-
mCurrent = array->raw_values() - mFirstIndex;
191+
mCurrent = reinterpret_cast<T const*>(array->raw_values() - mFirstIndex);
188192
mLast = mCurrent + array->length() + mFirstIndex;
189193
}
190194

@@ -195,7 +199,7 @@ class ColumnIterator : ChunkingPolicy
195199
mFirstIndex -= previousArray->length();
196200
mCurrentChunk--;
197201
auto array = std::static_pointer_cast<arrow_array_for_t<T>>(chunks->chunk(mCurrentChunk));
198-
mCurrent = array->raw_values() - mFirstIndex;
202+
mCurrent = reinterpret_cast<T const*>(array->raw_values() - mFirstIndex);
199203
mLast = mCurrent + array->length() + mFirstIndex;
200204
}
201205

@@ -220,7 +224,7 @@ class ColumnIterator : ChunkingPolicy
220224
auto array = std::static_pointer_cast<arrow_array_for_t<T>>(chunks->chunk(mCurrentChunk));
221225
assert(array.get());
222226
mFirstIndex = mColumn->length() - array->length();
223-
mCurrent = array->raw_values() - mFirstIndex;
227+
mCurrent = reinterpret_cast<T const*>(array->raw_values() - mFirstIndex);
224228
mLast = mCurrent + array->length() + mFirstIndex;
225229
}
226230

‎Framework/Core/include/Framework/TableBuilder.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ struct ConversionTraits {
5757
using ArrowType = ::arrow::ArrowType_; \
5858
};
5959

60-
O2_ARROW_STL_CONVERSION(bool, BooleanType)
60+
// FIXME: for now we use Int8 to store booleans
61+
O2_ARROW_STL_CONVERSION(bool, Int8Type)
6162
O2_ARROW_STL_CONVERSION(int8_t, Int8Type)
6263
O2_ARROW_STL_CONVERSION(int16_t, Int16Type)
6364
O2_ARROW_STL_CONVERSION(int32_t, Int32Type)

‎Framework/Core/test/test_ASoA.cxx‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,11 @@ DECLARE_SOA_TABLE(Points, "TST", "POINTS", test::X, test::Y);
3636

3737
namespace test
3838
{
39+
DECLARE_SOA_COLUMN(SomeBool, someBool, bool, "someBool");
3940
DECLARE_SOA_COLUMN(Color, color, int32_t, "color");
4041
} // namespace test
4142

42-
DECLARE_SOA_TABLE(Infos, "TST", "INFOS", test::Color);
43+
DECLARE_SOA_TABLE(Infos, "TST", "INFOS", test::Color, test::SomeBool);
4344

4445
namespace test
4546
{
@@ -456,11 +457,15 @@ BOOST_AUTO_TEST_CASE(TestDereference)
456457

457458
TableBuilder builderA2;
458459
auto infoWriter = builderA2.cursor<Infos>();
459-
infoWriter(0, 0);
460-
infoWriter(0, 1);
461-
infoWriter(0, 4);
460+
infoWriter(0, 0, 1);
461+
infoWriter(0, 1, 0);
462+
infoWriter(0, 4, 1);
462463
auto infosT = builderA2.finalize();
463464
Infos infos{infosT};
465+
BOOST_REQUIRE_EQUAL(infos.begin().someBool(), true);
466+
BOOST_REQUIRE_EQUAL((infos.begin() + 1).someBool(), false);
467+
BOOST_REQUIRE_EQUAL((infos.begin() + 2).someBool(), true);
468+
BOOST_REQUIRE_EQUAL((infos.begin() + 2).color(), 4);
464469
BOOST_REQUIRE_EQUAL(infosT->num_rows(), 3);
465470

466471
TableBuilder builderB;
@@ -485,6 +490,7 @@ BOOST_AUTO_TEST_CASE(TestDereference)
485490
i.bindExternalIndices(&points, &infos);
486491
BOOST_CHECK_EQUAL(i.n(), 10);
487492
BOOST_CHECK_EQUAL(i.info().color(), 4);
493+
BOOST_CHECK_EQUAL(i.info().someBool(), true);
488494
BOOST_CHECK_EQUAL(i.pointA().x(), 0);
489495
BOOST_CHECK_EQUAL(i.pointA().y(), 0);
490496
BOOST_CHECK_EQUAL(i.pointB().x(), 3);
@@ -494,6 +500,7 @@ BOOST_AUTO_TEST_CASE(TestDereference)
494500
auto j = segments.begin();
495501
BOOST_CHECK_EQUAL(j.n(), 10);
496502
BOOST_CHECK_EQUAL(j.info().color(), 4);
503+
BOOST_CHECK_EQUAL(j.info().someBool(), true);
497504
BOOST_CHECK_EQUAL(j.pointA().x(), 0);
498505
BOOST_CHECK_EQUAL(j.pointA().y(), 0);
499506
BOOST_CHECK_EQUAL(j.pointB().x(), 3);

‎Framework/Core/test/test_TableBuilder.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ BOOST_AUTO_TEST_CASE(TestTableBuilderMore)
151151
BOOST_REQUIRE_EQUAL(table->column(0)->type()->id(), arrow::int32()->id());
152152
BOOST_REQUIRE_EQUAL(table->column(1)->type()->id(), arrow::float32()->id());
153153
BOOST_REQUIRE_EQUAL(table->column(2)->type()->id(), arrow::utf8()->id());
154-
BOOST_REQUIRE_EQUAL(table->column(3)->type()->id(), arrow::boolean()->id());
154+
BOOST_REQUIRE_EQUAL(table->column(3)->type()->id(), arrow::int8()->id());
155155
}
156156

157157
// Use RDataFrame to build the table

0 commit comments

Comments
 (0)