Skip to content

Commit 56111e3

Browse files
matthiasrichtermatthiasrichter
committed
Make raw page data generator for DPLUtils unit tests a separate class
There will be more unit tests and micro benchmarks using the data generator.
1 parent 74698c3 commit 56111e3

4 files changed

Lines changed: 187 additions & 125 deletions

File tree

‎Framework/Utils/CMakeLists.txt‎

Lines changed: 12 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ o2_add_library(DPLUtils
1818
src/RawParser.cxx
1919
test/DPLBroadcasterMerger.cxx
2020
test/DPLOutputTest.cxx
21+
test/RawPageTestData.cxx
2122
PUBLIC_LINK_LIBRARIES O2::Framework)
2223

2324
o2_add_executable(raw-proxy
@@ -54,12 +55,6 @@ o2_add_test(DPLOutput
5455
LABELS long dplutils
5556
COMMAND_LINE_ARGS ${DPL_WORKFLOW_TESTS_EXTRA_OPTIONS} --run)
5657

57-
o2_add_test(RootTreeWriter
58-
SOURCES test/test_RootTreeWriter.cxx
59-
PUBLIC_LINK_LIBRARIES O2::DPLUtils
60-
COMPONENT_NAME DPLUtils
61-
LABELS dplutils)
62-
6358
o2_add_test(RootTreeWriterWorkflow
6459
NO_BOOST_TEST
6560
SOURCES test/test_RootTreeWriterWorkflow.cxx
@@ -76,17 +71,18 @@ o2_add_test(RootTreeReader
7671
LABELS dplutils
7772
COMMAND_LINE_ARGS ${DPL_WORKFLOW_TESTS_EXTRA_OPTIONS} --run)
7873

79-
o2_add_test(RawParser
80-
SOURCES test/test_RawParser.cxx
81-
PUBLIC_LINK_LIBRARIES O2::DPLUtils
82-
COMPONENT_NAME DPLUtils
83-
LABELS dplutils)
8474

85-
o2_add_test(DPLRawParser
86-
SOURCES test/test_DPLRawParser.cxx
87-
PUBLIC_LINK_LIBRARIES O2::DPLUtils
88-
COMPONENT_NAME DPLUtils
89-
LABELS dplutils)
75+
foreach(t
76+
RootTreeWriter
77+
RawParser
78+
DPLRawParser
79+
)
80+
o2_add_test(${t}
81+
SOURCES test/test_${t}.cxx
82+
PUBLIC_LINK_LIBRARIES O2::DPLUtils
83+
COMPONENT_NAME DPLUtils
84+
LABELS dplutils)
85+
endforeach()
9086

9187
if (TARGET benchmark::benchmark)
9288
foreach(b

‎Framework/Utils/test/RawPageTestData.cxx‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
/// @file RawPageTestData.cxx
13+
/// @author Matthias Richter
14+
/// @since 2021-06-21
15+
/// @brief Raw page test data generator
16+
17+
#include "RawPageTestData.h"
18+
#include "Headers/Stack.h"
19+
#include <random>
20+
21+
namespace o2::framework
22+
{
23+
namespace test
24+
{
25+
26+
DataSet createData(std::vector<InputSpec> const& inputspecs, std::vector<DataHeader> const& dataheaders, AmendRawDataHeader amendRdh)
27+
{
28+
// Create the routes we want for the InputRecord
29+
size_t i = 0;
30+
auto createRoute = [&i](std::string const& source, InputSpec const& spec) {
31+
return InputRoute{
32+
spec,
33+
i++,
34+
source};
35+
};
36+
37+
std::vector<InputRoute> schema;
38+
for (auto const& spec : inputspecs) {
39+
auto routename = spec.binding + "_source";
40+
schema.emplace_back(createRoute(routename, spec));
41+
}
42+
43+
std::random_device rd;
44+
std::uniform_int_distribution<> testvals(0, 42);
45+
auto randval = [&rd, &testvals]() {
46+
return testvals(rd);
47+
};
48+
std::vector<int> checkValues;
49+
DataSet::Messages messages;
50+
51+
auto initRawPage = [&checkValues, &amendRdh](char* buffer, size_t size, auto value) {
52+
char* wrtptr = buffer;
53+
while (wrtptr < buffer + size) {
54+
auto* header = reinterpret_cast<RAWDataHeader*>(wrtptr);
55+
*header = RAWDataHeader();
56+
if (amendRdh) {
57+
amendRdh(*header);
58+
}
59+
header->offsetToNext = PAGESIZE;
60+
*reinterpret_cast<decltype(value)*>(wrtptr + header->headerSize) = value;
61+
wrtptr += PAGESIZE;
62+
checkValues.emplace_back(value);
63+
++value;
64+
}
65+
};
66+
67+
auto createMessage = [&messages, &initRawPage, &randval](DataHeader dh) {
68+
DataProcessingHeader dph{0, 1};
69+
Stack stack{dh, dph};
70+
if (dh.splitPayloadParts == 0 || dh.splitPayloadIndex == 0) {
71+
// add new message collection
72+
messages.emplace_back();
73+
}
74+
messages.back().emplace_back(std::make_unique<std::vector<char>>(stack.size()));
75+
memcpy(messages.back().back()->data(), stack.data(), messages.back().back()->size());
76+
messages.back().emplace_back(std::make_unique<std::vector<char>>(dh.payloadSize));
77+
int value = randval();
78+
initRawPage(messages.back().back()->data(), messages.back().back()->size(), value);
79+
};
80+
81+
// create messages for the provided dataheaders
82+
for (auto header : dataheaders) {
83+
for (DataHeader::SplitPayloadIndexType index = 0; index == 0 || index < header.splitPayloadParts; index++) {
84+
header.splitPayloadIndex = index;
85+
createMessage(header);
86+
}
87+
}
88+
89+
return {std::move(schema), std::move(messages), std::move(checkValues)};
90+
}
91+
92+
} // namespace test
93+
} // namespace o2::framework

‎Framework/Utils/test/RawPageTestData.h‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
/// @file RawPageTestData.h
13+
/// @author Matthias Richter
14+
/// @since 2021-06-21
15+
/// @brief Raw page test data generator
16+
17+
#ifndef FRAMEWORK_UTILS_RAWPAGETESTDATA_H
18+
#define FRAMEWORK_UTILS_RAWPAGETESTDATA_H
19+
20+
#include "Framework/InputRecord.h"
21+
#include "Framework/InputSpan.h"
22+
#include "Headers/RAWDataHeader.h"
23+
#include "Headers/DataHeader.h"
24+
#include <vector>
25+
#include <memory>
26+
27+
using DataHeader = o2::header::DataHeader;
28+
using Stack = o2::header::Stack;
29+
using RAWDataHeaderV6 = o2::header::RAWDataHeaderV6;
30+
31+
namespace o2::framework
32+
{
33+
namespace test
34+
{
35+
using RAWDataHeader = RAWDataHeaderV6;
36+
static const size_t PAGESIZE = 8192;
37+
38+
/// @class DataSet
39+
/// @brief Simple helper struct to keep the InputRecord and ownership of messages
40+
/// together with some test data.
41+
struct DataSet {
42+
// not nice with the double vector but for quick unit test ok
43+
using Messages = std::vector<std::vector<std::unique_ptr<std::vector<char>>>>;
44+
DataSet(std::vector<InputRoute>&& s, Messages&& m, std::vector<int>&& v)
45+
: schema{std::move(s)},
46+
messages{std::move(m)},
47+
span{[this](size_t i, size_t part) {
48+
auto header = static_cast<char const*>(this->messages[i].at(2 * part)->data());
49+
auto payload = static_cast<char const*>(this->messages[i].at(2 * part + 1)->data());
50+
return DataRef{nullptr, header, payload};
51+
},
52+
[this](size_t i) { return i < this->messages.size() ? messages[i].size() / 2 : 0; }, this->messages.size()},
53+
record{schema, span},
54+
values{std::move(v)}
55+
{
56+
}
57+
58+
std::vector<InputRoute> schema;
59+
Messages messages;
60+
InputSpan span;
61+
InputRecord record;
62+
std::vector<int> values;
63+
};
64+
65+
using AmendRawDataHeader = std::function<void(RAWDataHeader&)>;
66+
DataSet createData(std::vector<InputSpec> const& inputspecs, std::vector<DataHeader> const& dataheaders, AmendRawDataHeader amendRdh = nullptr);
67+
68+
} // namespace test
69+
} // namespace o2::framework
70+
#endif // FRAMEWORK_UTILS_RAWPAGETESTDATA_H

‎Framework/Utils/test/test_DPLRawParser.cxx‎

Lines changed: 12 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -14,133 +14,36 @@
1414
#define BOOST_TEST_DYN_LINK
1515
#include <boost/test/unit_test.hpp>
1616
#include "DPLUtils/DPLRawParser.h"
17+
#include "RawPageTestData.h"
1718
#include "Framework/InputRecord.h"
18-
#include "Framework/InputSpan.h"
1919
#include "Framework/WorkflowSpec.h" // o2::framework::select
2020
#include "Headers/DataHeader.h"
21-
#include "Headers/Stack.h"
2221
#include <vector>
2322
#include <memory>
2423
#include <iostream>
2524

2625
using namespace o2::framework;
2726
using DataHeader = o2::header::DataHeader;
28-
using Stack = o2::header::Stack;
29-
using RAWDataHeaderV4 = o2::header::RAWDataHeaderV4;
30-
31-
static const size_t PAGESIZE = 8192;
32-
33-
// simple helper struct to keep the InputRecord and ownership of messages
34-
struct DataSet {
35-
// not nice with the double vector but for quick unit test ok
36-
using Messages = std::vector<std::vector<std::unique_ptr<std::vector<char>>>>;
37-
DataSet(std::vector<InputRoute>&& s, Messages&& m, std::vector<int>&& v)
38-
: schema{std::move(s)},
39-
messages{std::move(m)},
40-
span{[this](size_t i, size_t part) {
41-
BOOST_REQUIRE(i < this->messages.size());
42-
BOOST_REQUIRE(part < this->messages[i].size() / 2);
43-
auto header = static_cast<char const*>(this->messages[i].at(2 * part)->data());
44-
auto payload = static_cast<char const*>(this->messages[i].at(2 * part + 1)->data());
45-
return DataRef{nullptr, header, payload};
46-
},
47-
[this](size_t i) { return i < this->messages.size() ? messages[i].size() / 2 : 0; }, this->messages.size()},
48-
record{schema, span},
49-
values{std::move(v)}
50-
{
51-
BOOST_REQUIRE(messages.size() == schema.size());
52-
}
53-
54-
std::vector<InputRoute> schema;
55-
Messages messages;
56-
InputSpan span;
57-
InputRecord record;
58-
std::vector<int> values;
59-
};
27+
auto const PAGESIZE = test::PAGESIZE;
28+
using DataSet = test::DataSet;
6029

6130
DataSet createData()
6231
{
63-
// Create the routes we want for the InputRecord
6432
std::vector<InputSpec> inputspecs = {
6533
InputSpec{"tpc0", "TPC", "RAWDATA", 0, Lifetime::Timeframe},
6634
InputSpec{"its1", "ITS", "RAWDATA", 0, Lifetime::Timeframe},
6735
InputSpec{"its1", "ITS", "RAWDATA", 1, Lifetime::Timeframe}};
6836

69-
size_t i = 0;
70-
auto createRoute = [&i](const char* source, InputSpec& spec) {
71-
return InputRoute{
72-
spec,
73-
i++,
74-
source};
75-
};
76-
77-
std::vector<InputRoute> schema = {
78-
createRoute("tpc_source", inputspecs[0]),
79-
createRoute("its_source", inputspecs[1]),
80-
createRoute("tof_source", inputspecs[2])};
81-
82-
std::vector<int> checkValues;
83-
DataSet::Messages messages;
84-
85-
auto initRawPage = [&checkValues](char* buffer, size_t size, int value) {
86-
char* wrtptr = buffer;
87-
while (wrtptr < buffer + size) {
88-
auto* header = reinterpret_cast<RAWDataHeaderV4*>(wrtptr);
89-
*header = RAWDataHeaderV4();
90-
header->offsetToNext = PAGESIZE;
91-
*reinterpret_cast<decltype(value)*>(wrtptr + header->headerSize) = value;
92-
wrtptr += PAGESIZE;
93-
checkValues.emplace_back(value);
94-
++value;
95-
}
96-
};
97-
98-
auto createMessage = [&messages, &initRawPage](DataHeader dh, int value) {
99-
DataProcessingHeader dph{0, 1};
100-
Stack stack{dh, dph};
101-
if (dh.splitPayloadParts == 0 || dh.splitPayloadIndex == 0) {
102-
// add new message collection
103-
messages.emplace_back();
104-
}
105-
messages.back().emplace_back(std::make_unique<std::vector<char>>(stack.size()));
106-
memcpy(messages.back().back()->data(), stack.data(), messages.back().back()->size());
107-
messages.back().emplace_back(std::make_unique<std::vector<char>>(dh.payloadSize));
108-
initRawPage(messages.back().back()->data(), messages.back().back()->size(), value);
109-
};
110-
11137
// we create message for the 3 input routes, the messages have different page size
11238
// and the second messages has 3 parts, each with the same page size
11339
// the test value is written as payload after the RDH and all values are cached for
11440
// later checking when parsing the data set
115-
DataHeader dh1;
116-
dh1.dataDescription = "RAWDATA";
117-
dh1.dataOrigin = "TPC";
118-
dh1.subSpecification = 0;
119-
dh1.payloadSerializationMethod = o2::header::gSerializationMethodNone;
120-
dh1.payloadSize = 5 * PAGESIZE;
121-
DataHeader dh2;
122-
dh2.dataDescription = "RAWDATA";
123-
dh2.dataOrigin = "ITS";
124-
dh2.subSpecification = 0;
125-
dh2.payloadSerializationMethod = o2::header::gSerializationMethodNone;
126-
dh2.payloadSize = 3 * PAGESIZE;
127-
dh2.splitPayloadParts = 3;
128-
dh2.splitPayloadIndex = 0;
129-
DataHeader dh3;
130-
dh3.dataDescription = "RAWDATA";
131-
dh3.dataOrigin = "ITS";
132-
dh3.subSpecification = 1;
133-
dh3.payloadSerializationMethod = o2::header::gSerializationMethodNone;
134-
dh3.payloadSize = 4 * PAGESIZE;
135-
createMessage(dh1, 10);
136-
createMessage(dh2, 20);
137-
dh2.splitPayloadIndex++;
138-
createMessage(dh2, 23);
139-
dh2.splitPayloadIndex++;
140-
createMessage(dh2, 26);
141-
createMessage(dh3, 30);
41+
std::vector<DataHeader> dataheaders;
42+
dataheaders.emplace_back("RAWDATA", "TPC", 0, 5 * PAGESIZE);
43+
dataheaders.emplace_back("RAWDATA", "ITS", 0, 3 * PAGESIZE, 0, 3);
44+
dataheaders.emplace_back("RAWDATA", "ITS", 1, 4 * PAGESIZE);
14245

143-
return {std::move(schema), std::move(messages), std::move(checkValues)};
46+
return test::createData(inputspecs, dataheaders);
14447
}
14548

14649
BOOST_AUTO_TEST_CASE(test_DPLRawParser)
@@ -157,8 +60,8 @@ BOOST_AUTO_TEST_CASE(test_DPLRawParser)
15760
for (auto it = parser.begin(), end = parser.end(); it != end; ++it, ++count) {
15861
LOG(INFO) << "data " << count << " " << *((int*)it.data());
15962
// now check the iterator API
160-
// retrieving RDH v4
161-
auto const* rdh = it.get_if<o2::header::RAWDataHeaderV4>();
63+
// retrieving RDH
64+
auto const* rdh = it.get_if<test::RAWDataHeader>();
16265
// retrieving the raw pointer of the page
16366
auto const* raw = it.raw();
16467
// retrieving payload pointer of the page
@@ -168,10 +71,10 @@ BOOST_AUTO_TEST_CASE(test_DPLRawParser)
16871
// offset of payload in the raw page
16972
size_t offset = it.offset();
17073
BOOST_REQUIRE(rdh != nullptr);
171-
BOOST_REQUIRE(offset == sizeof(o2::header::RAWDataHeaderV4));
74+
BOOST_REQUIRE(offset == sizeof(test::RAWDataHeader));
17275
BOOST_REQUIRE(payload == raw + offset);
17376
BOOST_REQUIRE(*reinterpret_cast<int const*>(payload) == dataset.values[count]);
174-
BOOST_REQUIRE(payloadSize == PAGESIZE - sizeof(o2::header::RAWDataHeaderV4));
77+
BOOST_REQUIRE(payloadSize == PAGESIZE - sizeof(test::RAWDataHeader));
17578
auto const* dh = it.o2DataHeader();
17679
if (last != dh) {
17780
// this is a special wrapper to print the RDU info and table header, this will

0 commit comments

Comments
 (0)