diff --git a/Algorithm/CMakeLists.txt b/Algorithm/CMakeLists.txt index 14aeb5c47c4ba..65361dff57df4 100644 --- a/Algorithm/CMakeLists.txt +++ b/Algorithm/CMakeLists.txt @@ -41,6 +41,7 @@ set(TEST_SRCS test/headerstack.cxx test/parser.cxx test/tableview.cxx + test/pageparser.cxx ) O2_GENERATE_TESTS( diff --git a/Algorithm/include/Algorithm/PageParser.h b/Algorithm/include/Algorithm/PageParser.h new file mode 100644 index 0000000000000..285d35aad799b --- /dev/null +++ b/Algorithm/include/Algorithm/PageParser.h @@ -0,0 +1,287 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALGORITHM_PAGEPARSER_H +#define ALGORITHM_PAGEPARSER_H + +/// @file PageParser.h +/// @author Matthias Richter +/// @since 2017-09-27 +/// @brief Parser for a set of data objects in consecutive memory pages. + +#include +#include +#include +#include +#include + +namespace o2 { + +namespace algorithm { + +/** + * @class PageParser + * Parser for a set of data objects in consecutive memory pages. + * + * All memory pages have a fixed size and start with a page header. + * Depending on the page size and size of the data object, some + * objects can be split at the page boundary and have the page header + * embedded. + * + * The class iterator can be used to iterate over the data objects + * transparently. + * + * Usage: + * RawParser RawParser; + * RawParser parser(ptr, size); + * for (auto element : parser) { + * // do something with element + * } + */ +template +class PageParser { +public: + using PageHeaderType = PageHeaderT; + using BufferType = unsigned char; + using value_type = ElementT; + using GroupType = GroupT; + using GetNElements = std::function; + static const size_t page_size = PageSize; + + // at the moment an object can only be split among two pages + static_assert(PageSize >= sizeof(PageHeaderType) + sizeof(value_type), + "Page Header and at least one element have to fit into page"); + + // switches for the copy method, used to skip ill-formed expressions + using TargetInPageBuffer = std::true_type; + using SourceInPageBuffer = std::false_type; + + PageParser() = delete; + template + PageParser(T* buffer, size_t size, + GetNElements getNElementsFct = [] (const GroupType&) {return 0;} + ) + : mBuffer(nullptr) + , mBufferIsConst(std::is_const::value) + , mSize(size) + , mGetNElementsFct(getNElementsFct) + , mNPages(size>0?((size-1)/page_size)+1:0) + { + static_assert(sizeof(T) == sizeof(BufferType), + "buffer required to be byte-type"); + + // the buffer pointer is stored non-const, a runtime check ensures + // that iterator write works only for non-const buffers + mBuffer = const_cast(buffer); + } + ~PageParser() = default; + + template + using IteratorBase = std::iterator; + + template + class Iterator : public IteratorBase { + public: + using ParentType = PageParser; + using SelfType = Iterator; + using value_type = typename IteratorBase::value_type; + using reference = typename IteratorBase::reference; + using pointer = typename IteratorBase::pointer; + using ElementType = typename std::remove_const::type; + + + Iterator() = delete; + + Iterator(ParentType const * parent, size_t position = 0) + : mParent(parent) + { + mPosition = position; + mNextPosition = mParent->getElement(mPosition, mElement); + backup(); + } + ~Iterator() + { + sync(); + } + + // prefix increment + SelfType& operator++() { + sync(); + mPosition = mNextPosition; + mNextPosition = mParent->getElement(mPosition, mElement); + backup(); + return *this; + } + // postfix increment + SelfType operator++(int /*unused*/) { + SelfType copy(*this); operator++(); return copy; + } + // return reference + reference operator*() { + return mElement; + } + // comparison + bool operator==(const SelfType& rh) { + return mPosition == rh.mPosition; + } + // comparison + bool operator!=(const SelfType& rh) { + return mPosition != rh.mPosition; + } + + private: + // sync method for non-const iterator + template< typename U = void > + typename std::enable_if< !std::is_const::value, U >::type sync() { + if (std::memcmp(&mElement, &mBackup, sizeof(value_type)) != 0) { + // mElement is changed, sync to buffer + mParent->setElement(mPosition, mElement); + } + } + + // overload for const_iterator, empty function body + template< typename U = void > + typename std::enable_if< std::is_const::value, U >::type sync() {} + + // backup for non-const iterator + template< typename U = void > + typename std::enable_if< !std::is_const::value, U >::type backup() { + mBackup = mElement; + } + + // overload for const_iterator, empty function body + template< typename U = void > + typename std::enable_if< std::is_const::value, U >::type backup() {} + + int mPosition; + int mNextPosition; + ParentType const * mParent; + ElementType mElement; + ElementType mBackup; + }; + + /// set an object at position + size_t setElement(size_t position, const value_type& element) const { + // check if we are at the end + if (position >= mSize) { + assert(position == mSize); + return mSize; + } + + // check if there is space for one element + if (position + sizeof(value_type) > mSize) { + // format error, probably throw exception + return mSize; + } + + auto source = reinterpret_cast(&element); + auto target = mBuffer + position; + return position + copy(source, target, page_size - (position % page_size)); + } + + /// retrieve an object at position + size_t getElement(size_t position, value_type& element) const { + // check if we are at the end + if (position >= mSize) { + assert(position == mSize); + return mSize; + } + + // check if there is space for one element + if (position + sizeof(value_type) > mSize) { + // format error, probably throw exception + return mSize; + } + + auto source = mBuffer + position; + auto target = reinterpret_cast(&element); + return position + copy(source, target, page_size - (position % page_size)); + } + + // copy data, depending on compile time switch, either source or target + // pointer are treated as pointer in the raw page, i.e. can be additionally + // incremented by the page header + template + size_t copy(const BufferType* source, BufferType* target, size_t pageCapacity) const + { + size_t position = 0; + auto copySize = sizeof(value_type); + // choose which of the pointers needs additional PageHeader offsets + auto pageOffsetTarget = SwitchT::value? &target : const_cast(&source); + if (pageCapacity == page_size) { + // skip the page header at beginning of page + position += sizeof(PageHeaderType); + pageCapacity -= sizeof(PageHeaderType); + *pageOffsetTarget += sizeof(PageHeaderType); + } + if (copySize > pageCapacity) { + // object is split at the page boundary, copy the part + // in the current page first + copySize = pageCapacity; + } + if (copySize > 0) { + memcpy(target, source, copySize); + position += copySize; + source += copySize; + target += copySize; + } + copySize = sizeof(value_type) - copySize; + if (copySize > 0) { + // skip page header at beginning of new page and copy + // remaining part of the element + position += sizeof(PageHeaderType); + *pageOffsetTarget += sizeof(PageHeaderType); + memcpy(target, source, copySize); + position += copySize; + } + return position; + } + + using iterator = Iterator; + using const_iterator = Iterator; + + const_iterator begin() const { + return const_iterator(this, 0); + } + + const_iterator end() const { + return const_iterator(this, mSize); + } + + iterator begin() { + if (mBufferIsConst) { + // did not find a way to do this at compile time in the constructor, + // probably one needs to make the buffer type a template parameter + // to the class + throw std::runtime_error("the underlying buffer is not writeable"); + } + return iterator(this, 0); + } + + iterator end() { + return iterator(this, mSize); + } + +private: + BufferType* mBuffer = nullptr; + bool mBufferIsConst = false; + size_t mSize = 0; + GetNElements mGetNElementsFct; + size_t mNPages = 0; +}; + +} +} + +#endif diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx new file mode 100644 index 0000000000000..ac4d5cada8904 --- /dev/null +++ b/Algorithm/test/pageparser.cxx @@ -0,0 +1,165 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file parser.cxx +/// @author Matthias Richter +/// @since 2017-09-27 +/// @brief Unit test for parser of objects in memory pages + +#define BOOST_TEST_MODULE Test Algorithm Parser +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include +#include +#include +#include +#include "Headers/DataHeader.h" // hexdump +#include "../include/Algorithm/PageParser.h" +#include "StaticSequenceAllocator.h" + +struct PageHeader { + uint32_t magic = 0x45474150; + uint32_t pageid; + + PageHeader(uint32_t id) : pageid(id) {} +}; + +struct ClusterData { + uint32_t magic = 0x54534c43; + uint32_t clusterid; + uint16_t x; + uint16_t y; + uint16_t z; + uint8_t e; + + ClusterData() + : clusterid(0) + , x(0) + , y(0) + , z(0) + , e(0) + {} + + ClusterData(uint32_t _id, uint16_t _x, uint16_t _y, uint16_t _z, uint8_t _e) + : clusterid(_id) + , x(_x) + , y(_y) + , z(_z) + , e(_e) + {} + + bool operator==(const ClusterData& rhs) { + return clusterid == rhs.clusterid + && x == rhs.x + && y == rhs.y + && z == rhs.z + && e == rhs.e; + } +}; + +template +std::pair, size_t> MakeBuffer(unsigned pagesize, + PageHeaderT pageheader, + ListT& dataset) +{ + auto totalSize = dataset.size() * sizeof(typename ListT::value_type); + unsigned nPages = 0; + do { + totalSize += sizeof(PageHeaderT); + ++nPages; + } while (nPages * pagesize < totalSize); + + std::unique_ptr buffer(new uint8_t[totalSize]); + + unsigned position = 0; + auto target = buffer.get(); + for (auto element : dataset) { + auto source = reinterpret_cast(&element); + auto copySize = sizeof(typename ListT::value_type); + if ((position % pagesize) == 0) { + memcpy(target, &pageheader, sizeof(PageHeaderT)); + position += sizeof(PageHeaderT); + target += sizeof(PageHeaderT); + } + if ((position % pagesize) + copySize > pagesize) { + copySize = ((position % pagesize) + copySize) - pagesize; + } + if (copySize > 0) { + memcpy(target, source, copySize); + position += copySize; + target += copySize; + source += copySize; + } + copySize = sizeof(typename ListT::value_type) - copySize; + if (copySize > 0) { + memcpy(target, &pageheader, sizeof(PageHeaderT)); + position += sizeof(PageHeaderT); + target += sizeof(PageHeaderT); + memcpy(target, source, copySize); + } + position += copySize; + target += copySize; + } + + std::pair, size_t> result; + result.first = std::move(buffer); + result.second = totalSize; + return result; +} + +template +void FillData(ListT& dataset, unsigned entries) +{ + for (unsigned i = 0; i < entries; i++) { + dataset.emplace_back(i, 0xaa, 0xbb, 0xcc, 0xd); + } +} + +BOOST_AUTO_TEST_CASE(test_pageparser) +{ + constexpr unsigned pagesize = 128; + std::vector dataset; + FillData(dataset, 20); + auto buffer = MakeBuffer(pagesize, PageHeader(0), dataset); + o2::Header::hexDump("pagebuffer", buffer.first.get(), buffer.second); + + using RawParser = o2::algorithm::PageParser; + const RawParser parser(buffer.first.get(), buffer.second); + + unsigned dataidx = 0; + for (auto i : parser) { + o2::Header::hexDump("clusterdata", &i, sizeof(ClusterData)); + BOOST_REQUIRE( i == dataset[dataidx++]); + } + + std::vector linearizedData; + linearizedData.insert(linearizedData.begin(), parser.begin(), parser.end()); + dataidx = 0; + for (auto i : linearizedData) { + BOOST_REQUIRE( i == dataset[dataidx++]); + } + + dataidx = 0; + RawParser writer(buffer.first.get(), buffer.second); + std::vector> xvalues; + for (auto &i : writer) { + i.x = (dataidx * 3) % 7; + xvalues.emplace_back(i.x, dataidx); + ++dataidx; + } + o2::Header::hexDump("changed buffer", buffer.first.get(), buffer.second); + + dataidx = 0; + for (auto i : parser) { + o2::Header::hexDump("clusterdata", &i, sizeof(ClusterData)); + BOOST_REQUIRE( i.x == xvalues[dataidx++].first); + } +}