From 73aa5d3c87616346e3ada4d2ede31eccc587b6ed Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 27 Sep 2017 23:28:41 +0200 Subject: [PATCH 1/2] Adding parser for a set of data objects in consecutive memory pages Memory pages have a fixed size and start with a page header. The parser provides transparent iteration through data objects and handles page boundaries. --- Algorithm/CMakeLists.txt | 1 + Algorithm/include/Algorithm/PageParser.h | 173 +++++++++++++++++++++++ Algorithm/test/pageparser.cxx | 149 +++++++++++++++++++ 3 files changed, 323 insertions(+) create mode 100644 Algorithm/include/Algorithm/PageParser.h create mode 100644 Algorithm/test/pageparser.cxx diff --git a/Algorithm/CMakeLists.txt b/Algorithm/CMakeLists.txt index 14aeb5c47c4ba..65361dff57df4 100644 --- a/Algorithm/CMakeLists.txt +++ b/Algorithm/CMakeLists.txt @@ -41,6 +41,7 @@ set(TEST_SRCS test/headerstack.cxx test/parser.cxx test/tableview.cxx + test/pageparser.cxx ) O2_GENERATE_TESTS( diff --git a/Algorithm/include/Algorithm/PageParser.h b/Algorithm/include/Algorithm/PageParser.h new file mode 100644 index 0000000000000..314f03faf9699 --- /dev/null +++ b/Algorithm/include/Algorithm/PageParser.h @@ -0,0 +1,173 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALGORITHM_PAGEPARSER_H +#define ALGORITHM_PAGEPARSER_H + +/// @file PageParser.h +/// @author Matthias Richter +/// @since 2017-09-27 +/// @brief Parser for a set of data objects in consecutive memory pages. + +#include +#include + +namespace o2 { + +namespace algorithm { + +/** + * @class PageParser + * Parser for a set of data objects in consecutive memory pages. + * + * All memory pages have a fixed size and start with a page header. + * Depending on the page size and size of the data object, some + * objects can be split at the page boundary and have the page header + * embedded. + * + * The class iterator can be used to iterate over the data objects + * transparently. + * + * Usage: + * RawParser RawParser; + * RawParser parser(ptr, size); + * for (auto element : parser) { + * // do something with element + * } + */ +template +class PageParser { +public: + using PageHeaderType = PageHeaderT; + using BufferType = unsigned char; + using value_type = ElementType; + using GroupType = GroupT; + using GetNElements = std::function; + static const size_t page_size = PageSize; + + PageParser() = delete; + PageParser(const BufferType* buffer, size_t size, + GetNElements getNElementsFct = [] (const GroupType&) {return 0;} + ) + : mBuffer(buffer) + , mSize(size) + , mGetNElementsFct(getNElementsFct) + , mNPages(size>0?(size/page_size)+1:0) + { + } + ~PageParser() = default; + + using IteratorBase = std::iterator; + + class Iterator : public IteratorBase { + public: + using ParentType = PageParser; + using SelfType = Iterator; + using value_type = typename IteratorBase::value_type; + using reference = typename IteratorBase::reference; + using pointer = typename IteratorBase::pointer; + + Iterator() = delete; + + Iterator(const ParentType & parent, size_t position = 0) + : mParent(parent) + { + mPosition = mParent.getElement(position, mElement); + } + ~Iterator() = default; + + // prefix increment + SelfType& operator++() { + mPosition = mParent.getElement(mPosition, mElement); + return *this; + } + // postfix increment + SelfType operator++(int /*unused*/) { + SelfType copy(*this); operator++(); return copy; + } + // return reference + reference operator*() { + return mElement; + } + // comparison + bool operator==(const SelfType& rh) { + return mPosition == rh.mPosition; + } + // comparison + bool operator!=(const SelfType& rh) { + return mPosition != rh.mPosition; + } + + private: + int mPosition; + const ParentType& mParent; + value_type mElement; + }; + + /// retrieve an object at position + size_t getElement(size_t position, value_type& element) const { + // check if we are at the end + if (position == mSize) return position; + + // check if there is space for one element + if (position + sizeof(value_type) > mSize) { + // format error, probably throw exception + return mSize; + } + auto copySize = sizeof(value_type); + auto target = reinterpret_cast(&element); + if ((position % page_size) == 0) { + // skip the page header at beginning of page + position += sizeof(PageHeaderType); + } + if ((position % page_size) + copySize > page_size) { + // object is split at the page boundary, copy the part + // in the current page first + copySize = ((position % page_size) + copySize) - page_size; + } + if (copySize > 0) { + memcpy(target, mBuffer + position, copySize); + position += copySize; + target += copySize; + } + copySize = sizeof(value_type) - copySize; + if (copySize > 0) { + // skip page header at beginning of new page and copy + // remaining part of the element + position += sizeof(PageHeaderType); + memcpy(target, mBuffer + position, copySize); + position += copySize; + } + return position; + } + + Iterator begin() const { + return Iterator(*this, 0); + } + + Iterator end() const { + return Iterator(*this, mSize); + } + +private: + const BufferType* mBuffer = nullptr; + size_t mSize = 0; + GetNElements mGetNElementsFct; + size_t mNPages = 0; +}; + +} +} + +#endif diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx new file mode 100644 index 0000000000000..f6385ddfc9de4 --- /dev/null +++ b/Algorithm/test/pageparser.cxx @@ -0,0 +1,149 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file parser.cxx +/// @author Matthias Richter +/// @since 2017-09-27 +/// @brief Unit test for parser of objects in memory pages + +#define BOOST_TEST_MODULE Test Algorithm Parser +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include +#include +#include +#include +#include "Headers/DataHeader.h" // hexdump +#include "../include/Algorithm/PageParser.h" +#include "StaticSequenceAllocator.h" + +struct PageHeader { + uint32_t magic = 0x45474150; + uint32_t pageid; + + PageHeader(uint32_t id) : pageid(id) {} +}; + +struct ClusterData { + uint32_t magic = 0x54534c43; + uint32_t clusterid; + uint16_t x; + uint16_t y; + uint16_t z; + uint8_t e; + + ClusterData() + : clusterid(0) + , x(0) + , y(0) + , z(0) + , e(0) + {} + + ClusterData(uint32_t _id, uint16_t _x, uint16_t _y, uint16_t _z, uint8_t _e) + : clusterid(_id) + , x(_x) + , y(_y) + , z(_z) + , e(_e) + {} + + bool operator==(const ClusterData& rhs) { + return clusterid == rhs.clusterid + && x == rhs.x + && y == rhs.y + && z == rhs.z + && e == rhs.e; + } +}; + +template +std::pair, size_t> MakeBuffer(unsigned pagesize, + PageHeaderT pageheader, + ListT& dataset) +{ + auto totalSize = dataset.size() * sizeof(typename ListT::value_type); + unsigned nPages = 0; + do { + totalSize += sizeof(PageHeaderT); + ++nPages; + } while (nPages * pagesize < totalSize); + + std::unique_ptr buffer(new uint8_t[totalSize]); + + unsigned position = 0; + auto target = buffer.get(); + for (auto element : dataset) { + auto source = reinterpret_cast(&element); + auto copySize = sizeof(typename ListT::value_type); + if ((position % pagesize) == 0) { + memcpy(target, &pageheader, sizeof(PageHeaderT)); + position += sizeof(PageHeaderT); + target += sizeof(PageHeaderT); + } + if ((position % pagesize) + copySize > pagesize) { + copySize = ((position % pagesize) + copySize) - pagesize; + } + if (copySize > 0) { + memcpy(target, source, copySize); + position += copySize; + target += copySize; + source += copySize; + } + copySize = sizeof(typename ListT::value_type) - copySize; + if (copySize > 0) { + memcpy(target, &pageheader, sizeof(PageHeaderT)); + position += sizeof(PageHeaderT); + target += sizeof(PageHeaderT); + memcpy(target, source, copySize); + } + position += copySize; + target += copySize; + } + + std::pair, size_t> result; + result.first = std::move(buffer); + result.second = totalSize; + return result; +} + +template +void FillData(ListT& dataset, unsigned entries) +{ + for (unsigned i = 0; i < entries; i++) { + dataset.emplace_back(i, 0xaa, 0xbb, 0xcc, 0xd); + } +} + +BOOST_AUTO_TEST_CASE(test_pageparser) +{ + constexpr unsigned pagesize = 128; + std::vector dataset; + FillData(dataset, 20); + auto buffer = MakeBuffer(pagesize, PageHeader(0), dataset); + o2::Header::hexDump("pagebuffer", buffer.first.get(), buffer.second); + + using RawParser = o2::algorithm::PageParser; + RawParser parser(buffer.first.get(), buffer.second); + + unsigned dataidx = 0; + for (auto i : parser) { + o2::Header::hexDump("clusterdata", &i, sizeof(ClusterData)); + BOOST_REQUIRE( i == dataset[dataidx++]); + } + + std::vector linearizedData; + linearizedData.insert(linearizedData.begin(), parser.begin(), parser.end()); + dataidx = 0; + for (auto i : linearizedData) { + BOOST_REQUIRE( i == dataset[dataidx++]); + } +} From bc280d7282607e3961a87d0f55fcb14ca3cbf3b3 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Mon, 16 Oct 2017 13:00:14 +0200 Subject: [PATCH 2/2] Adding write functionality for parser iterator - bidirectional copy function - runtype check for the buffer const'ness - supporting iterator and const_iterator - fixing the assignment operator problem on xcode, now internally using pointer instead of reference. Using reference was resulting in making and assigning a copy. --- Algorithm/include/Algorithm/PageParser.h | 170 +++++++++++++++++++---- Algorithm/test/pageparser.cxx | 18 ++- 2 files changed, 159 insertions(+), 29 deletions(-) diff --git a/Algorithm/include/Algorithm/PageParser.h b/Algorithm/include/Algorithm/PageParser.h index 314f03faf9699..285d35aad799b 100644 --- a/Algorithm/include/Algorithm/PageParser.h +++ b/Algorithm/include/Algorithm/PageParser.h @@ -18,6 +18,9 @@ #include #include +#include +#include +#include namespace o2 { @@ -44,52 +47,80 @@ namespace algorithm { */ template class PageParser { public: using PageHeaderType = PageHeaderT; using BufferType = unsigned char; - using value_type = ElementType; + using value_type = ElementT; using GroupType = GroupT; using GetNElements = std::function; static const size_t page_size = PageSize; + // at the moment an object can only be split among two pages + static_assert(PageSize >= sizeof(PageHeaderType) + sizeof(value_type), + "Page Header and at least one element have to fit into page"); + + // switches for the copy method, used to skip ill-formed expressions + using TargetInPageBuffer = std::true_type; + using SourceInPageBuffer = std::false_type; + PageParser() = delete; - PageParser(const BufferType* buffer, size_t size, + template + PageParser(T* buffer, size_t size, GetNElements getNElementsFct = [] (const GroupType&) {return 0;} ) - : mBuffer(buffer) + : mBuffer(nullptr) + , mBufferIsConst(std::is_const::value) , mSize(size) , mGetNElementsFct(getNElementsFct) - , mNPages(size>0?(size/page_size)+1:0) + , mNPages(size>0?((size-1)/page_size)+1:0) { + static_assert(sizeof(T) == sizeof(BufferType), + "buffer required to be byte-type"); + + // the buffer pointer is stored non-const, a runtime check ensures + // that iterator write works only for non-const buffers + mBuffer = const_cast(buffer); } ~PageParser() = default; - using IteratorBase = std::iterator; + template + using IteratorBase = std::iterator; - class Iterator : public IteratorBase { + template + class Iterator : public IteratorBase { public: using ParentType = PageParser; using SelfType = Iterator; - using value_type = typename IteratorBase::value_type; - using reference = typename IteratorBase::reference; - using pointer = typename IteratorBase::pointer; + using value_type = typename IteratorBase::value_type; + using reference = typename IteratorBase::reference; + using pointer = typename IteratorBase::pointer; + using ElementType = typename std::remove_const::type; + Iterator() = delete; - Iterator(const ParentType & parent, size_t position = 0) + Iterator(ParentType const * parent, size_t position = 0) : mParent(parent) { - mPosition = mParent.getElement(position, mElement); + mPosition = position; + mNextPosition = mParent->getElement(mPosition, mElement); + backup(); + } + ~Iterator() + { + sync(); } - ~Iterator() = default; // prefix increment SelfType& operator++() { - mPosition = mParent.getElement(mPosition, mElement); + sync(); + mPosition = mNextPosition; + mNextPosition = mParent->getElement(mPosition, mElement); + backup(); return *this; } // postfix increment @@ -110,35 +141,99 @@ class PageParser { } private: + // sync method for non-const iterator + template< typename U = void > + typename std::enable_if< !std::is_const::value, U >::type sync() { + if (std::memcmp(&mElement, &mBackup, sizeof(value_type)) != 0) { + // mElement is changed, sync to buffer + mParent->setElement(mPosition, mElement); + } + } + + // overload for const_iterator, empty function body + template< typename U = void > + typename std::enable_if< std::is_const::value, U >::type sync() {} + + // backup for non-const iterator + template< typename U = void > + typename std::enable_if< !std::is_const::value, U >::type backup() { + mBackup = mElement; + } + + // overload for const_iterator, empty function body + template< typename U = void > + typename std::enable_if< std::is_const::value, U >::type backup() {} + int mPosition; - const ParentType& mParent; - value_type mElement; + int mNextPosition; + ParentType const * mParent; + ElementType mElement; + ElementType mBackup; }; + /// set an object at position + size_t setElement(size_t position, const value_type& element) const { + // check if we are at the end + if (position >= mSize) { + assert(position == mSize); + return mSize; + } + + // check if there is space for one element + if (position + sizeof(value_type) > mSize) { + // format error, probably throw exception + return mSize; + } + + auto source = reinterpret_cast(&element); + auto target = mBuffer + position; + return position + copy(source, target, page_size - (position % page_size)); + } + /// retrieve an object at position size_t getElement(size_t position, value_type& element) const { // check if we are at the end - if (position == mSize) return position; + if (position >= mSize) { + assert(position == mSize); + return mSize; + } // check if there is space for one element if (position + sizeof(value_type) > mSize) { // format error, probably throw exception return mSize; } - auto copySize = sizeof(value_type); + + auto source = mBuffer + position; auto target = reinterpret_cast(&element); - if ((position % page_size) == 0) { + return position + copy(source, target, page_size - (position % page_size)); + } + + // copy data, depending on compile time switch, either source or target + // pointer are treated as pointer in the raw page, i.e. can be additionally + // incremented by the page header + template + size_t copy(const BufferType* source, BufferType* target, size_t pageCapacity) const + { + size_t position = 0; + auto copySize = sizeof(value_type); + // choose which of the pointers needs additional PageHeader offsets + auto pageOffsetTarget = SwitchT::value? &target : const_cast(&source); + if (pageCapacity == page_size) { // skip the page header at beginning of page position += sizeof(PageHeaderType); + pageCapacity -= sizeof(PageHeaderType); + *pageOffsetTarget += sizeof(PageHeaderType); } - if ((position % page_size) + copySize > page_size) { + if (copySize > pageCapacity) { // object is split at the page boundary, copy the part // in the current page first - copySize = ((position % page_size) + copySize) - page_size; + copySize = pageCapacity; } if (copySize > 0) { - memcpy(target, mBuffer + position, copySize); + memcpy(target, source, copySize); position += copySize; + source += copySize; target += copySize; } copySize = sizeof(value_type) - copySize; @@ -146,22 +241,41 @@ class PageParser { // skip page header at beginning of new page and copy // remaining part of the element position += sizeof(PageHeaderType); - memcpy(target, mBuffer + position, copySize); + *pageOffsetTarget += sizeof(PageHeaderType); + memcpy(target, source, copySize); position += copySize; } return position; } - Iterator begin() const { - return Iterator(*this, 0); + using iterator = Iterator; + using const_iterator = Iterator; + + const_iterator begin() const { + return const_iterator(this, 0); + } + + const_iterator end() const { + return const_iterator(this, mSize); + } + + iterator begin() { + if (mBufferIsConst) { + // did not find a way to do this at compile time in the constructor, + // probably one needs to make the buffer type a template parameter + // to the class + throw std::runtime_error("the underlying buffer is not writeable"); + } + return iterator(this, 0); } - Iterator end() const { - return Iterator(*this, mSize); + iterator end() { + return iterator(this, mSize); } private: - const BufferType* mBuffer = nullptr; + BufferType* mBuffer = nullptr; + bool mBufferIsConst = false; size_t mSize = 0; GetNElements mGetNElementsFct; size_t mNPages = 0; diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx index f6385ddfc9de4..ac4d5cada8904 100644 --- a/Algorithm/test/pageparser.cxx +++ b/Algorithm/test/pageparser.cxx @@ -132,7 +132,7 @@ BOOST_AUTO_TEST_CASE(test_pageparser) o2::Header::hexDump("pagebuffer", buffer.first.get(), buffer.second); using RawParser = o2::algorithm::PageParser; - RawParser parser(buffer.first.get(), buffer.second); + const RawParser parser(buffer.first.get(), buffer.second); unsigned dataidx = 0; for (auto i : parser) { @@ -146,4 +146,20 @@ BOOST_AUTO_TEST_CASE(test_pageparser) for (auto i : linearizedData) { BOOST_REQUIRE( i == dataset[dataidx++]); } + + dataidx = 0; + RawParser writer(buffer.first.get(), buffer.second); + std::vector> xvalues; + for (auto &i : writer) { + i.x = (dataidx * 3) % 7; + xvalues.emplace_back(i.x, dataidx); + ++dataidx; + } + o2::Header::hexDump("changed buffer", buffer.first.get(), buffer.second); + + dataidx = 0; + for (auto i : parser) { + o2::Header::hexDump("clusterdata", &i, sizeof(ClusterData)); + BOOST_REQUIRE( i.x == xvalues[dataidx++].first); + } }