From 9bb9ac8563e4afa50f9c9cb07e0827d24dfcc278 Mon Sep 17 00:00:00 2001 From: Matthias Richter Date: Wed, 27 Sep 2017 23:28:41 +0200 Subject: [PATCH] Adding parser for a set of data objects in consecutive memory pages Memory pages have a fixed size and start with a page header. The parser provides transparent iteration through data objects and handles page boundaries. --- Algorithm/CMakeLists.txt | 1 + Algorithm/include/Algorithm/PageParser.h | 173 +++++++++++++++++++++++ Algorithm/test/pageparser.cxx | 149 +++++++++++++++++++ 3 files changed, 323 insertions(+) create mode 100644 Algorithm/include/Algorithm/PageParser.h create mode 100644 Algorithm/test/pageparser.cxx diff --git a/Algorithm/CMakeLists.txt b/Algorithm/CMakeLists.txt index 14aeb5c47c4ba..65361dff57df4 100644 --- a/Algorithm/CMakeLists.txt +++ b/Algorithm/CMakeLists.txt @@ -41,6 +41,7 @@ set(TEST_SRCS test/headerstack.cxx test/parser.cxx test/tableview.cxx + test/pageparser.cxx ) O2_GENERATE_TESTS( diff --git a/Algorithm/include/Algorithm/PageParser.h b/Algorithm/include/Algorithm/PageParser.h new file mode 100644 index 0000000000000..314f03faf9699 --- /dev/null +++ b/Algorithm/include/Algorithm/PageParser.h @@ -0,0 +1,173 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef ALGORITHM_PAGEPARSER_H +#define ALGORITHM_PAGEPARSER_H + +/// @file PageParser.h +/// @author Matthias Richter +/// @since 2017-09-27 +/// @brief Parser for a set of data objects in consecutive memory pages. + +#include +#include + +namespace o2 { + +namespace algorithm { + +/** + * @class PageParser + * Parser for a set of data objects in consecutive memory pages. + * + * All memory pages have a fixed size and start with a page header. + * Depending on the page size and size of the data object, some + * objects can be split at the page boundary and have the page header + * embedded. + * + * The class iterator can be used to iterate over the data objects + * transparently. + * + * Usage: + * RawParser RawParser; + * RawParser parser(ptr, size); + * for (auto element : parser) { + * // do something with element + * } + */ +template +class PageParser { +public: + using PageHeaderType = PageHeaderT; + using BufferType = unsigned char; + using value_type = ElementType; + using GroupType = GroupT; + using GetNElements = std::function; + static const size_t page_size = PageSize; + + PageParser() = delete; + PageParser(const BufferType* buffer, size_t size, + GetNElements getNElementsFct = [] (const GroupType&) {return 0;} + ) + : mBuffer(buffer) + , mSize(size) + , mGetNElementsFct(getNElementsFct) + , mNPages(size>0?(size/page_size)+1:0) + { + } + ~PageParser() = default; + + using IteratorBase = std::iterator; + + class Iterator : public IteratorBase { + public: + using ParentType = PageParser; + using SelfType = Iterator; + using value_type = typename IteratorBase::value_type; + using reference = typename IteratorBase::reference; + using pointer = typename IteratorBase::pointer; + + Iterator() = delete; + + Iterator(const ParentType & parent, size_t position = 0) + : mParent(parent) + { + mPosition = mParent.getElement(position, mElement); + } + ~Iterator() = default; + + // prefix increment + SelfType& operator++() { + mPosition = mParent.getElement(mPosition, mElement); + return *this; + } + // postfix increment + SelfType operator++(int /*unused*/) { + SelfType copy(*this); operator++(); return copy; + } + // return reference + reference operator*() { + return mElement; + } + // comparison + bool operator==(const SelfType& rh) { + return mPosition == rh.mPosition; + } + // comparison + bool operator!=(const SelfType& rh) { + return mPosition != rh.mPosition; + } + + private: + int mPosition; + const ParentType& mParent; + value_type mElement; + }; + + /// retrieve an object at position + size_t getElement(size_t position, value_type& element) const { + // check if we are at the end + if (position == mSize) return position; + + // check if there is space for one element + if (position + sizeof(value_type) > mSize) { + // format error, probably throw exception + return mSize; + } + auto copySize = sizeof(value_type); + auto target = reinterpret_cast(&element); + if ((position % page_size) == 0) { + // skip the page header at beginning of page + position += sizeof(PageHeaderType); + } + if ((position % page_size) + copySize > page_size) { + // object is split at the page boundary, copy the part + // in the current page first + copySize = ((position % page_size) + copySize) - page_size; + } + if (copySize > 0) { + memcpy(target, mBuffer + position, copySize); + position += copySize; + target += copySize; + } + copySize = sizeof(value_type) - copySize; + if (copySize > 0) { + // skip page header at beginning of new page and copy + // remaining part of the element + position += sizeof(PageHeaderType); + memcpy(target, mBuffer + position, copySize); + position += copySize; + } + return position; + } + + Iterator begin() const { + return Iterator(*this, 0); + } + + Iterator end() const { + return Iterator(*this, mSize); + } + +private: + const BufferType* mBuffer = nullptr; + size_t mSize = 0; + GetNElements mGetNElementsFct; + size_t mNPages = 0; +}; + +} +} + +#endif diff --git a/Algorithm/test/pageparser.cxx b/Algorithm/test/pageparser.cxx new file mode 100644 index 0000000000000..f6385ddfc9de4 --- /dev/null +++ b/Algorithm/test/pageparser.cxx @@ -0,0 +1,149 @@ +// Copyright CERN and copyright holders of ALICE O2. This software is +// distributed under the terms of the GNU General Public License v3 (GPL +// Version 3), copied verbatim in the file "COPYING". +// +// See http://alice-o2.web.cern.ch/license for full licensing information. +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file parser.cxx +/// @author Matthias Richter +/// @since 2017-09-27 +/// @brief Unit test for parser of objects in memory pages + +#define BOOST_TEST_MODULE Test Algorithm Parser +#define BOOST_TEST_MAIN +#define BOOST_TEST_DYN_LINK +#include +#include +#include +#include +#include "Headers/DataHeader.h" // hexdump +#include "../include/Algorithm/PageParser.h" +#include "StaticSequenceAllocator.h" + +struct PageHeader { + uint32_t magic = 0x45474150; + uint32_t pageid; + + PageHeader(uint32_t id) : pageid(id) {} +}; + +struct ClusterData { + uint32_t magic = 0x54534c43; + uint32_t clusterid; + uint16_t x; + uint16_t y; + uint16_t z; + uint8_t e; + + ClusterData() + : clusterid(0) + , x(0) + , y(0) + , z(0) + , e(0) + {} + + ClusterData(uint32_t _id, uint16_t _x, uint16_t _y, uint16_t _z, uint8_t _e) + : clusterid(_id) + , x(_x) + , y(_y) + , z(_z) + , e(_e) + {} + + bool operator==(const ClusterData& rhs) { + return clusterid == rhs.clusterid + && x == rhs.x + && y == rhs.y + && z == rhs.z + && e == rhs.e; + } +}; + +template +std::pair, size_t> MakeBuffer(unsigned pagesize, + PageHeaderT pageheader, + ListT& dataset) +{ + auto totalSize = dataset.size() * sizeof(typename ListT::value_type); + unsigned nPages = 0; + do { + totalSize += sizeof(PageHeaderT); + ++nPages; + } while (nPages * pagesize < totalSize); + + std::unique_ptr buffer(new uint8_t[totalSize]); + + unsigned position = 0; + auto target = buffer.get(); + for (auto element : dataset) { + auto source = reinterpret_cast(&element); + auto copySize = sizeof(typename ListT::value_type); + if ((position % pagesize) == 0) { + memcpy(target, &pageheader, sizeof(PageHeaderT)); + position += sizeof(PageHeaderT); + target += sizeof(PageHeaderT); + } + if ((position % pagesize) + copySize > pagesize) { + copySize = ((position % pagesize) + copySize) - pagesize; + } + if (copySize > 0) { + memcpy(target, source, copySize); + position += copySize; + target += copySize; + source += copySize; + } + copySize = sizeof(typename ListT::value_type) - copySize; + if (copySize > 0) { + memcpy(target, &pageheader, sizeof(PageHeaderT)); + position += sizeof(PageHeaderT); + target += sizeof(PageHeaderT); + memcpy(target, source, copySize); + } + position += copySize; + target += copySize; + } + + std::pair, size_t> result; + result.first = std::move(buffer); + result.second = totalSize; + return result; +} + +template +void FillData(ListT& dataset, unsigned entries) +{ + for (unsigned i = 0; i < entries; i++) { + dataset.emplace_back(i, 0xaa, 0xbb, 0xcc, 0xd); + } +} + +BOOST_AUTO_TEST_CASE(test_pageparser) +{ + constexpr unsigned pagesize = 128; + std::vector dataset; + FillData(dataset, 20); + auto buffer = MakeBuffer(pagesize, PageHeader(0), dataset); + o2::Header::hexDump("pagebuffer", buffer.first.get(), buffer.second); + + using RawParser = o2::algorithm::PageParser; + RawParser parser(buffer.first.get(), buffer.second); + + unsigned dataidx = 0; + for (auto i : parser) { + o2::Header::hexDump("clusterdata", &i, sizeof(ClusterData)); + BOOST_REQUIRE( i == dataset[dataidx++]); + } + + std::vector linearizedData; + linearizedData.insert(linearizedData.begin(), parser.begin(), parser.end()); + dataidx = 0; + for (auto i : linearizedData) { + BOOST_REQUIRE( i == dataset[dataidx++]); + } +}