Adding generic parser for o2 data format

matthiasrichter · sawenzel · commit b8f85e5a58a9 · 2017-10-25T10:12:41.000+02:00
Generic parser for an input list of messages attempting to interpret the O2
header payload sequence.

The following callbacks are mandadory to be provided, e.g. through lambdas
- insert function with signature (const DataHeader&amp;, ptr, size)
  auto insertFct = [&amp;] (const auto &amp; dataheader,
                        auto ptr,
                        auto size) {
    // do something with dataheader and buffer
  };
- getter for the message pointer, e.g. provided std::pair is used
  auto getPointerFct = [] (auto &amp; arg) {return arg.first;};
- getter for the message size, e.g. provided std::pair is used
  auto getSizeFct = [] (auto &amp; arg) {return arg.second;};

Optionally, also callbacks for individual headers in the header stack can be
provided.
diff --git a/Algorithm/CMakeLists.txt b/Algorithm/CMakeLists.txt
@@ -38,6 +38,7 @@ EndForEach (_file RANGE 0 ${_length})
 endif()
 
 set(TEST_SRCS
+  test/o2formatparser.cxx
   test/headerstack.cxx
   test/parser.cxx
   test/tableview.cxx
diff --git a/Algorithm/include/Algorithm/O2FormatParser.h b/Algorithm/include/Algorithm/O2FormatParser.h
@@ -0,0 +1,95 @@
+// Copyright CERN and copyright holders of ALICE O2. This software is
+// distributed under the terms of the GNU General Public License v3 (GPL
+// Version 3), copied verbatim in the file "COPYING".
+//
+// See http://alice-o2.web.cern.ch/license for full licensing information.
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALGORITHM_O2FORMATPARSER_H
+#define ALGORITHM_O2FORMATPARSER_H
+
+/// @file   O2FormatParser.h
+/// @author Matthias Richter
+/// @since  2017-10-18
+/// @brief  Parser for the O2 data format
+
+#include "HeaderStack.h"
+
+namespace o2 {
+
+namespace algorithm {
+
+/**
+ * parse an input list and try to interpret in O2 data format
+ * O2 format consist of header-payload message pairs. The header message
+ * always starts with the DataHeader, optionally there can be more
+ * headers in the header stack.
+ *
+ * The following callbacks are mandadory to be provided, e.g. through lambdas
+ * - insert function with signature (const DataHeader&, ptr, size)
+ *   auto insertFct = [&] (const auto & dataheader,
+ *                         auto ptr,
+ *                         auto size) {
+ *     // do something with dataheader and buffer
+ *   };
+ * - getter for the message pointer, e.g. provided std::pair is used
+ *   auto getPointerFct = [] (const auto & arg) {return arg.first;};
+ * - getter for the message size, e.g. provided std::pair is used
+ *   auto getSizeFct = [] (const auto & arg) {return arg.second;};
+ *
+ * Optionally, also the header stack can be parsed by specifying further
+ * arguments. For every header supposed to be parsed, a pair of a dummy object
+ * and callback has to be specified, e.g.
+ *   // handler callback for MyHeaderStruct
+ *   auto onMyHeaderStruct = [&] (const auto & mystruct) {
+ *     // do something with mystruct
+ *   }; // end handler callback
+ *
+ *   parseO2Format(list, insertFct, MyHeaderStruct(), onMyHeaderStruct);
+ *
+ */
+template<
+  typename InputListT
+  , typename GetPointerFctT
+  , typename GetSizeFctT
+  , typename InsertFctT // (const auto&, ptr, size)
+  , typename... HeaderStackTypes // pairs of HeaderType and CallbackType
+  >
+int parseO2Format(const InputListT& list,
+                  GetPointerFctT getPointer,
+                  GetSizeFctT getSize,
+                  InsertFctT insert,
+                  HeaderStackTypes&&... stackArgs
+                  )
+{
+  const o2::Header::DataHeader* dh = nullptr;
+  for (auto & part : list) {
+    if (!dh) {
+      // new header - payload pair, read DataHeader
+      dh = o2::Header::get<o2::Header::DataHeader>(getPointer(part), getSize(part));
+      if (!dh) {
+        return -ENOMSG;
+      }
+      o2::algorithm::dispatchHeaderStackCallback(getPointer(part),
+                                                 getSize(part),
+                                                 stackArgs...
+                                                 );
+    } else {
+      insert(*dh, getPointer(part), getSize(part));
+      dh = nullptr;
+    }
+  }
+  if (dh) {
+    return -ENOMSG;
+  }
+  return list.size()/2;
+}
+
+} // namespace algorithm
+
+} // namespace o2
+
+#endif // ALGORITHM_O2FORMATPARSER_H
diff --git a/Algorithm/test/o2formatparser.cxx b/Algorithm/test/o2formatparser.cxx
@@ -0,0 +1,81 @@
+// Copyright CERN and copyright holders of ALICE O2. This software is
+// distributed under the terms of the GNU General Public License v3 (GPL
+// Version 3), copied verbatim in the file "COPYING".
+//
+// See http://alice-o2.web.cern.ch/license for full licensing information.
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// @file   o2formatparser.cxx
+/// @author Matthias Richter
+/// @since  2017-10-18
+/// @brief  Unit test for O2 format parser
+
+#define BOOST_TEST_MODULE Test Algorithm HeaderStack
+#define BOOST_TEST_MAIN
+#define BOOST_TEST_DYN_LINK
+#include <boost/test/unit_test.hpp>
+#include <iostream>
+#include <iomanip>
+#include <cstring> // memcmp
+#include "Headers/DataHeader.h" // hexdump, DataHeader
+#include "../include/Algorithm/O2FormatParser.h"
+
+template<typename... Targs>
+void hexDump(Targs... Fargs) {
+  // a simple redirect to enable/disable the hexdump printout
+  o2::Header::hexDump(Fargs...);
+}
+
+BOOST_AUTO_TEST_CASE(test_o2formatparser)
+{
+  std::vector<const char*> thedata = {
+    "I'm raw data",
+    "reconstructed data"
+  };
+  unsigned dataidx = 0;
+  std::vector<o2::Header::DataHeader> dataheaders;
+  dataheaders.emplace_back(o2::Header::DataDescription("RAWDATA"),
+                           o2::Header::DataOrigin("DET"),
+                           0,
+                           strlen(thedata[dataidx++]));
+  dataheaders.emplace_back(o2::Header::DataDescription("RECODATA"),
+                           o2::Header::DataOrigin("DET"),
+                           0,
+                           strlen(thedata[dataidx++]));
+
+  std::vector<std::pair<const char*, size_t>> messages;
+  for (dataidx = 0; dataidx < thedata.size(); ++dataidx) {
+    messages.emplace_back(reinterpret_cast<char*>(&dataheaders[dataidx]),
+                          sizeof(o2::Header::DataHeader));
+    messages.emplace_back(thedata[dataidx],
+                          dataheaders[dataidx].payloadSize);
+  }
+
+  // handler callback for parseO2Format method
+  auto insertFct = [&] (const auto & dataheader,
+                        auto ptr,
+                        auto size) {
+    hexDump("header", &dataheader, sizeof(dataheader));
+    hexDump("data", ptr, size);
+    BOOST_CHECK(dataheader == dataheaders[dataidx]);
+    BOOST_CHECK(strncmp(ptr, thedata[dataidx], size) == 0);
+    ++dataidx;
+  }; // end handler callback
+
+  // handler callback to get the pointer for message
+  auto getPointerFct = [] (auto arg) {return arg.first;};
+  // handler callback to get the size for message
+  auto getSizeFct = [] (auto arg) {return arg.second;};
+
+  dataidx = 0;
+  auto result = o2::algorithm::parseO2Format(messages,
+                                             getPointerFct,
+                                             getSizeFct,
+                                             insertFct);
+
+  BOOST_REQUIRE(result >= 0);
+  BOOST_CHECK(result == 2);
+}
