Skip to content

Commit d4d6070

Browse files
ktfktf
committed
DPL driver: increase input parsing performance
Remove a bit of technical debt when processing the inputs from the children dataprocessors. In particular: * Most of parsing / processing now uses a string_view so that the unparsed buffers are not copied around multiple times. * Read buffer is now 16kB (was 1kB) which seems to avoid having to split the processing in multiple iterations. * Add tracing information for data being read from children and for data processed by the driver.
1 parent b2a3037 commit d4d6070

9 files changed

Lines changed: 51 additions & 27 deletions

‎Framework/Core/include/Framework/DeviceMetricsInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include <cstddef>
1616
#include <functional>
1717
#include <string>
18+
#include <string_view>
1819
#include <vector>
1920

2021
namespace o2
@@ -87,7 +88,7 @@ struct DeviceMetricsHelper {
8788
using NewMetricCallback = std::function<void(std::string const&, MetricInfo const&, int value, size_t metricIndex)>;
8889

8990
/// Helper function to parse a metric string.
90-
static bool parseMetric(const std::string& s, ParsedMetricMatch& results);
91+
static bool parseMetric(std::string_view const s, ParsedMetricMatch& results);
9192

9293
/// Processes a parsed metric and stores in the backend store.
9394
///

‎Framework/Core/include/Framework/LogParsingHelpers.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#define FRAMEWORK_LOGPARSINGHELPERS
1212

1313
#include <string>
14+
#include <string_view>
1415

1516
namespace o2
1617
{
@@ -36,7 +37,7 @@ struct LogParsingHelpers {
3637
/// Token style can then be used for colouring the logs
3738
/// in the GUI or to exit with error if a sufficient
3839
/// number of LogLevel::Error is found.
39-
static LogLevel parseTokenLevel(const std::string &s);
40+
static LogLevel parseTokenLevel(std::string_view const s);
4041
};
4142

4243
} // namespace framework

‎Framework/Core/include/Framework/TextControlService.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
#include "Framework/ControlService.h"
1414
#include <string>
15+
#include <string_view>
1516
#include <regex>
1617

1718
namespace o2
@@ -41,7 +42,7 @@ class TextControlService : public ControlService {
4142
bool mOnce = false;
4243
};
4344

44-
bool parseControl(const std::string &s, std::smatch &match);
45+
bool parseControl(std::string_view s, std::smatch& match);
4546

4647
} // namespace framework
4748
} // namespace o2

‎Framework/Core/src/DataProcessingStatus.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,13 @@ enum struct MonitoringStatus : uint32_t {
3333
FLUSH = 1,
3434
};
3535

36+
enum struct DriverStatus : uint32_t {
37+
ID = 2,
38+
BYTES_READ = 0,
39+
BYTES_PROCESSED = 1,
40+
BUFFER_OVERFLOWS = 2
41+
};
42+
3643
} // namespace framework
3744
} // namespace o2
3845

‎Framework/Core/src/DeviceMetricsInfo.cxx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
#include <algorithm>
1717
#include <regex>
18+
#include <string_view>
1819
#include <tuple>
1920
#include <iostream>
2021

@@ -26,7 +27,7 @@ namespace framework
2627
// Parses a metric in the form
2728
//
2829
// [METRIC] <name>,<type> <value> <timestamp> [<tag>,<tag>]
29-
bool DeviceMetricsHelper::parseMetric(const std::string& s, ParsedMetricMatch& match)
30+
bool DeviceMetricsHelper::parseMetric(std::string_view const s, ParsedMetricMatch& match)
3031
{
3132
/// Must start with "[METRIC] "
3233
/// 012345678

‎Framework/Core/src/LogParsingHelpers.cxx‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,16 @@ namespace framework
1616
{
1717

1818
char const* const LogParsingHelpers::LOG_LEVELS[(int)LogParsingHelpers::LogLevel::Size] = {
19-
"DEBUG",
20-
"INFO",
21-
"WARNING",
22-
"ERROR",
23-
"UNKNOWN"
24-
};
19+
"DEBUG",
20+
"INFO",
21+
"WARNING",
22+
"ERROR",
23+
"UNKNOWN"
24+
};
2525
using LogLevel = o2::framework::LogParsingHelpers::LogLevel;
2626

27-
LogLevel LogParsingHelpers::parseTokenLevel(const std::string &s) {
27+
LogLevel LogParsingHelpers::parseTokenLevel(std::string_view const s)
28+
{
2829

2930
// Example format: [99:99:99][ERROR] (string begins with that, longest is 17 chars)
3031
constexpr size_t MAXPREFLEN = 17;
@@ -55,6 +56,5 @@ LogLevel LogParsingHelpers::parseTokenLevel(const std::string &s) {
5556
}
5657
return LogLevel::Unknown;
5758
}
58-
59-
}
60-
}
59+
} // namespace framework
60+
} // namespace o2

‎Framework/Core/src/TextControlService.cxx‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "Framework/TextControlService.h"
1111
#include "FairMQLogger.h"
1212
#include <string>
13+
#include <string_view>
1314
#include <regex>
1415
#include <iostream>
1516

@@ -31,13 +32,16 @@ void TextControlService::readyToQuit(bool all) {
3132
}
3233
}
3334

34-
bool parseControl(const std::string &s, std::smatch &match) {
35+
bool parseControl(std::string_view s, std::smatch& match)
36+
{
3537
const static std::regex controlRE("READY_TO_(QUIT)_(ME|ALL)", std::regex::optimize);
3638
auto idx = s.find("CONTROL_ACTION: ");
3739
if (idx == std::string::npos) {
3840
return false;
3941
}
40-
return std::regex_search(s.begin() + idx, s.end(), match, controlRE);
42+
s.remove_prefix(idx);
43+
std::string rs{ s };
44+
return std::regex_search(rs, match, controlRE);
4145
}
4246

4347
} // framework

‎Framework/Core/src/runDataProcessing.cxx‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,12 @@
2828
#include "Framework/ParallelContext.h"
2929
#include "Framework/RawDeviceService.h"
3030
#include "Framework/SimpleRawDeviceService.h"
31+
#include "Framework/Signpost.h"
3132
#include "Framework/TextControlService.h"
3233
#include "Framework/CallbackService.h"
3334
#include "Framework/WorkflowSpec.h"
3435

36+
#include "DataProcessingStatus.h"
3537
#include "DDSConfigHelpers.h"
3638
#include "O2ControlHelpers.h"
3739
#include "DeviceSpecHelpers.h"
@@ -131,12 +133,13 @@ std::ostream& operator<<(std::ostream& out, const enum TerminationPolicy& policy
131133
// FIXME: We should really print full lines.
132134
bool getChildData(int infd, DeviceInfo& outinfo)
133135
{
134-
char buffer[1024];
136+
char buffer[1024 * 16];
135137
int bytes_read;
136138
// NOTE: do not quite understand read ends up blocking if I read more than
137139
// once. Oh well... Good enough for now.
140+
O2_SIGNPOST_START(DriverStatus::ID, DriverStatus::BYTES_READ, outinfo.pid, infd, 0);
138141
// do {
139-
bytes_read = read(infd, buffer, 1024);
142+
bytes_read = read(infd, buffer, 1024 * 16);
140143
if (bytes_read == 0) {
141144
return false;
142145
}
@@ -150,7 +153,8 @@ bool getChildData(int infd, DeviceInfo& outinfo)
150153
}
151154
assert(bytes_read > 0);
152155
outinfo.unprinted += std::string(buffer, bytes_read);
153-
// } while (bytes_read != 0);
156+
// } while (bytes_read != 0);
157+
O2_SIGNPOST_END(DriverStatus::ID, DriverStatus::BYTES_READ, bytes_read, 0, 0);
154158
return true;
155159
}
156160

@@ -372,7 +376,6 @@ void processChildrenOutput(DriverInfo& driverInfo, DeviceInfos& infos, DeviceSpe
372376
assert(infos.size() == controls.size());
373377
std::smatch match;
374378
ParsedMetricMatch metricMatch;
375-
std::string token;
376379
const std::string delimiter("\n");
377380
bool hasNewMetric = false;
378381
for (size_t di = 0, de = infos.size(); di < de; ++di) {
@@ -384,7 +387,9 @@ void processChildrenOutput(DriverInfo& driverInfo, DeviceInfos& infos, DeviceSpe
384387
continue;
385388
}
386389

387-
auto s = info.unprinted;
390+
O2_SIGNPOST_START(DriverStatus::ID, DriverStatus::BYTES_PROCESSED, info.pid, 0, 0);
391+
392+
std::string_view s = info.unprinted;
388393
size_t pos = 0;
389394
info.history.resize(info.historySize);
390395
info.historyLevel.resize(info.historySize);
@@ -400,7 +405,7 @@ void processChildrenOutput(DriverInfo& driverInfo, DeviceInfos& infos, DeviceSpe
400405
};
401406

402407
while ((pos = s.find(delimiter)) != std::string::npos) {
403-
token = s.substr(0, pos);
408+
std::string token{ s.substr(0, pos) };
404409
auto logLevel = LogParsingHelpers::parseTokenLevel(token);
405410

406411
// Check if the token is a metric from SimpleMetricsService
@@ -431,7 +436,7 @@ void processChildrenOutput(DriverInfo& driverInfo, DeviceInfos& infos, DeviceSpe
431436
}
432437
}
433438
}
434-
} else if (!control.quiet && (strstr(token.c_str(), control.logFilter) != nullptr) &&
439+
} else if (!control.quiet && (token.find(control.logFilter) != std::string::npos) &&
435440
logLevel >= control.logLevel) {
436441
assert(info.historyPos >= 0);
437442
assert(info.historyPos < info.history.size());
@@ -449,9 +454,11 @@ void processChildrenOutput(DriverInfo& driverInfo, DeviceInfos& infos, DeviceSpe
449454
if (logLevel == LogParsingHelpers::LogLevel::Error) {
450455
info.lastError = token;
451456
}
452-
s.erase(0, pos + delimiter.length());
457+
s.remove_prefix(pos + delimiter.length());
453458
}
459+
size_t oldSize = info.unprinted.size();
454460
info.unprinted = s;
461+
O2_SIGNPOST_END(DriverStatus::ID, DriverStatus::BYTES_PROCESSED, oldSize - info.unprinted.size(), 0, 0);
455462
}
456463
if (hasNewMetric) {
457464
hasNewMetric = false;

‎Framework/Core/test/test_DeviceMetricsInfo.cxx‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,19 @@
1515
#include <boost/test/unit_test.hpp>
1616
#include <iostream>
1717
#include <regex>
18-
18+
#include <string_view>
1919

2020
BOOST_AUTO_TEST_CASE(TestDeviceMetricsInfo) {
2121
using namespace o2::framework;
22-
std::string metric;
22+
std::string metricString;
2323
ParsedMetricMatch match;
2424
bool result;
2525
DeviceMetricsInfo info;
2626

2727
// Parse a simple metric
28-
metric = "[METRIC] bkey,0 12 1789372894 hostname=test.cern.ch";
28+
metricString = "foo[METRIC] bkey,0 12 1789372894 hostname=test.cern.chbar";
29+
std::string_view metric{ metricString.data() + 3, metricString.size() - 6 };
30+
BOOST_REQUIRE_EQUAL(metric, std::string("[METRIC] bkey,0 12 1789372894 hostname=test.cern.ch"));
2931
result = DeviceMetricsHelper::parseMetric(metric, match);
3032
BOOST_REQUIRE_EQUAL(result, true);
3133
BOOST_CHECK(strncmp(match.beginKey, "bkey", 4) == 0);

0 commit comments

Comments
 (0)