Allow tokenization w/o empty elements exclusion

shahor02 · shahor02 · commit 4e31171f2690 · 2022-05-28T02:57:56.000+04:00
diff --git a/Common/Utils/include/CommonUtils/StringUtils.h b/Common/Utils/include/CommonUtils/StringUtils.h
@@ -107,7 +107,7 @@ struct Str {
   }
 
   // return vector of tokens from the string with provided delimiter. If requested, trim the spaces from tokens
-  static std::vector<std::string> tokenize(const std::string& src, char delim, bool trimToken = true);
+  static std::vector<std::string> tokenize(const std::string& src, char delim, bool trimToken = true, bool skipEmpty = true);
 
   // concatenate arbitrary number of strings
   template <typename... Ts>
diff --git a/Common/Utils/src/StringUtils.cxx b/Common/Utils/src/StringUtils.cxx
@@ -15,7 +15,7 @@
 
 using namespace o2::utils;
 
-std::vector<std::string> Str::tokenize(const std::string& src, char delim, bool trimToken)
+std::vector<std::string> Str::tokenize(const std::string& src, char delim, bool trimToken, bool skipEmpty)
 {
   std::stringstream ss(src);
   std::string token;
@@ -25,7 +25,7 @@ std::vector<std::string> Str::tokenize(const std::string& src, char delim, bool
     if (trimToken) {
       trim(token);
     }
-    if (!token.empty()) {
+    if (!token.empty() || !skipEmpty) {
       tokens.push_back(std::move(token));
     }
   }

Original file line number	Diff line number	Diff line change
`@@ -107,7 +107,7 @@ struct Str {`
`107`	`107`	`}`
`108`	`108`
`109`	`109`	`// return vector of tokens from the string with provided delimiter. If requested, trim the spaces from tokens`
`110`		`- static std::vector<std::string> tokenize(const std::string& src, char delim, bool trimToken = true);`
	`110`	`+ static std::vector<std::string> tokenize(const std::string& src, char delim, bool trimToken = true, bool skipEmpty = true);`
`111`	`111`
`112`	`112`	`// concatenate arbitrary number of strings`
`113`	`113`	`template <typename... Ts>`
Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@`
`15`	`15`
`16`	`16`	`using namespace o2::utils;`
`17`	`17`
`18`		`-std::vector<std::string> Str::tokenize(const std::string& src, char delim, bool trimToken)`
	`18`	`+std::vector<std::string> Str::tokenize(const std::string& src, char delim, bool trimToken, bool skipEmpty)`
`19`	`19`	`{`
`20`	`20`	`std::stringstream ss(src);`
`21`	`21`	`std::string token;`
`@@ -25,7 +25,7 @@ std::vector<std::string> Str::tokenize(const std::string& src, char delim, bool`
`25`	`25`	`if (trimToken) {`
`26`	`26`	`trim(token);`
`27`	`27`	`}`
`28`		`- if (!token.empty()) {`
	`28`	`+ if (!token.empty() \|\| !skipEmpty) {`
`29`	`29`	`tokens.push_back(std::move(token));`
`30`	`30`	`}`
`31`	`31`	`}`