Skip to content

Commit b71fd81

Browse files
ktfktf
authored
DPL: add metadata to InputSpec (#4598)
This will be useful for things like enumerations and CCDB access.
1 parent a740637 commit b71fd81

5 files changed

Lines changed: 113 additions & 36 deletions

File tree

‎Framework/Core/include/Framework/InputSpec.h‎

Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,38 +13,60 @@
1313
#include "Framework/Lifetime.h"
1414
#include "Framework/ConcreteDataMatcher.h"
1515
#include "Framework/DataDescriptorMatcher.h"
16+
#include "Framework/ConfigParamSpec.h"
1617

1718
#include <string>
1819
#include <ostream>
1920
#include <variant>
2021

21-
namespace o2
22-
{
23-
namespace framework
22+
namespace o2::framework
2423
{
2524

2625
/// A selector for some kind of data being processed, either in
2726
/// input or in output. This can be used, for example to match
2827
/// specific payloads in a timeframe.
2928
struct InputSpec {
3029
/// Create a fully qualified InputSpec
31-
InputSpec(std::string binding_, header::DataOrigin origin_, header::DataDescription description_, header::DataHeader::SubSpecificationType subSpec_, enum Lifetime lifetime_ = Lifetime::Timeframe);
30+
InputSpec(std::string binding_,
31+
header::DataOrigin origin_,
32+
header::DataDescription description_,
33+
header::DataHeader::SubSpecificationType subSpec_,
34+
enum Lifetime lifetime_ = Lifetime::Timeframe,
35+
std::vector<ConfigParamSpec> const& metadata_ = {});
3236
/// Create a fully qualified InputSpec (alternative syntax)
33-
InputSpec(std::string binding_, ConcreteDataMatcher const& dataType, enum Lifetime lifetime_ = Lifetime::Timeframe);
37+
InputSpec(std::string binding_,
38+
ConcreteDataMatcher const& dataType,
39+
enum Lifetime lifetime_ = Lifetime::Timeframe,
40+
std::vector<ConfigParamSpec> const& metadata_ = {});
3441
/// Create a fully qualified InputSpec where the subSpec is 0
35-
InputSpec(std::string binding_, header::DataOrigin origin_, header::DataDescription description_, enum Lifetime lifetime_ = Lifetime::Timeframe);
42+
InputSpec(std::string binding_,
43+
header::DataOrigin origin_,
44+
header::DataDescription description_,
45+
enum Lifetime lifetime_ = Lifetime::Timeframe,
46+
std::vector<ConfigParamSpec> const& metadata_ = {});
3647
/// Create an InputSpec which does not check for the subSpec.
37-
InputSpec(std::string binding_, ConcreteDataTypeMatcher const& dataType, enum Lifetime lifetime_ = Lifetime::Timeframe);
38-
InputSpec(std::string binding, data_matcher::DataDescriptorMatcher&& matcher);
48+
InputSpec(std::string binding_,
49+
ConcreteDataTypeMatcher const& dataType,
50+
enum Lifetime lifetime_ = Lifetime::Timeframe,
51+
std::vector<ConfigParamSpec> const& metadata_ = {});
52+
InputSpec(std::string binding,
53+
data_matcher::DataDescriptorMatcher&& matcher,
54+
std::vector<ConfigParamSpec> const& metadata_ = {});
3955

56+
/// A mnemonic name for the input spec.
4057
std::string binding;
58+
59+
/// The actual matcher for the input spec.
4160
std::variant<ConcreteDataMatcher, data_matcher::DataDescriptorMatcher> matcher;
61+
4262
enum Lifetime lifetime;
4363

64+
/// A set of configurables which can be used to customise the InputSpec.
65+
std::vector<ConfigParamSpec> metadata;
66+
4467
friend std::ostream& operator<<(std::ostream& stream, InputSpec const& arg);
4568
bool operator==(InputSpec const& that) const;
4669
};
4770

48-
} // namespace framework
4971
} // namespace o2
5072
#endif

‎Framework/Core/src/InputSpec.cxx‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,44 +12,67 @@
1212
#include "Framework/DataSpecUtils.h"
1313

1414
#include <variant>
15+
#include <vector>
1516

1617
namespace o2
1718
{
1819
namespace framework
1920
{
2021

21-
InputSpec::InputSpec(std::string binding_, ConcreteDataMatcher const& concrete, enum Lifetime lifetime_)
22+
InputSpec::InputSpec(std::string binding_,
23+
ConcreteDataMatcher const& concrete,
24+
enum Lifetime lifetime_,
25+
std::vector<ConfigParamSpec> const& metadata_)
2226
: binding{binding_},
2327
matcher{concrete},
24-
lifetime{lifetime_}
28+
lifetime{lifetime_},
29+
metadata{metadata_}
2530
{
2631
}
2732

28-
InputSpec::InputSpec(std::string binding_, header::DataOrigin origin_, header::DataDescription description_, header::DataHeader::SubSpecificationType subSpec_, enum Lifetime lifetime_)
33+
InputSpec::InputSpec(std::string binding_,
34+
header::DataOrigin origin_,
35+
header::DataDescription description_,
36+
header::DataHeader::SubSpecificationType subSpec_,
37+
enum Lifetime lifetime_,
38+
std::vector<ConfigParamSpec> const& metadata_)
2939
: binding{binding_},
3040
matcher{ConcreteDataMatcher{origin_, description_, subSpec_}},
31-
lifetime{lifetime_}
41+
lifetime{lifetime_},
42+
metadata{metadata_}
3243
{
3344
}
3445

35-
InputSpec::InputSpec(std::string binding_, header::DataOrigin origin_, header::DataDescription description_, enum Lifetime lifetime_)
46+
InputSpec::InputSpec(std::string binding_,
47+
header::DataOrigin origin_,
48+
header::DataDescription description_,
49+
enum Lifetime lifetime_,
50+
std::vector<ConfigParamSpec> const& metadata_)
3651
: binding{binding_},
3752
matcher{ConcreteDataMatcher{origin_, description_, 0}},
38-
lifetime{lifetime_}
53+
lifetime{lifetime_},
54+
metadata{metadata_}
3955
{
4056
}
4157

42-
InputSpec::InputSpec(std::string binding_, ConcreteDataTypeMatcher const& dataType, enum Lifetime lifetime_)
58+
InputSpec::InputSpec(std::string binding_,
59+
ConcreteDataTypeMatcher const& dataType,
60+
enum Lifetime lifetime_,
61+
std::vector<ConfigParamSpec> const& metadata_)
4362
: binding{binding_},
4463
matcher{DataSpecUtils::dataDescriptorMatcherFrom(dataType)},
45-
lifetime{lifetime_}
64+
lifetime{lifetime_},
65+
metadata{metadata_}
4666
{
4767
}
4868

49-
InputSpec::InputSpec(std::string binding_, data_matcher::DataDescriptorMatcher&& matcher_)
69+
InputSpec::InputSpec(std::string binding_,
70+
data_matcher::DataDescriptorMatcher&& matcher_,
71+
std::vector<ConfigParamSpec> const& metadata_)
5072
: binding{binding_},
5173
matcher{matcher_},
52-
lifetime{Lifetime::Timeframe}
74+
lifetime{Lifetime::Timeframe},
75+
metadata{metadata_}
5376
{
5477
}
5578

‎Framework/Core/src/WorkflowSerializationHelpers.cxx‎

Lines changed: 47 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
5151
IN_INPUT_DESCRIPTION,
5252
IN_INPUT_SUBSPEC,
5353
IN_INPUT_LIFETIME,
54+
IN_INPUT_OPTIONS,
5455
IN_OUTPUT,
5556
IN_OUTPUT_BINDING,
5657
IN_OUTPUT_ORIGIN,
@@ -135,6 +136,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
135136
case State::IN_INPUT_LIFETIME:
136137
s << "IN_INPUT_LIFETIME";
137138
break;
139+
case State::IN_INPUT_OPTIONS:
140+
s << "IN_INPUT_OPTIONS";
141+
break;
138142
case State::IN_OUTPUT:
139143
s << "IN_OUTPUT";
140144
break;
@@ -202,7 +206,7 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
202206
WorkflowImporter(std::vector<DataProcessorSpec>& o,
203207
std::vector<DataProcessorInfo>& m)
204208
: states{},
205-
output{o},
209+
dataProcessors{o},
206210
metadata{m}
207211
{
208212
push(State::IN_START);
@@ -215,9 +219,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
215219
push(State::IN_EXECUTION);
216220
} else if (in(State::IN_DATAPROCESSORS)) {
217221
push(State::IN_DATAPROCESSOR);
218-
output.push_back(DataProcessorSpec{});
222+
dataProcessors.push_back(DataProcessorSpec{});
219223
} else if (in(State::IN_DATAPROCESSOR)) {
220-
output.push_back(DataProcessorSpec{});
224+
dataProcessors.push_back(DataProcessorSpec{});
221225
} else if (in(State::IN_INPUTS)) {
222226
push(State::IN_INPUT);
223227
inputHasSubSpec = false;
@@ -242,16 +246,17 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
242246
enter("END_OBJECT");
243247
if (in(State::IN_INPUT)) {
244248
if (inputHasSubSpec) {
245-
output.back().inputs.push_back(InputSpec(binding, origin, description, subspec, lifetime));
249+
dataProcessors.back().inputs.push_back(InputSpec(binding, origin, description, subspec, lifetime, inputOptions));
246250
} else {
247-
output.back().inputs.push_back(InputSpec(binding, {origin, description}, lifetime));
251+
dataProcessors.back().inputs.push_back(InputSpec(binding, {origin, description}, lifetime, inputOptions));
248252
}
253+
inputOptions.clear();
249254
inputHasSubSpec = false;
250255
} else if (in(State::IN_OUTPUT)) {
251256
if (outputHasSubSpec) {
252-
output.back().outputs.push_back(OutputSpec({binding}, origin, description, subspec, lifetime));
257+
dataProcessors.back().outputs.push_back(OutputSpec({binding}, origin, description, subspec, lifetime));
253258
} else {
254-
output.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
259+
dataProcessors.back().outputs.push_back(OutputSpec({binding}, {origin, description}, lifetime));
255260
}
256261
outputHasSubSpec = false;
257262
} else if (in(State::IN_OPTION)) {
@@ -282,9 +287,11 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
282287
}
283288
// Depending on the previous state, push options to the right place.
284289
if (previousIs(State::IN_OPTIONS)) {
285-
output.back().options.push_back(*opt);
290+
dataProcessors.back().options.push_back(*opt);
286291
} else if (previousIs(State::IN_WORKFLOW_OPTIONS)) {
287292
metadata.back().workflowOptions.push_back(*opt);
293+
} else if (previousIs(State::IN_INPUT_OPTIONS)) {
294+
inputOptions.push_back(*opt);
288295
} else {
289296
assert(false);
290297
}
@@ -301,6 +308,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
301308
} else if (in(State::IN_INPUTS)) {
302309
push(State::IN_INPUT);
303310
inputHasSubSpec = false;
311+
} else if (in(State::IN_INPUT_OPTIONS)) {
312+
push(State::IN_OPTION);
304313
} else if (in(State::IN_OUTPUTS)) {
305314
push(State::IN_OUTPUT);
306315
outputHasSubSpec = false;
@@ -345,6 +354,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
345354
inputHasSubSpec = true;
346355
} else if (in(State::IN_INPUT) && strncmp(str, "lifetime", length) == 0) {
347356
push(State::IN_INPUT_LIFETIME);
357+
} else if (in(State::IN_INPUT) && strncmp(str, "metadata", length) == 0) {
358+
push(State::IN_INPUT_OPTIONS);
348359
} else if (in(State::IN_OUTPUT) && strncmp(str, "binding", length) == 0) {
349360
push(State::IN_OUTPUT_BINDING);
350361
} else if (in(State::IN_OUTPUT) && strncmp(str, "origin", length) == 0) {
@@ -404,8 +415,8 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
404415
enter(str);
405416
auto s = std::string(str, length);
406417
if (in(State::IN_DATAPROCESSOR_NAME)) {
407-
assert(output.size());
408-
output.back().name = s;
418+
assert(dataProcessors.size());
419+
dataProcessors.back().name = s;
409420
} else if (in(State::IN_METADATUM_NAME)) {
410421
assert(metadata.size());
411422
metadata.back().name = s;
@@ -459,13 +470,13 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
459470
} else if (in(State::IN_OUTPUT_LIFETIME)) {
460471
lifetime = (Lifetime)i;
461472
} else if (in(State::IN_DATAPROCESSOR_RANK)) {
462-
output.back().rank = i;
473+
dataProcessors.back().rank = i;
463474
} else if (in(State::IN_DATAPROCESSOR_N_SLOTS)) {
464-
output.back().nSlots = i;
475+
dataProcessors.back().nSlots = i;
465476
} else if (in(State::IN_DATAPROCESSOR_TIMESLICE_ID)) {
466-
output.back().inputTimeSliceId = i;
477+
dataProcessors.back().inputTimeSliceId = i;
467478
} else if (in(State::IN_DATAPROCESSOR_MAX_TIMESLICES)) {
468-
output.back().maxInputTimeslices = i;
479+
dataProcessors.back().maxInputTimeslices = i;
469480
}
470481
pop();
471482
return true;
@@ -526,8 +537,9 @@ struct WorkflowImporter : public rapidjson::BaseReaderHandler<rapidjson::UTF8<>,
526537
std::ostringstream debug;
527538
std::vector<State> states;
528539
std::string spec;
529-
std::vector<DataProcessorSpec>& output;
540+
std::vector<DataProcessorSpec>& dataProcessors;
530541
std::vector<DataProcessorInfo>& metadata;
542+
std::vector<ConfigParamSpec> inputOptions;
531543
std::string binding;
532544
header::DataOrigin origin;
533545
header::DataDescription description;
@@ -612,6 +624,26 @@ void WorkflowSerializationHelpers::dump(std::ostream& out,
612624
}
613625
w.Key("lifetime");
614626
w.Uint((int)input.lifetime);
627+
if (input.metadata.empty() == false) {
628+
w.Key("metadata");
629+
w.StartArray();
630+
for (auto& metadata : input.metadata) {
631+
w.StartObject();
632+
w.Key("name");
633+
w.String(metadata.name.c_str());
634+
auto s = std::to_string(int(metadata.type));
635+
w.Key("type");
636+
w.String(s.c_str());
637+
std::ostringstream oss;
638+
oss << metadata.defaultValue;
639+
w.Key("defaultValue");
640+
w.String(oss.str().c_str());
641+
w.Key("help");
642+
w.String(metadata.help.c_str());
643+
w.EndObject();
644+
}
645+
w.EndArray();
646+
}
615647
w.EndObject();
616648
}
617649
w.EndArray();

‎Framework/Core/test/test_WorkflowSerialization.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ BOOST_AUTO_TEST_CASE(TestVerifyWorkflow)
2222
using namespace o2::framework;
2323
WorkflowSpec w0{
2424
DataProcessorSpec{"A",
25-
{InputSpec{"foo", "A", "COLLISIONCONTEXT", 1, Lifetime::Condition}},
25+
{InputSpec{"foo", "A", "COLLISIONCONTEXT", 1, Lifetime::Condition, {ConfigParamSpec{"aUrl", VariantType::String, "foo/bar", {"A InputSpec option"}}}}},
2626
{OutputSpec{{"bar"}, "C", "D", 2, Lifetime::Timeframe}},
2727
AlgorithmSpec{[](ProcessingContext& ctx) {}},
2828
{ConfigParamSpec{"aInt", VariantType::Int, 0, {"An Int"}},

‎Framework/TestWorkflows/src/o2DiamondWorkflow.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& specs)
6565
})},
6666
{ConfigParamSpec{"some-device-param", VariantType::Int, 1, {"Some device parameter"}}}},
6767
{"B",
68-
{InputSpec{"x", "TST", "A1"}},
68+
{InputSpec{"x", "TST", "A1", Lifetime::Timeframe, {ConfigParamSpec{"somestring", VariantType::String, "", {"Some input param"}}}}},
6969
{OutputSpec{{"b1"}, "TST", "B1"}},
7070
simplePipe("b1", 0)},
7171
{"C",

0 commit comments

Comments
 (0)