forked from mcoquet642/AliceO2
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathDataSampling.cxx
More file actions
127 lines (106 loc) · 4.61 KB
/
DataSampling.cxx
File metadata and controls
127 lines (106 loc) · 4.61 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
// Copyright CERN and copyright holders of ALICE O2. This software is
// distributed under the terms of the GNU General Public License v3 (GPL
// Version 3), copied verbatim in the file "COPYING".
//
// See http://alice-o2.web.cern.ch/license for full licensing information.
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
/// \file DataSampling.cxx
/// \brief Implementation of O2 Data Sampling, v1.0
///
/// \author Piotr Konopka, piotr.jan.konopka@cern.ch
#include "Framework/DataSampling.h"
#include "Framework/Dispatcher.h"
#include "Framework/CompletionPolicyHelpers.h"
#include "Framework/DataSpecUtils.h"
#include "Framework/Logger.h"
#include <Configuration/ConfigurationInterface.h>
#include <Configuration/ConfigurationFactory.h>
using namespace o2::configuration;
using SubSpecificationType = o2::header::DataHeader::SubSpecificationType;
namespace o2
{
namespace framework
{
std::string DataSampling::createDispatcherName()
{
return std::string("Dispatcher"); //_") + getenv("HOSTNAME");
}
void DataSampling::GenerateInfrastructure(WorkflowSpec& workflow, const std::string& policiesSource, size_t threads)
{
LOG(DEBUG) << "Generating Data Sampling infrastructure...";
Dispatcher dispatcher(createDispatcherName(), policiesSource);
Options options;
std::unique_ptr<ConfigurationInterface> cfg = ConfigurationFactory::getConfiguration(policiesSource);
auto policiesTree = cfg->getRecursive("dataSamplingPolicies");
for (auto&& policyConfig : policiesTree) {
std::unique_ptr<DataSamplingPolicy> policy;
// We don't want the Dispatcher to exit due to one faulty Policy
try {
policy = std::make_unique<DataSamplingPolicy>(policyConfig.second);
} catch (const std::exception& ex) {
LOG(WARN) << "Could not load the Data Sampling Policy '"
<< policyConfig.second.get_optional<std::string>("id").value_or("") << "', because: " << ex.what();
continue;
} catch (...) {
LOG(WARN) << "Could not load the Data Sampling Policy '"
<< policyConfig.second.get_optional<std::string>("id").value_or("") << "'";
continue;
}
for (const auto& path : policy->getPathMap()) {
dispatcher.registerPath({path.first, path.second});
}
if (!policy->getFairMQOutputChannel().empty()) {
options.push_back({"channel-config", VariantType::String, policy->getFairMQOutputChannel().c_str(), {"Out-of-band channel config"}});
LOG(DEBUG) << " - registering output FairMQ channel '" << policy->getFairMQOutputChannel() << "'";
}
}
if (dispatcher.getInputSpecs().size() > 0) {
DataProcessorSpec spec;
spec.name = dispatcher.getName();
spec.inputs = dispatcher.getInputSpecs();
spec.outputs = dispatcher.getOutputSpecs();
spec.algorithm = adaptFromTask<Dispatcher>(std::move(dispatcher));
spec.maxInputTimeslices = threads;
spec.labels = {{"DataSampling"}, {"Dispatcher"}};
spec.options = options;
workflow.emplace_back(std::move(spec));
} else {
LOG(DEBUG) << "No input to this dispatcher, it won't be added to the workflow.";
}
}
void DataSampling::CustomizeInfrastructure(std::vector<CompletionPolicy>& policies)
{
CompletionPolicy dispatcherConsumesASAP = CompletionPolicyHelpers::defineByName(createDispatcherName(), CompletionPolicy::CompletionOp::Consume);
policies.push_back(dispatcherConsumesASAP);
}
void DataSampling::CustomizeInfrastructure(std::vector<ChannelConfigurationPolicy>& policies)
{
// todo: add push-pull for channels that require blocking
// now it cannot be done, since matching is possible only using data processors names
}
std::vector<InputSpec> DataSampling::InputSpecsForPolicy(const std::string& policiesSource, const std::string& policyName)
{
std::unique_ptr<ConfigurationInterface> config = ConfigurationFactory::getConfiguration(policiesSource);
return InputSpecsForPolicy(config.get(), policyName);
}
std::vector<InputSpec> DataSampling::InputSpecsForPolicy(ConfigurationInterface* const config, const std::string& policyName)
{
std::vector<InputSpec> inputs;
auto policiesTree = config->getRecursive("dataSamplingPolicies");
for (auto&& policyConfig : policiesTree) {
if (policyConfig.second.get<std::string>("id") == policyName) {
DataSamplingPolicy policy(policyConfig.second);
for (const auto& path : policy.getPathMap()) {
InputSpec input = DataSpecUtils::matchingInput(path.second);
inputs.push_back(input);
}
break;
}
}
return inputs;
}
} // namespace framework
} // namespace o2