Skip to content

Commit b479f27

Browse files
knopers8knopers8
authored
[QC-443] Represent the whole configuration file in C++ structures (#736)
* [QC-443] Refactor configuration of TaskRunner and lay foundation for others Now TaskRunner only gets TaskRunnerConfig with all the information it needs in order to run. It will be able to update the configuration by retrieving it from DPL and letting TaskRunnerFactory provide updated TaskConfig. * test fixes * Fix testTaskRunner * Review fixes * Add missing detector name in CheckWorkflow test
1 parent fd98b05 commit b479f27

30 files changed

Lines changed: 777 additions & 296 deletions

‎Framework/CMakeLists.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ add_library(O2QualityControl
5353
src/TaskInterface.cxx
5454
src/RepositoryBenchmark.cxx
5555
src/InfrastructureGenerator.cxx
56+
src/InfrastructureSpecReader.cxx
5657
src/Check.cxx
5758
src/Aggregator.cxx
5859
src/ServiceDiscovery.cxx
@@ -73,7 +74,8 @@ add_library(O2QualityControl
7374
src/UpdatePolicyManager.cxx
7475
src/AdvancedWorkflow.cxx
7576
src/QualitiesToTRFCollectionConverter.cxx
76-
src/Calculators.cxx)
77+
src/Calculators.cxx
78+
src/DataSourceSpec.cxx)
7779

7880
target_include_directories(
7981
O2QualityControl

‎Framework/include/QualityControl/CommonSpec.h‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#ifndef QUALITYCONTROL_COMMONSPEC_H
12+
#define QUALITYCONTROL_COMMONSPEC_H
13+
14+
///
15+
/// \file CommonSpec.h
16+
/// \author Piotr Konopka
17+
///
18+
19+
#include <string>
20+
#include <unordered_map>
21+
#include <boost/property_tree/ptree_fwd.hpp>
22+
23+
namespace o2::quality_control::core
24+
{
25+
26+
struct CommonSpec {
27+
CommonSpec() = default;
28+
29+
std::unordered_map<std::string, std::string> database;
30+
int activityNumber;
31+
int activityType;
32+
std::string monitoringUrl = "infologger:///debug?qc";
33+
std::string consulUrl;
34+
std::string conditionDBUrl = "http://ccdb-test.cern.ch:8080";
35+
bool infologgerFilterDiscardDebug = false;
36+
int infologgerDiscardLevel = 21;
37+
38+
std::string configurationSource;
39+
};
40+
41+
} // namespace o2::quality_control::core
42+
43+
#endif //QUALITYCONTROL_COMMONSPEC_H

‎Framework/include/QualityControl/DataSourceSpec.h‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#ifndef QUALITYCONTROL_DATASOURCESPEC_H
12+
#define QUALITYCONTROL_DATASOURCESPEC_H
13+
14+
///
15+
/// \file DataSourceSpec.h
16+
/// \author Piotr Konopka
17+
///
18+
19+
#include <string>
20+
#include <unordered_map>
21+
#include <Framework/InputSpec.h>
22+
#include <type_traits>
23+
24+
namespace o2::quality_control::core
25+
{
26+
27+
enum class DataSourceType {
28+
DataSamplingPolicy,
29+
Direct,
30+
Task,
31+
Check,
32+
Aggregator,
33+
PostProcessingTask,
34+
ExternalTask,
35+
Invalid
36+
};
37+
38+
// this should allow us to represent all data sources which come from DPL (and maybe CCDB).
39+
struct DataSourceSpec {
40+
explicit DataSourceSpec(DataSourceType type = DataSourceType::Invalid, std::unordered_map<std::string, std::string> params = {});
41+
42+
// todo: use c++20 concepts when available
43+
template <class... Args, class Enable = std::enable_if_t<(... && std::is_convertible_v<Args, DataSourceType>)>>
44+
bool isOneOf(Args... dataSourceType) const
45+
{
46+
return (... || (dataSourceType == type));
47+
}
48+
49+
DataSourceType type;
50+
std::unordered_map<std::string, std::string> typeSpecificParams;
51+
std::vector<framework::InputSpec> inputs;
52+
};
53+
54+
} // namespace o2::quality_control::core
55+
56+
#endif //QUALITYCONTROL_DATASOURCESPEC_H

‎Framework/include/QualityControl/InfrastructureGenerator.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,9 @@ class InfrastructureGenerator
7171
/// configuration to be 'local'.
7272
///
7373
/// \param configurationSource - full path to configuration file, preceded with the backend (f.e. "json://")
74-
/// \param host - name of the machine
74+
/// \param targetHost - name of the machine
7575
/// \return generated local QC workflow
76-
static framework::WorkflowSpec generateLocalInfrastructure(std::string configurationSource, std::string host);
76+
static framework::WorkflowSpec generateLocalInfrastructure(std::string configurationSource, std::string targetHost);
7777

7878
/// \brief Generates the local part of the QC infrastructure for a specified host.
7979
///

‎Framework/include/QualityControl/InfrastructureSpec.h‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#ifndef QUALITYCONTROL_INFRASTRUCTURESPEC_H
12+
#define QUALITYCONTROL_INFRASTRUCTURESPEC_H
13+
14+
///
15+
/// \file InfrastructureSpec.h
16+
/// \author Piotr Konopka
17+
///
18+
19+
#include "QualityControl/CommonSpec.h"
20+
#include "QualityControl/TaskSpec.h"
21+
22+
#include <vector>
23+
24+
namespace o2::quality_control::core
25+
{
26+
27+
struct InfrastructureSpec {
28+
CommonSpec common;
29+
std::vector<TaskSpec> tasks;
30+
// todo: add other actors
31+
};
32+
33+
} // namespace o2::quality_control::core
34+
35+
#endif //QUALITYCONTROL_INFRASTRUCTURESPEC_H

‎Framework/include/QualityControl/InfrastructureSpecReader.h‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Copyright CERN and copyright holders of ALICE O2. This software is
2+
// distributed under the terms of the GNU General Public License v3 (GPL
3+
// Version 3), copied verbatim in the file "COPYING".
4+
//
5+
// See http://alice-o2.web.cern.ch/license for full licensing information.
6+
//
7+
// In applying this license CERN does not waive the privileges and immunities
8+
// granted to it by virtue of its status as an Intergovernmental Organization
9+
// or submit itself to any jurisdiction.
10+
11+
#ifndef QUALITYCONTROL_INFRASTRUCTURESPECREADER_H
12+
#define QUALITYCONTROL_INFRASTRUCTURESPECREADER_H
13+
14+
///
15+
/// \file InfrastructureSpecReader.h
16+
/// \author Piotr Konopka
17+
///
18+
19+
#include "QualityControl/InfrastructureSpec.h"
20+
#include "QualityControl/TaskSpec.h"
21+
#include "QualityControl/CommonSpec.h"
22+
#include "QualityControl/DataSourceSpec.h"
23+
#include <boost/property_tree/ptree_fwd.hpp>
24+
25+
namespace o2::quality_control::core
26+
{
27+
28+
// If have to increase the performance of reading,
29+
// we can probably improve it by writing a proper parser like for WorkflowSerializationHelpers in O2
30+
// Also, move operators could be implemented.
31+
32+
class InfrastructureSpecReader
33+
{
34+
public:
35+
/// \brief Reads the full QC configuration file.
36+
// todo remove configurationSource when it is possible
37+
static InfrastructureSpec readInfrastructureSpec(const boost::property_tree::ptree&, const std::string& configurationSource);
38+
39+
// readers for separate parts
40+
static CommonSpec readCommonSpec(const boost::property_tree::ptree& config, const std::string& configurationSource);
41+
static TaskSpec readTaskSpec(std::string taskName, const boost::property_tree::ptree& taskSpec, const std::string& configurationSource);
42+
static DataSourceSpec readDataSourceSpec(const boost::property_tree::ptree& dataSourceSpec, const std::string& configurationSource);
43+
44+
static std::string validateDetectorName(std::string name);
45+
};
46+
47+
} // namespace o2::quality_control::core
48+
49+
#endif //QUALITYCONTROL_INFRASTRUCTURESPECREADER_H

‎Framework/include/QualityControl/ObjectsManager.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
// QC
2121
#include "QualityControl/MonitorObject.h"
2222
#include "QualityControl/MonitorObjectCollection.h"
23-
#include "QualityControl/TaskConfig.h"
2423
// stl
2524
#include <string>
2625
#include <memory>

‎Framework/include/QualityControl/TaskFactory.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
// STL
2121
#include <memory>
2222
// QC
23-
#include "QualityControl/TaskConfig.h"
23+
#include "QualityControl/TaskRunnerConfig.h"
2424
#include "QualityControl/TaskInterface.h"
2525

2626
namespace o2::quality_control::core
@@ -43,7 +43,7 @@ class TaskFactory
4343
/// The TaskInterface actual class is decided based on the parameters passed.
4444
/// \todo make it static ?
4545
/// \author Barthelemy von Haller
46-
TaskInterface* create(TaskConfig& taskConfig, std::shared_ptr<ObjectsManager> objectsManager);
46+
TaskInterface* create(TaskRunnerConfig& taskConfig, std::shared_ptr<ObjectsManager> objectsManager);
4747
};
4848

4949
} // namespace o2::quality_control::core

‎Framework/include/QualityControl/TaskRunner.h‎

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
#ifndef QC_CORE_TASKRUNNER_H
1919
#define QC_CORE_TASKRUNNER_H
2020

21-
#include <boost/property_tree/ptree.hpp>
22-
2321
// O2
2422
#include <Common/Timer.h>
2523
#include <Framework/Task.h>
@@ -30,7 +28,7 @@
3028
#include <Headers/DataHeader.h>
3129
#include <Framework/InitContext.h>
3230
// QC
33-
#include "QualityControl/TaskConfig.h"
31+
#include "QualityControl/TaskRunnerConfig.h"
3432
#include "QualityControl/TaskInterface.h"
3533

3634
namespace o2::configuration
@@ -75,7 +73,7 @@ class TaskRunner : public framework::Task
7573
/// \param taskName - name of the task, which exists in tasks list in the configuration file
7674
/// \param configurationSource - absolute path to configuration file, preceded with backend (f.e. "json://")
7775
/// \param id - subSpecification for taskRunner's OutputSpec, useful to avoid outputs collisions one more complex topologies
78-
TaskRunner(const std::string& taskName, const std::string& configurationSource, size_t id = 0);
76+
TaskRunner(const TaskRunnerConfig& config);
7977
~TaskRunner() override = default;
8078

8179
/// \brief TaskRunner's init callback
@@ -86,13 +84,10 @@ class TaskRunner : public framework::Task
8684
/// \brief TaskRunner's completion policy callback
8785
static framework::CompletionPolicy::CompletionOp completionPolicyCallback(o2::framework::InputSpan const& inputs);
8886

89-
std::string getDeviceName() { return mDeviceName; };
90-
const framework::Inputs& getInputsSpecs() { return mInputSpecs; };
91-
const framework::OutputSpec getOutputSpec() { return mMonitorObjectsSpec; };
92-
const framework::Options getOptions() { return mOptions; };
93-
94-
/// \brief Makes TaskRunner invoke TaskInterface::reset() each n cycles. n = 0 means never.
95-
void setResetAfterCycles(size_t n = 0);
87+
std::string getDeviceName() const { return mTaskConfig.deviceName; };
88+
const framework::Inputs& getInputsSpecs() const { return mTaskConfig.inputSpecs; };
89+
const framework::OutputSpec& getOutputSpec() const { return mTaskConfig.moSpec; };
90+
const framework::Options& getOptions() const { return mTaskConfig.options; };
9691

9792
/// \brief ID string for all TaskRunner devices
9893
static std::string createTaskRunnerIdString();
@@ -114,7 +109,6 @@ class TaskRunner : public framework::Task
114109

115110
std::tuple<bool /*data ready*/, bool /*timer ready*/> validateInputs(const framework::InputRecord&);
116111
void loadTaskConfig();
117-
void loadTopologyConfig();
118112
void startOfActivity();
119113
void endOfActivity();
120114
void startCycle();
@@ -124,24 +118,13 @@ class TaskRunner : public framework::Task
124118
void saveToFile();
125119

126120
private:
127-
std::string mDeviceName;
128-
TaskConfig mTaskConfig;
129-
std::shared_ptr<configuration::ConfigurationInterface> mConfigFile; // used in init only
121+
TaskRunnerConfig mTaskConfig;
130122
std::shared_ptr<monitoring::Monitoring> mCollector;
131123
std::shared_ptr<TaskInterface> mTask;
132-
size_t mResetAfterCycles = 0;
133124
std::shared_ptr<ObjectsManager> mObjectsManager;
134125
int mRunNumber;
135126

136-
std::string validateDetectorName(std::string name) const;
137-
boost::property_tree::ptree getTaskConfigTree() const;
138127
void updateMonitoringStats(framework::ProcessingContext& pCtx);
139-
void computeRunNumber(const framework::ServiceRegistry& services);
140-
141-
// consider moving these to TaskConfig
142-
framework::Inputs mInputSpecs;
143-
framework::OutputSpec mMonitorObjectsSpec;
144-
framework::Options mOptions;
145128

146129
bool mCycleOn = false;
147130
bool mNoMoreCycles = false;

Framework/include/QualityControl/TaskConfig.h renamed to Framework/include/QualityControl/TaskRunnerConfig.h

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
// or submit itself to any jurisdiction.
1111

1212
///
13-
/// \file TaskConfig.h
13+
/// \file TaskRunnerConfig.h
1414
/// \author Barthelemy von Haller
1515
///
1616

@@ -19,23 +19,37 @@
1919

2020
#include <string>
2121
#include <unordered_map>
22+
#include <vector>
23+
24+
#include <Framework/DataProcessorSpec.h>
2225

2326
namespace o2::quality_control::core
2427
{
2528

2629
/// \brief Container for the configuration of a Task
27-
struct TaskConfig {
30+
struct TaskRunnerConfig {
31+
std::string deviceName;
2832
std::string taskName;
2933
std::string moduleName;
3034
std::string className;
3135
int cycleDurationSeconds;
3236
int maxNumberCycles;
33-
std::string consulUrl;
34-
std::string conditionUrl = "";
37+
std::string consulUrl{};
38+
std::string conditionUrl{};
39+
std::string monitoringUrl{};
40+
framework::Inputs inputSpecs{};
41+
framework::OutputSpec moSpec{ "XXX", "INVALID" };
42+
framework::Options options{};
3543
std::unordered_map<std::string, std::string> customParameters = {};
3644
std::string detectorName = "MISC"; // intended to be the 3 letters code
3745
int parallelTaskID = 0; // ID to differentiate parallel local Tasks from one another. 0 means this is the only one.
38-
std::string saveToFile = "";
46+
std::string saveToFile{};
47+
int resetAfterCycles = 0;
48+
bool infologgerFilterDiscardDebug = false;
49+
int infologgerDiscardLevel = 21;
50+
int activityType = 0;
51+
int defaultRunNumber = 0;
52+
std::string configurationSource{};
3953
};
4054

4155
} // namespace o2::quality_control::core

0 commit comments

Comments
 (0)