Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Common/Utils/include/CommonUtils/ConfigurableParam.h
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,8 @@ class ConfigurableParam
// writes a human readable INI or JSON file depending on the extension
static void write(std::string const& filename, std::string const& keyOnly = "");

static std::string asJSON(std::string const& keyOnly = "");

// can be used instead of using API on concrete child classes
template <typename T>
static T getValueAs(std::string key)
Expand Down Expand Up @@ -494,6 +496,9 @@ class ConfigurableParam
// be updated, absence of data for any of requested params will lead to fatal
static void updateFromFile(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false);

// update from a JSON string with the same filtering semantics as updateFromFile
static void updateFromJSONString(std::string const&, std::string const& paramsList = "", bool unchangedOnly = false);

// interface for use from the CCDB API; allows to sync objects read from CCDB with the information
// stored in the registry; modifies given object as well as registry
virtual void syncCCDBandRegistry(void* obj) = 0;
Expand Down
98 changes: 76 additions & 22 deletions Common/Utils/src/ConfigurableParam.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
#endif
#include <cassert>
#include <iostream>
#include <sstream>
#include <string>
#include <fairlogger/Logger.h>
#include <typeindex>
Expand Down Expand Up @@ -754,6 +755,29 @@ void ConfigurableParam::writeJSON(std::string const& filename, std::string const

// ------------------------------------------------------------------

std::string ConfigurableParam::asJSON(std::string const& keyOnly)
{
initPropertyTree(); // update the boost tree before writing
std::ostringstream os;
if (!keyOnly.empty()) { // write ini for selected key only
try {
boost::property_tree::ptree kTree;
auto keys = o2::utils::Str::tokenize(keyOnly, " ,;", true, true);
for (const auto& k : keys) {
kTree.add_child(k, sPtree->get_child(k));
}
boost::property_tree::write_json(os, kTree);
} catch (const boost::property_tree::ptree_bad_path& err) {
LOG(fatal) << "non-existing key " << keyOnly << " provided to writeJSON";
}
} else {
boost::property_tree::write_json(os, *sPtree);
}
return os.str();
}

// ------------------------------------------------------------------

void ConfigurableParam::initPropertyTree()
{
sPtree->clear();
Expand Down Expand Up @@ -870,26 +894,10 @@ void ConfigurableParam::printAllRegisteredParamNames()

// ------------------------------------------------------------------

// Update the storage map of params from the given configuration file.
// It can be in JSON or INI format.
// If nonempty comma-separated paramsList is provided, only those params will
// be updated, absence of data for any of requested params will lead to fatal
// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated
// (to allow preference of run-time settings)
void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly)
namespace
{
void updateFromPropertyTree(boost::property_tree::ptree const& pt, std::string const& source, std::string const& paramsList, bool unchangedOnly)
{
if (!sIsFullyInitialized) {
initialize();
}

auto cfgfile = o2::utils::Str::trim_copy(configFile);

if (cfgfile.length() == 0) {
return;
}

boost::property_tree::ptree pt = ConfigurableParamReaders::readConfigFile(cfgfile);

std::vector<std::pair<std::string, std::string>> keyValPairs;
auto request = o2::utils::Str::tokenize(paramsList, ',', true);
std::unordered_map<std::string, int> requestMap;
Expand All @@ -913,7 +921,7 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
auto name = subKey.first;
auto value = subKey.second.get_value<std::string>();
std::string key = mainKey + "." + name;
if (!unchangedOnly || getProvenance(key) == kCODE) {
if (!unchangedOnly || ConfigurableParam::getProvenance(key) == ConfigurableParam::kCODE) {
std::pair<std::string, std::string> pair = std::make_pair(key, o2::utils::Str::trim_copy(value));
keyValPairs.push_back(pair);
}
Expand All @@ -928,16 +936,62 @@ void ConfigurableParam::updateFromFile(std::string const& configFile, std::strin
// make sure all requested params were retrieved
for (const auto& req : requestMap) {
if (req.second == 0) {
throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, configFile));
throw std::runtime_error(fmt::format("Param {:s} was not found in {:s}", req.first, source));
}
}

try {
setValues(keyValPairs);
ConfigurableParam::setValues(keyValPairs);
} catch (std::exception const& error) {
LOG(error) << "Error while setting values " << error.what();
}
}
} // namespace

// Update the storage map of params from the given configuration file.
// It can be in JSON or INI format.
// If nonempty comma-separated paramsList is provided, only those params will
// be updated, absence of data for any of requested params will lead to fatal
// If unchangedOnly is true, then only those parameters whose provenance is kCODE will be updated
// (to allow preference of run-time settings)
void ConfigurableParam::updateFromFile(std::string const& configFile, std::string const& paramsList, bool unchangedOnly)
{
if (!sIsFullyInitialized) {
initialize();
}

auto cfgfile = o2::utils::Str::trim_copy(configFile);

if (cfgfile.length() == 0) {
return;
}

updateFromPropertyTree(ConfigurableParamReaders::readConfigFile(cfgfile), configFile, paramsList, unchangedOnly);
}

// ------------------------------------------------------------------

void ConfigurableParam::updateFromJSONString(std::string const& configJSON, std::string const& paramsList, bool unchangedOnly)
{
if (!sIsFullyInitialized) {
initialize();
}

auto json = o2::utils::Str::trim_copy(configJSON);
if (json.length() == 0) {
return;
}

boost::property_tree::ptree pt;
std::istringstream input(json);
try {
boost::property_tree::read_json(input, pt);
} catch (const boost::property_tree::ptree_error& e) {
LOG(fatal) << "Failed to read JSON config string (" << e.what() << ")";
}

updateFromPropertyTree(pt, "provided JSON string", paramsList, unchangedOnly);
}

// ------------------------------------------------------------------
// ------------------------------------------------------------------
Expand Down
36 changes: 36 additions & 0 deletions Common/Utils/test/testConfigurableParam.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,42 @@ BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_Json)
std::remove(testFileName.c_str());
}

BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_UnchangedOnly)
{
ConfigurableParam::setValue("TestParam.ulValue", "2");
ConfigurableParam::setProvenance("TestParam", "lValue", ConfigurableParam::kCODE);
ConfigurableParam::setProvenance("TestParam", "ulValue", ConfigurableParam::kRT);
ConfigurableParam::updateFromJSONString(R"json({"TestParam":{"lValue":"77","ulValue":"88"}})json", "TestParam", true);

BOOST_CHECK_EQUAL(TestParam::Instance().lValue, 77);
BOOST_CHECK_EQUAL(TestParam::Instance().ulValue, 2);
}

BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_FromAsJSON)
{
ConfigurableParam::setValue("TestParam.iValue", "321");
ConfigurableParam::setValue("TestParam.sValue", "json-source");
ConfigurableParam::setValues({{"TestParam.map", "{7:8,9:10}"}});
const auto mapBefore = TestParam::Instance().map;
const auto json = ConfigurableParam::asJSON("TestParam");

ConfigurableParam::setValue("TestParam.iValue", "999");
ConfigurableParam::setValue("TestParam.sValue", "json-modified");
ConfigurableParam::setValues({{"TestParam.map", "{1:2}"}});
ConfigurableParam::updateFromJSONString(json, "TestParam");

BOOST_CHECK_EQUAL(TestParam::Instance().iValue, 321);
BOOST_CHECK_EQUAL(TestParam::Instance().sValue, "json-source");
BOOST_CHECK_EQUAL(TestParam::Instance().map.size(), mapBefore.size());
BOOST_CHECK_EQUAL(TestParam::Instance().map.at(7), mapBefore.at(7));
BOOST_CHECK_EQUAL(TestParam::Instance().map.at(9), mapBefore.at(9));
}

BOOST_AUTO_TEST_CASE(ConfigurableParam_JSONString_ParamsListMissing)
{
BOOST_CHECK_THROW(ConfigurableParam::updateFromJSONString(ConfigurableParam::asJSON("TestParam"), "MissingParam"), std::runtime_error);
}

BOOST_AUTO_TEST_CASE(ConfigurableParam_FileIO_ROOT)
{
// test for root file serialization
Expand Down
15 changes: 12 additions & 3 deletions Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
// or submit itself to any jurisdiction.

#include <vector>

#include <TMap.h>
#include <TObjString.h>
#include "Framework/ControlService.h"
#include "Framework/ConfigParamRegistry.h"
#include "Framework/CCDBParamSpec.h"
Expand Down Expand Up @@ -67,8 +68,15 @@ void TrackerDPL::run(ProcessingContext& pc)
if (first) {
first = false;
if (pc.services().get<const o2::framework::DeviceSpec>().inputTimesliceId == 0) {
o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, o2::its::VertexerParamConfig::Instance().getName()), o2::its::VertexerParamConfig::Instance().getName());
o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, o2::its::TrackerParamConfig::Instance().getName()), o2::its::TrackerParamConfig::Instance().getName());
const auto& vtconf = o2::its::VertexerParamConfig::Instance();
const auto& trconf = o2::its::TrackerParamConfig::Instance();
o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, vtconf.getName()), vtconf.getName());
o2::conf::ConfigurableParam::write(o2::base::NameConf::getConfigOutputFileName(pc.services().get<const o2::framework::DeviceSpec>().name, trconf.getName()), trconf.getName());
TMap md;
md.SetOwnerKeyValue();
md.Add(new TObjString(vtconf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(vtconf.getName()).c_str()));
md.Add(new TObjString(trconf.getName().c_str()), new TObjString(o2::conf::ConfigurableParam::asJSON(trconf.getName()).c_str()));
pc.outputs().snapshot(Output{"META", "ITSTRACKER", 0}, md);
}
}
}
Expand Down Expand Up @@ -138,6 +146,7 @@ DataProcessorSpec getTrackerSpec(bool useMC, bool doStag, bool useGeom, int trgT
outputs.emplace_back("ITS", "VERTICESMCPUR", 0, Lifetime::Timeframe);
outputs.emplace_back("ITS", "TRACKSMCTR", 0, Lifetime::Timeframe);
}
outputs.emplace_back("META", "ITSTRACKER", 0, Lifetime::Sporadic);

return DataProcessorSpec{
.name = "its-tracker",
Expand Down
47 changes: 47 additions & 0 deletions Framework/AnalysisSupport/src/AODWriterHelpers.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "Framework/EndOfStreamContext.h"
#include "Framework/ProcessingContext.h"
#include "Framework/InitContext.h"
#include "Framework/Output.h"
#include "Framework/CallbackService.h"
#include "Framework/AnalysisSupportHelpers.h"
#include "Framework/TableConsumer.h"
Expand All @@ -31,6 +32,10 @@
#include <TMap.h>
#include <TObjString.h>
#include <arrow/table.h>
#include <algorithm>
#include <string>
#include <utility>
#include <vector>

O2_DECLARE_DYNAMIC_LOG(histogram_registry);

Expand Down Expand Up @@ -477,4 +482,46 @@ AlgorithmSpec AODWriterHelpers::getOutputObjHistWriter(ConfigContext const& /*ct
};
}};
}

AlgorithmSpec AODWriterHelpers::getMetadataCollector(ConfigContext const& /*ctx*/)
{
return AlgorithmSpec{[](InitContext&) -> std::function<void(ProcessingContext&)> {
// Accumulated metadata, last writer wins per key.
auto merged = std::make_shared<std::vector<std::pair<std::string, std::string>>>();
return [merged](ProcessingContext& pc) -> void {
auto nParts = pc.inputs().getNofParts(0);
for (auto pi = 0U; pi < nParts; ++pi) {
auto part = pc.inputs().get<TMap*>("meta", pi);
if (!part) {
continue;
}
TIter next(part.get());
while (TObject* key = next()) {
TObject* value = part->GetValue(key);
std::string k = key->GetName();
std::string v = value != nullptr ? value->GetName() : "";
auto it = std::ranges::find_if(*merged,
[&k](auto const& e) { return e.first == k; });
if (it != merged->end()) {
it->second = std::move(v);
} else {
merged->emplace_back(std::move(k), std::move(v));
}
}
}
// Emit the keys/vals vectors the AOD writer already turns into the AO2D
// metaData TMap, so no special handling is needed there.
std::vector<TString> keys, vals;
keys.reserve(merged->size());
vals.reserve(merged->size());
for (auto const& [k, v] : *merged) {
keys.emplace_back(k);
vals.emplace_back(v);
}
LOG(debug) << "metadata-collector: emitting " << keys.size() << " aggregated metadata entries";
pc.outputs().snapshot(Output{"AMD", "AODMetadataKeys", 0}, keys);
pc.outputs().snapshot(Output{"AMD", "AODMetadataVals", 0}, vals);
};
}};
}
} // namespace o2::framework::writers
1 change: 1 addition & 0 deletions Framework/AnalysisSupport/src/AODWriterHelpers.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ namespace o2::framework::writers
struct AODWriterHelpers {
static AlgorithmSpec getOutputObjHistWriter(ConfigContext const& context);
static AlgorithmSpec getOutputTTreeWriter(ConfigContext const& context);
static AlgorithmSpec getMetadataCollector(ConfigContext const& context);
};

} // namespace o2::framework::writers
Expand Down
8 changes: 8 additions & 0 deletions Framework/AnalysisSupport/src/Plugin.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,13 @@ struct ROOTTTreeWriter : o2::framework::AlgorithmPlugin {
}
};

struct ROOTMetadataCollector : o2::framework::AlgorithmPlugin {
o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const& config) override
{
return o2::framework::writers::AODWriterHelpers::getMetadataCollector(config);
}
};

using namespace o2::framework;
struct RunSummary : o2::framework::ServicePlugin {
o2::framework::ServiceSpec* create() final
Expand Down Expand Up @@ -286,6 +293,7 @@ DEFINE_DPL_PLUGINS_BEGIN
DEFINE_DPL_PLUGIN_INSTANCE(ROOTFileReader, CustomAlgorithm);
DEFINE_DPL_PLUGIN_INSTANCE(ROOTObjWriter, CustomAlgorithm);
DEFINE_DPL_PLUGIN_INSTANCE(ROOTTTreeWriter, CustomAlgorithm);
DEFINE_DPL_PLUGIN_INSTANCE(ROOTMetadataCollector, CustomAlgorithm);
DEFINE_DPL_PLUGIN_INSTANCE(RunSummary, CustomService);
DEFINE_DPL_PLUGIN_INSTANCE(DiscoverMetadataInAOD, ConfigDiscovery);
DEFINE_DPL_PLUGINS_END
3 changes: 3 additions & 0 deletions Framework/Core/include/Framework/AnalysisSupportHelpers.h
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ struct AnalysisSupportHelpers {
static DataProcessorSpec getOutputObjHistSink(ConfigContext const&);
/// writes inputs of kind AOD to file
static DataProcessorSpec getGlobalAODSink(ConfigContext const&);
/// Match all inputs of kind META, merge them and republish as the AOD
/// metadata keys/vals consumed by the AOD writer.
static DataProcessorSpec getMetadataCollectorSink(ConfigContext const&);
/// Get the data director
static std::shared_ptr<DataOutputDirector> getDataOutputDirector(ConfigContext const& ctx);
};
Expand Down
19 changes: 19 additions & 0 deletions Framework/Core/src/AnalysisSupportHelpers.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -220,4 +220,23 @@ DataProcessorSpec

return spec;
}

DataProcessorSpec
AnalysisSupportHelpers::getMetadataCollectorSink(ConfigContext const& ctx)
{
// Lifetime is sporadic because META messages are not produced every
// timeframe. The oldest-possible-timeframe completion policy (registered
// in CompletionPolicy::createDefaultPolicies) decides when the collected
// parts are merged and republished as the AOD metadata keys/vals that the
// AOD writer turns into the AO2D metaData object.
DataProcessorSpec spec{
.name = "internal-dpl-metadata-collector",
.inputs = {InputSpec("meta", DataSpecUtils::dataDescriptorMatcherFrom(header::DataOrigin{"META"}), Lifetime::Sporadic)},
.outputs = {OutputSpec{OutputLabel{"keys"}, header::DataOrigin{"AMD"}, header::DataDescription{"AODMetadataKeys"}, 0, Lifetime::Sporadic},
OutputSpec{OutputLabel{"vals"}, header::DataOrigin{"AMD"}, header::DataDescription{"AODMetadataVals"}, 0, Lifetime::Sporadic}},
.algorithm = PluginManager::loadAlgorithmFromPlugin("O2FrameworkAnalysisSupport", "ROOTMetadataCollector", ctx),
};

return spec;
}
} // namespace o2::framework
1 change: 1 addition & 0 deletions Framework/Core/src/CompletionPolicy.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ std::vector<CompletionPolicy>
return {
CompletionPolicyHelpers::consumeWhenAllOrdered("internal-dpl-aod-writer"),
CompletionPolicyHelpers::consumeWhenAnyZeroCount("internal-dpl-injected-dummy-sink", [](DeviceSpec const& s) { return s.name.find("internal-dpl-injected-dummy-sink") != std::string::npos; }),
CompletionPolicyHelpers::consumeWhenPastOldestPossibleTimeframe("internal-dpl-metadata-collector", [](DeviceSpec const& s) { return s.name == "internal-dpl-metadata-collector"; }),
CompletionPolicyHelpers::consumeWhenAll()};
}

Expand Down
Loading
Loading