From cc1a7def2296addec7476126e5646116acd6996c Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Fri, 8 Sep 2023 14:45:00 +0200 Subject: [PATCH] DPL Analysis: provide metadata to workflow construction --- .../src/AODJAlienReaderHelpers.cxx | 4 +- Framework/Core/CMakeLists.txt | 1 + .../include/Framework/ConfigParamDiscovery.h | 34 +++++++++ .../include/Framework/runDataProcessing.h | 3 + Framework/Core/src/ConfigParamDiscovery.cxx | 71 +++++++++++++++++++ .../Core/src/WorkflowCustomizationHelpers.cxx | 4 ++ Framework/Core/src/WorkflowHelpers.cxx | 2 +- .../TestWorkflows/src/o2AnalysisWorkflow.cxx | 13 +++- 8 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 Framework/Core/include/Framework/ConfigParamDiscovery.h create mode 100644 Framework/Core/src/ConfigParamDiscovery.cxx diff --git a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx index 1e20f46c2e464..9cd7d5affc379 100644 --- a/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx +++ b/Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx @@ -133,12 +133,12 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback() stats.updateStats({static_cast(ProcessingStatsId::ARROW_MESSAGES_DESTROYED), DataProcessingStats::Op::Set, 0}); stats.updateStats({static_cast(ProcessingStatsId::ARROW_BYTES_EXPIRED), DataProcessingStats::Op::Set, 0}); - if (!options.isSet("aod-file")) { + if (!options.isSet("aod-file-private")) { LOGP(fatal, "No input file defined!"); throw std::runtime_error("Processing is stopped!"); } - auto filename = options.get("aod-file"); + auto filename = options.get("aod-file-private"); std::string parentFileReplacement; if (options.isSet("aod-parent-base-path-replacement")) { diff --git a/Framework/Core/CMakeLists.txt b/Framework/Core/CMakeLists.txt index eabc3f7c30593..f71744f33f77e 100644 --- a/Framework/Core/CMakeLists.txt +++ b/Framework/Core/CMakeLists.txt @@ -45,6 +45,7 @@ o2_add_library(Framework src/DefaultsHelpers.cxx src/DomainInfoHeader.cxx src/ProcessingPoliciesHelpers.cxx + src/ConfigParamDiscovery.cxx src/ConfigParamStore.cxx src/ConfigParamsHelper.cxx src/ChannelParamSpec.cxx diff --git a/Framework/Core/include/Framework/ConfigParamDiscovery.h b/Framework/Core/include/Framework/ConfigParamDiscovery.h new file mode 100644 index 0000000000000..c4bdb497366b7 --- /dev/null +++ b/Framework/Core/include/Framework/ConfigParamDiscovery.h @@ -0,0 +1,34 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifndef O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_ +#define O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_ + +#include "Framework/ConfigParamRegistry.h" + +namespace o2::framework +{ +struct ConfigParamDiscovery { + /// Given a ConfigParamRegistry, discovers all the extrac ConfigParam + /// which require an additional lookup starting from the ones already + /// present. E.g. you could use this to populate the registry with + /// ConfigParams defined in a file. + static void discover(ConfigParamRegistry& registry); +}; + +/// A plugin which is able to discover more option then the ones +/// provided on the command line. +struct ConfigDiscoveryPlugin { + virtual void discover(ConfigParamRegistry& registry) = 0; +}; + +} // namespace o2::framework +#endif // O2_FRAMEWORK_CONFIGPARAMDISCOVERY_H_ diff --git a/Framework/Core/include/Framework/runDataProcessing.h b/Framework/Core/include/Framework/runDataProcessing.h index bfda47d07971c..2112dc7cbfd5d 100644 --- a/Framework/Core/include/Framework/runDataProcessing.h +++ b/Framework/Core/include/Framework/runDataProcessing.h @@ -30,6 +30,7 @@ #include "Framework/Logger.h" #include "Framework/CheckTypes.h" #include "Framework/StructToTuple.h" +#include "Framework/ConfigParamDiscovery.h" #include #include @@ -203,6 +204,8 @@ int mainNoCatch(int argc, char** argv) workflowOptionsStore->preload(); workflowOptionsStore->activate(); ConfigParamRegistry workflowOptionsRegistry(std::move(workflowOptionsStore)); + ConfigParamDiscovery::discover(workflowOptionsRegistry); + ConfigContext configContext(workflowOptionsRegistry, argc, argv); o2::framework::WorkflowSpec specs = defineDataProcessing(configContext); overrideCloning(configContext, specs); diff --git a/Framework/Core/src/ConfigParamDiscovery.cxx b/Framework/Core/src/ConfigParamDiscovery.cxx new file mode 100644 index 0000000000000..e392bf155d457 --- /dev/null +++ b/Framework/Core/src/ConfigParamDiscovery.cxx @@ -0,0 +1,71 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "Framework/ConfigParamDiscovery.h" +#include "Framework/Plugins.h" +#include +#include +#include + +namespace o2::framework +{ + +void ConfigParamDiscovery::discover(ConfigParamRegistry& registry) +{ + // Keep track of the plugins + std::vector plugins; + if (registry.hasOption("aod-file") && registry.isSet("aod-file")) { + auto filename = registry.get("aod-file"); + if (filename.empty()) { + return; + } + TFile* currentFile = nullptr; + if (filename.at(0) == '@') { + filename.erase(0, 1); + // read the text file and set filename to the contents of the first line + std::ifstream file(filename); + if (!file.is_open()) { + LOGP(fatal, "Couldn't open file \"{}\"!", filename); + } + std::getline(file, filename); + file.close(); + currentFile = TFile::Open(filename.c_str()); + } else { + currentFile = TFile::Open(filename.c_str()); + } + if (!currentFile) { + LOGP(fatal, "Couldn't open file \"{}\"!", filename); + } + + // Get the metadata, if any + auto m = (TMap*)currentFile->Get("metaData"); + if (!m) { + LOGP(warning, "No metadata found in file \"{}\"", filename); + return; + } + auto it = m->MakeIterator(); + + // Serialise metadata into a ; separated string with : separating key and value + bool first = true; + while (auto obj = it->Next()) { + if (first) { + LOGP(info, "Metadata for file \"{}\":", filename); + first = false; + } + auto objString = (TObjString*)m->GetValue(obj); + LOGP(info, "- {}: {}", obj->GetName(), objString->String()); + std::string key = "aod-metadata-" + std::string(obj->GetName()); + registry.override(key.c_str(), objString->String()); + } + } +} + +} // namespace o2::framework diff --git a/Framework/Core/src/WorkflowCustomizationHelpers.cxx b/Framework/Core/src/WorkflowCustomizationHelpers.cxx index da21168a8fe44..efdc535c683c5 100644 --- a/Framework/Core/src/WorkflowCustomizationHelpers.cxx +++ b/Framework/Core/src/WorkflowCustomizationHelpers.cxx @@ -54,6 +54,10 @@ std::vector WorkflowCustomizationHelpers::requiredWorkflowOptio // options for AOD rate limiting {"aod-memory-rate-limit", VariantType::Int64, 0LL, {"Rate limit AOD processing based on memory"}}, + // options for the AOD reader + // aod-file needs to be available as workflow option, because we + // can configure the workflow based on the contents of the first file. + {"aod-file", VariantType::String, "", {"Input AOD file"}}, // options for AOD writer {"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}}, {"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}}, diff --git a/Framework/Core/src/WorkflowHelpers.cxx b/Framework/Core/src/WorkflowHelpers.cxx index ce56385f71dd4..c2049dedcfc8f 100644 --- a/Framework/Core/src/WorkflowHelpers.cxx +++ b/Framework/Core/src/WorkflowHelpers.cxx @@ -297,7 +297,7 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext static_cast(compile_time_hash("internal-dpl-aod-reader")), aodLifetime}}, .algorithm = AlgorithmSpec::dummyAlgorithm(), - .options = {ConfigParamSpec{"aod-file", VariantType::String, {"Input AOD file"}}, + .options = {ConfigParamSpec{"aod-file-private", VariantType::String, ctx.options().get("aod-file"), {"AOD file"}}, ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}}, ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}}, ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}}, diff --git a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx index c0da7a24c1a90..b1c6da0775b2e 100644 --- a/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx +++ b/Framework/TestWorkflows/src/o2AnalysisWorkflow.cxx @@ -49,7 +49,14 @@ struct EtaAndClsHistograms { WorkflowSpec defineDataProcessing(ConfigContext const& cfgc) { - return WorkflowSpec{ - adaptAnalysisTask(cfgc), - }; + // For the sake of running without an option, we do not throw an exception + // in case the option is not present. + if (cfgc.options().hasOption("aod-metadata-Run") == false || + cfgc.options().get("aod-metadata-Run") == "2") { + return WorkflowSpec{ + adaptAnalysisTask(cfgc), + }; + } else { + throw std::runtime_error("Unsupported run type"); + } }