Skip to content

Commit 6431b60

Browse files
shahor02shahor02
committed
Fetch entropy dicts from CCDB(default) or local file
Option --ctf-dict <OPT> steers for entropy encoders and decoders of all detectors where the entropy dictionary is taken from. The choices for the OPT are: 1) "": empty string leads to using CCDB objec fetching by the DPL CCDB service (default) 2) filename: use the dictionary from provided file (either tree-based format or flat one in CCDB format) 3) "none": do not use external dictionary, instead per-TF dictionaries will be stored in the CTF
1 parent dfe1eaf commit 6431b60

87 files changed

Lines changed: 621 additions & 369 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Detectors/Base/include/DetectorsBase/CTFCoderBase.h‎

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,16 @@
2525
#include "DetectorsCommonDataFormats/CTFHeader.h"
2626
#include "rANS/rans.h"
2727
#include <filesystem>
28+
#include "Framework/InitContext.h"
29+
#include "Framework/ConcreteDataMatcher.h"
30+
#include "Framework/ConfigParamRegistry.h"
2831

2932
namespace o2
3033
{
34+
namespace framework
35+
{
36+
class ProcessingContext;
37+
}
3138
namespace ctf
3239
{
3340

@@ -45,6 +52,7 @@ class CTFCoderBase
4552

4653
CTFCoderBase() = delete;
4754
CTFCoderBase(int n, DetID det, float memFactor = 1.f) : mCoders(n), mDet(det), mMemMarginFactor(memFactor > 1.f ? memFactor : 1.f) {}
55+
CTFCoderBase(OpType op, int n, DetID det, float memFactor = 1.f) : mOpType(op), mCoders(n), mDet(det), mMemMarginFactor(memFactor > 1.f ? memFactor : 1.f) {}
4856
virtual ~CTFCoderBase() = default;
4957

5058
virtual void createCoders(const std::vector<char>& bufVec, o2::ctf::CTFCoderBase::OpType op) = 0;
@@ -85,9 +93,23 @@ class CTFCoderBase
8593
void setVerbosity(int v) { mVerbosity = v; }
8694
int getVerbosity() const { return mVerbosity; }
8795

96+
const CTFDictHeader& getExtDictHeader() const { return mExtHeader; }
97+
8898
template <typename T>
8999
static bool readFromTree(TTree& tree, const std::string brname, T& dest, int ev = 0);
90100

101+
// these are the helper methods for the parent encoding/decoding task
102+
template <typename CTF>
103+
void init(o2::framework::InitContext& ic);
104+
105+
template <typename CTF, typename BUF>
106+
size_t finaliseCTFOutput(BUF& buffer);
107+
108+
template <typename CTF>
109+
bool finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj);
110+
111+
void updateTimeDependentParams(o2::framework::ProcessingContext& pc);
112+
91113
protected:
92114
std::string getPrefix() const { return o2::utils::Str::concat_string(mDet.getName(), "_CTF: "); }
93115
void assignDictVersion(CTFDictHeader& h) const
@@ -103,6 +125,8 @@ class CTFCoderBase
103125
DetID mDet;
104126
CTFDictHeader mExtHeader; // external dictionary header
105127
float mMemMarginFactor = 1.0f; // factor for memory allocation in EncodedBlocks
128+
bool mLoadDictFromCCDB{true};
129+
OpType mOpType; // Encoder or Decoder
106130
int mVerbosity = 0;
107131
};
108132

@@ -170,15 +194,15 @@ std::vector<char> CTFCoderBase::readDictionaryFromFile(const std::string& dictPa
170194
} else if (bv) {
171195
bufVec.swap(*bv);
172196
if (bufVec.size()) {
173-
auto dictHeader = static_cast<o2::ctf::CTFDictHeader&>(CTF::get(bufVec.data())->getHeader());
197+
auto dictHeader = static_cast<const o2::ctf::CTFDictHeader&>(CTF::get(bufVec.data())->getHeader());
174198
if (dictHeader.det != mDet) {
175199
throw std::runtime_error(fmt::format("{} contains dictionary vector for {}, expected {}", dictPath, dictHeader.det.getName(), mDet.getName()));
176200
}
177201
}
178202
}
179203
if (bufVec.size()) {
180204
mExtHeader = static_cast<CTFDictHeader&>(CTF::get(bufVec.data())->getHeader());
181-
LOGP(info, "Found {} in {}", mExtHeader.asString(), dictPath);
205+
LOGP(debug, "Found {} in {}", mExtHeader.asString(), dictPath);
182206
} else {
183207
std::string errstr = fmt::format("CTF dictionary file for detector {} is empty", mDet.getName());
184208
if (mayFail) {
@@ -190,6 +214,57 @@ std::vector<char> CTFCoderBase::readDictionaryFromFile(const std::string& dictPa
190214
return bufVec;
191215
}
192216

217+
///________________________________
218+
template <typename CTF>
219+
void CTFCoderBase::init(o2::framework::InitContext& ic)
220+
{
221+
if (ic.options().hasOption("mem-factor")) {
222+
setMemMarginFactor(ic.options().get<float>("mem-factor"));
223+
}
224+
auto dict = ic.options().get<std::string>("ctf-dict");
225+
if (dict.empty()) { // load from CCDB
226+
mLoadDictFromCCDB = true;
227+
} else {
228+
if (dict != "none") { // none means per-CTF dictionary will created on the fly
229+
createCodersFromFile<CTF>(dict, mOpType);
230+
LOGP(info, "Loaded {} from {}", mExtHeader.asString(), dict);
231+
} else {
232+
LOGP(info, "Internal per-TF CTF Dict will be created");
233+
}
234+
mLoadDictFromCCDB = false; // don't try to load from CCDB
235+
}
236+
}
237+
238+
///________________________________
239+
template <typename CTF, typename BUF>
240+
size_t CTFCoderBase::finaliseCTFOutput(BUF& buffer)
241+
{
242+
auto eeb = CTF::get(buffer.data()); // cast to container pointer
243+
eeb->compactify(); // eliminate unnecessary padding
244+
buffer.resize(eeb->size()); // shrink buffer to strictly necessary size
245+
// eeb->print();
246+
return eeb->size();
247+
}
248+
249+
///________________________________
250+
template <typename CTF>
251+
bool CTFCoderBase::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
252+
{
253+
bool match = false;
254+
if (mLoadDictFromCCDB && (match = (matcher == o2::framework::ConcreteDataMatcher(mDet.getDataOrigin(), "CTFDICT", 0)))) {
255+
const auto* dict = (std::vector<char>*)obj;
256+
if (dict->empty()) {
257+
LOGP(info, "Empty dictionary object fetched from CCDB, internal per-TF CTF Dict will be created");
258+
} else {
259+
createCoders(*dict, mOpType);
260+
mExtHeader = static_cast<const CTFDictHeader&>(CTF::get(dict->data())->getHeader());
261+
LOGP(info, "Loaded {} from CCDB", mExtHeader.asString());
262+
}
263+
mLoadDictFromCCDB = false; // we read the dictionary at most once!
264+
}
265+
return match;
266+
}
267+
193268
} // namespace ctf
194269
} // namespace o2
195270

‎Detectors/Base/src/CTFCoderBase.cxx‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,12 @@
1414
/// \author ruben.shahoyan@cern.ch
1515

1616
#include "DetectorsBase/CTFCoderBase.h"
17+
#include "Framework/ControlService.h"
18+
#include "Framework/ProcessingContext.h"
19+
#include "Framework/InputRecord.h"
1720

1821
using namespace o2::ctf;
22+
using namespace o2::framework;
1923

2024
void CTFCoderBase::checkDictVersion(const CTFDictHeader& h) const
2125
{
@@ -25,3 +29,10 @@ void CTFCoderBase::checkDictVersion(const CTFDictHeader& h) const
2529
}
2630
}
2731
}
32+
33+
void CTFCoderBase::updateTimeDependentParams(ProcessingContext& pc)
34+
{
35+
if (mLoadDictFromCCDB) {
36+
pc.inputs().get<std::vector<char>*>("ctfdict"); // just to trigger the finaliseCCDB
37+
}
38+
}

‎Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ namespace cpv
3636
class CTFCoder : public o2::ctf::CTFCoderBase
3737
{
3838
public:
39-
CTFCoder() : o2::ctf::CTFCoderBase(CTF::getNBlocks(), o2::detectors::DetID::CPV) {}
39+
CTFCoder(o2::ctf::CTFCoderBase::OpType op) : o2::ctf::CTFCoderBase(op, CTF::getNBlocks(), o2::detectors::DetID::CPV) {}
4040
~CTFCoder() final = default;
4141

4242
/// entropy-encode data to buffer with CTF

‎Detectors/CPV/workflow/include/CPVWorkflow/EntropyDecoderSpec.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class EntropyDecoderSpec : public o2::framework::Task
3333
void run(o2::framework::ProcessingContext& pc) final;
3434
void init(o2::framework::InitContext& ic) final;
3535
void endOfStream(o2::framework::EndOfStreamContext& ec) final;
36+
void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final;
3637

3738
private:
3839
o2::cpv::CTFCoder mCTFCoder;

‎Detectors/CPV/workflow/include/CPVWorkflow/EntropyEncoderSpec.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class EntropyEncoderSpec : public o2::framework::Task
3333
void run(o2::framework::ProcessingContext& pc) final;
3434
void init(o2::framework::InitContext& ic) final;
3535
void endOfStream(o2::framework::EndOfStreamContext& ec) final;
36+
void finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) final;
3637

3738
private:
3839
o2::cpv::CTFCoder mCTFCoder;

‎Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,26 +24,31 @@ namespace o2
2424
namespace cpv
2525
{
2626

27-
EntropyDecoderSpec::EntropyDecoderSpec(int verbosity)
27+
EntropyDecoderSpec::EntropyDecoderSpec(int verbosity) : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Decoder)
2828
{
2929
mTimer.Stop();
3030
mTimer.Reset();
3131
mCTFCoder.setVerbosity(verbosity);
3232
}
3333

34-
void EntropyDecoderSpec::init(o2::framework::InitContext& ic)
34+
void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
3535
{
36-
std::string dictPath = ic.options().get<std::string>("ctf-dict");
37-
if (!dictPath.empty() && dictPath != "none") {
38-
mCTFCoder.createCodersFromFile<CTF>(dictPath, o2::ctf::CTFCoderBase::OpType::Decoder);
36+
if (mCTFCoder.finaliseCCDB<CTF>(matcher, obj)) {
37+
return;
3938
}
4039
}
4140

41+
void EntropyDecoderSpec::init(o2::framework::InitContext& ic)
42+
{
43+
mCTFCoder.init<CTF>(ic);
44+
}
45+
4246
void EntropyDecoderSpec::run(ProcessingContext& pc)
4347
{
4448
auto cput = mTimer.CpuTime();
4549
mTimer.Start(false);
4650

51+
mCTFCoder.updateTimeDependentParams(pc);
4752
auto buff = pc.inputs().get<gsl::span<o2::ctf::BufferType>>("ctf");
4853

4954
auto& triggers = pc.outputs().make<std::vector<TriggerRecord>>(OutputRef{"triggers"});
@@ -75,7 +80,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec)
7580
Inputs{InputSpec{"ctf", "CPV", "CTFDATA", sspec, Lifetime::Timeframe}},
7681
outputs,
7782
AlgorithmSpec{adaptFromTask<EntropyDecoderSpec>(verbosity)},
78-
Options{{"ctf-dict", VariantType::String, o2::base::NameConf::getCTFDictFileName(), {"File of CTF decoding dictionary"}}}};
83+
Options{{"ctf-dict", VariantType::String, "", {"CTF dictionary: empty=CCDB, none=no external dictionary otherwise: local filename"}}}};
7984
}
8085

8186
} // namespace cpv

‎Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
#include "Framework/ControlService.h"
1717
#include "Framework/ConfigParamRegistry.h"
18+
#include "Framework/CCDBParamSpec.h"
1819
#include "CPVWorkflow/EntropyEncoderSpec.h"
1920
#include "DetectorsCommonDataFormats/DetID.h"
2021

@@ -25,36 +26,37 @@ namespace o2
2526
namespace cpv
2627
{
2728

28-
EntropyEncoderSpec::EntropyEncoderSpec()
29+
EntropyEncoderSpec::EntropyEncoderSpec() : mCTFCoder(o2::ctf::CTFCoderBase::OpType::Encoder)
2930
{
3031
mTimer.Stop();
3132
mTimer.Reset();
3233
}
3334

34-
void EntropyEncoderSpec::init(o2::framework::InitContext& ic)
35+
void EntropyEncoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj)
3536
{
36-
std::string dictPath = ic.options().get<std::string>("ctf-dict");
37-
mCTFCoder.setMemMarginFactor(ic.options().get<float>("mem-factor"));
38-
if (!dictPath.empty() && dictPath != "none") {
39-
mCTFCoder.createCodersFromFile<CTF>(dictPath, o2::ctf::CTFCoderBase::OpType::Encoder);
37+
if (mCTFCoder.finaliseCCDB<CTF>(matcher, obj)) {
38+
return;
4039
}
4140
}
4241

42+
void EntropyEncoderSpec::init(o2::framework::InitContext& ic)
43+
{
44+
mCTFCoder.init<CTF>(ic);
45+
}
46+
4347
void EntropyEncoderSpec::run(ProcessingContext& pc)
4448
{
4549
auto cput = mTimer.CpuTime();
4650
mTimer.Start(false);
51+
mCTFCoder.updateTimeDependentParams(pc);
4752
auto triggers = pc.inputs().get<gsl::span<TriggerRecord>>("triggers");
4853
auto clusters = pc.inputs().get<gsl::span<Cluster>>("clusters");
4954

5055
auto& buffer = pc.outputs().make<std::vector<o2::ctf::BufferType>>(Output{"CPV", "CTFDATA", 0, Lifetime::Timeframe});
5156
mCTFCoder.encode(buffer, triggers, clusters);
52-
auto eeb = CTF::get(buffer.data()); // cast to container pointer
53-
eeb->compactify(); // eliminate unnecessary padding
54-
buffer.resize(eeb->size()); // shrink buffer to strictly necessary size
55-
// eeb->print();
57+
auto sz = mCTFCoder.finaliseCTFOutput<CTF>(buffer);
5658
mTimer.Stop();
57-
LOG(info) << "Created encoded data of size " << eeb->size() << " for CPV in " << mTimer.CpuTime() - cput << " s";
59+
LOG(info) << "Created encoded data of size " << sz << " for CPV in " << mTimer.CpuTime() - cput << " s";
5860
}
5961

6062
void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec)
@@ -68,13 +70,14 @@ DataProcessorSpec getEntropyEncoderSpec()
6870
std::vector<InputSpec> inputs;
6971
inputs.emplace_back("triggers", "CPV", "CLUSTERTRIGRECS", 0, Lifetime::Timeframe);
7072
inputs.emplace_back("clusters", "CPV", "CLUSTERS", 0, Lifetime::Timeframe);
73+
inputs.emplace_back("ctfdict", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionary"));
7174

7275
return DataProcessorSpec{
7376
"cpv-entropy-encoder",
7477
inputs,
7578
Outputs{{"CPV", "CTFDATA", 0, Lifetime::Timeframe}},
7679
AlgorithmSpec{adaptFromTask<EntropyEncoderSpec>()},
77-
Options{{"ctf-dict", VariantType::String, o2::base::NameConf::getCTFDictFileName(), {"File of CTF encoding dictionary"}},
80+
Options{{"ctf-dict", VariantType::String, "", {"CTF dictionary: empty=CCDB, none=no external dictionary otherwise: local filename"}},
7881
{"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}};
7982
}
8083

‎Detectors/CTF/test/test_ctf_io_cpv.cxx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
5454
sw.Start();
5555
std::vector<o2::ctf::BufferType> vec;
5656
{
57-
CTFCoder coder;
57+
CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
5858
coder.encode(vec, triggers, clusters); // compress
5959
}
6060
sw.Stop();
@@ -92,7 +92,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
9292
sw.Start();
9393
const auto ctfImage = o2::cpv::CTF::getImage(vec.data());
9494
{
95-
CTFCoder coder;
95+
CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Decoder);
9696
coder.decode(ctfImage, triggersD, clustersD); // decompress
9797
}
9898
sw.Stop();

‎Detectors/CTF/test/test_ctf_io_ctp.cxx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
4848
sw.Start();
4949
std::vector<o2::ctf::BufferType> vec;
5050
{
51-
CTFCoder coder;
51+
CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
5252
coder.encode(vec, digits); // compress
5353
}
5454
sw.Stop();
@@ -84,7 +84,7 @@ BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled())
8484
sw.Start();
8585
const auto ctfImage = o2::ctp::CTF::getImage(vec.data());
8686
{
87-
CTFCoder coder;
87+
CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Decoder);
8888
coder.decode(ctfImage, digitsD); // decompress
8989
}
9090
sw.Stop();

‎Detectors/CTF/test/test_ctf_io_emcal.cxx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
5252
sw.Start();
5353
std::vector<o2::ctf::BufferType> vec;
5454
{
55-
CTFCoder coder;
55+
CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder);
5656
coder.encode(vec, triggers, cells); // compress
5757
}
5858
sw.Stop();
@@ -89,7 +89,7 @@ BOOST_AUTO_TEST_CASE(CTFTest)
8989
sw.Start();
9090
const auto ctfImage = o2::emcal::CTF::getImage(vec.data());
9191
{
92-
CTFCoder coder;
92+
CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Decoder);
9393
coder.decode(ctfImage, triggersD, cellsD); // decompress
9494
}
9595
sw.Stop();

0 commit comments

Comments
 (0)