Skip to content

Commit 80005d7

Browse files
shahor02shahor02
committed
Allocate large CTF buffer in advance to avoid rebooking
1 parent 354c2f1 commit 80005d7

11 files changed

Lines changed: 166 additions & 2 deletions

File tree

‎DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -786,8 +786,9 @@ void EncodedBlocks<H, N, W>::encode(const S* const srcBegin, // begin of source
786786
}
787787

788788
// estimate size of encode buffer
789-
int dataSize = rans::calculateMaxBufferSize(messageLength, encoder->getAlphabetRangeBits(), sizeof(S));
789+
int dataSize = rans::calculateMaxBufferSize(messageLength, encoder->getAlphabetRangeBits(), sizeof(S)); // size in bytes
790790
// preliminary expansion of storage based on dict size + estimated size of encode buffer
791+
dataSize = dataSize / sizeof(W) + (sizeof(S) < sizeof(W)); // size in words of output stream
791792
expandStorage(dictSize + dataSize);
792793
//store dictionary first
793794
if (dictSize) {

‎Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class CTFCoder : public o2::ctf::CTFCoderBase
5252
private:
5353
/// compres digits clusters to CompressedDigits
5454
void compress(CompressedDigits& cd, const gsl::span<const Digit>& digitVec, const gsl::span<const ChannelData>& channelVec);
55+
size_t estimateCompressedSize(const CompressedDigits& cc);
5556

5657
/// decompress CompressedDigits to digits
5758
template <typename VDIG, typename VCHAN>
@@ -82,6 +83,11 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const Digit>& digitVec, const g
8283
};
8384
CompressedDigits cd;
8485
compress(cd, digitVec, channelVec);
86+
87+
// book output size with some margin
88+
auto szIni = estimateCompressedSize(cd);
89+
buff.resize(szIni);
90+
8591
auto ec = CTF::create(buff);
8692
using ECB = CTF::base;
8793

‎Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,3 +134,27 @@ void CTFCoder::createCoders(const std::string& dictPath, o2::ctf::CTFCoderBase::
134134
MAKECODER(cd.qtcAmpl, CTF::BLC_qtcAmpl);
135135
// clang-format on
136136
}
137+
138+
///________________________________
139+
size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd)
140+
{
141+
size_t sz = 0;
142+
// clang-format off
143+
// RS FIXME this is very crude estimate, instead, an empirical values should be used
144+
#define VTP(vec) typename std::remove_reference<decltype(vec)>::type::value_type
145+
#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \
146+
rans::calculateMaxBufferSize(vec.size(), reinterpret_cast<const o2::rans::LiteralEncoder64<VTP(vec)>*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec))
147+
sz += ESTSIZE(cd.trigger, CTF::BLC_trigger);
148+
sz += ESTSIZE(cd.bcInc, CTF::BLC_bcInc);
149+
sz += ESTSIZE(cd.orbitInc, CTF::BLC_orbitInc);
150+
sz += ESTSIZE(cd.nChan, CTF::BLC_nChan);
151+
// sz += ESTSIZE(cd.eventFlags, CTF::BLC_flags);
152+
sz += ESTSIZE(cd.idChan, CTF::BLC_idChan);
153+
sz += ESTSIZE(cd.qtcChain, CTF::BLC_qtcChain);
154+
sz += ESTSIZE(cd.cfdTime, CTF::BLC_cfdTime);
155+
sz += ESTSIZE(cd.qtcAmpl, CTF::BLC_qtcAmpl);
156+
// clang-format on
157+
158+
LOG(INFO) << "Estimated output size is " << sz << " bytes";
159+
return sz;
160+
}

‎Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class CTFCoder : public o2::ctf::CTFCoderBase
5252
private:
5353
/// compres digits clusters to CompressedDigits
5454
void compress(CompressedDigits& cd, const gsl::span<const BCData>& digitVec, const gsl::span<const ChannelData>& channelVec);
55+
size_t estimateCompressedSize(const CompressedDigits& cc);
5556

5657
/// decompress CompressedDigits to digits
5758
template <typename VDIG, typename VCHAN>
@@ -80,6 +81,11 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const BCData>& digitVec, const
8081
};
8182
CompressedDigits cd;
8283
compress(cd, digitVec, channelVec);
84+
85+
// book output size with some margin
86+
auto szIni = estimateCompressedSize(cd);
87+
buff.resize(szIni);
88+
8389
auto ec = CTF::create(buff);
8490
using ECB = CTF::base;
8591

‎Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,3 +128,25 @@ void CTFCoder::createCoders(const std::string& dictPath, o2::ctf::CTFCoderBase::
128128
MAKECODER(cd.charge, CTF::BLC_charge);
129129
// clang-format on
130130
}
131+
132+
///________________________________
133+
size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd)
134+
{
135+
size_t sz = 0;
136+
// clang-format off
137+
// RS FIXME this is very crude estimate, instead, an empirical values should be used
138+
#define VTP(vec) typename std::remove_reference<decltype(vec)>::type::value_type
139+
#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \
140+
rans::calculateMaxBufferSize(vec.size(), reinterpret_cast<const o2::rans::LiteralEncoder64<VTP(vec)>*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec))
141+
sz += ESTSIZE(cd.bcInc, CTF::BLC_bcInc);
142+
sz += ESTSIZE(cd.orbitInc, CTF::BLC_orbitInc);
143+
sz += ESTSIZE(cd.nChan, CTF::BLC_nChan);
144+
145+
sz += ESTSIZE(cd.idChan, CTF::BLC_idChan);
146+
sz += ESTSIZE(cd.time, CTF::BLC_time);
147+
sz += ESTSIZE(cd.charge, CTF::BLC_charge);
148+
// clang-format on
149+
150+
LOG(INFO) << "Estimated output size is " << sz << " bytes";
151+
return sz;
152+
}

‎Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/CTFCoder.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class CTFCoder : public o2::ctf::CTFCoderBase
5151
private:
5252
/// compres compact clusters to CompressedClusters
5353
void compress(CompressedClusters& cc, const gsl::span<const ROFRecord>& rofRecVec, const gsl::span<const CompClusterExt>& cclusVec, const gsl::span<const unsigned char>& pattVec);
54+
size_t estimateCompressedSize(const CompressedClusters& cc);
5455

5556
/// decompress CompressedClusters to compact clusters
5657
template <typename VROF, typename VCLUS, typename VPAT>
@@ -83,6 +84,10 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const ROFRecord>& rofRecVec, co
8384
};
8485
CompressedClusters cc;
8586
compress(cc, rofRecVec, cclusVec, pattVec);
87+
// book output size with some margin
88+
auto szIni = estimateCompressedSize(cc);
89+
buff.resize(szIni);
90+
8691
auto ec = CTF::create(buff);
8792
using ECB = CTF::base;
8893

‎Detectors/ITSMFT/common/reconstruction/src/CTFCoder.cxx‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,3 +159,30 @@ void CTFCoder::createCoders(const std::string& dictPath, o2::ctf::CTFCoderBase::
159159
MAKECODER(cc.pattMap, CTF::BLCpattMap );
160160
// clang-format on
161161
}
162+
163+
///________________________________
164+
size_t CTFCoder::estimateCompressedSize(const CompressedClusters& cc)
165+
{
166+
size_t sz = 0;
167+
// clang-format off
168+
// RS FIXME this is very crude estimate, instead, an empirical values should be used
169+
#define VTP(vec) typename std::remove_reference<decltype(vec)>::type::value_type
170+
#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \
171+
rans::calculateMaxBufferSize(vec.size(), reinterpret_cast<const o2::rans::LiteralEncoder64<VTP(vec)>*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec))
172+
sz += ESTSIZE(cc.firstChipROF, CTF::BLCfirstChipROF);
173+
sz += ESTSIZE(cc.bcIncROF, CTF::BLCbcIncROF );
174+
sz += ESTSIZE(cc.orbitIncROF, CTF::BLCorbitIncROF );
175+
sz += ESTSIZE(cc.nclusROF, CTF::BLCnclusROF );
176+
//
177+
sz += ESTSIZE(cc.chipInc, CTF::BLCchipInc );
178+
sz += ESTSIZE(cc.chipMul, CTF::BLCchipMul );
179+
sz += ESTSIZE(cc.row, CTF::BLCrow );
180+
sz += ESTSIZE(cc.colInc, CTF::BLCcolInc );
181+
sz += ESTSIZE(cc.pattID, CTF::BLCpattID );
182+
sz += ESTSIZE(cc.pattMap, CTF::BLCpattMap );
183+
184+
// clang-format on
185+
186+
LOG(INFO) << "Estimated output size is " << sz << " bytes";
187+
return sz;
188+
}

‎Detectors/TOF/reconstruction/include/TOFReconstruction/CTFCoder.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ class CTFCoder : public o2::ctf::CTFCoderBase
5050
private:
5151
/// compres compact clusters to CompressedInfos
5252
void compress(CompressedInfos& cc, const gsl::span<const ReadoutWindowData>& rofRecVec, const gsl::span<const Digit>& cdigVec, const gsl::span<const uint32_t>& pattVec);
53-
53+
size_t estimateCompressedSize(const CompressedInfos& cc);
5454
/// decompress CompressedInfos to compact clusters
5555
template <typename VROF, typename VDIG, typename VPAT>
5656
void decompress(const CompressedInfos& cc, VROF& rofRecVec, VDIG& cdigVec, VPAT& pattVec);
@@ -83,6 +83,10 @@ void CTFCoder::encode(VEC& buff, const gsl::span<const ReadoutWindowData>& rofRe
8383
};
8484
CompressedInfos cc;
8585
compress(cc, rofRecVec, cdigVec, pattVec);
86+
// book output size with some margin
87+
auto szIni = estimateCompressedSize(cc);
88+
buff.resize(szIni);
89+
8690
auto ec = CTF::create(buff);
8791
using ECB = CTF::base;
8892

‎Detectors/TOF/reconstruction/src/CTFCoder.cxx‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,3 +185,29 @@ void CTFCoder::createCoders(const std::string& dictPath, o2::ctf::CTFCoderBase::
185185
MAKECODER(cc.pattMap, CTF::BLCpattMap);
186186
// clang-format on
187187
}
188+
189+
///________________________________
190+
size_t CTFCoder::estimateCompressedSize(const CompressedInfos& cc)
191+
{
192+
size_t sz = 0;
193+
// clang-format off
194+
// RS FIXME this is very crude estimate, instead, an empirical values should be used
195+
#define VTP(vec) typename std::remove_reference<decltype(vec)>::type::value_type
196+
#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \
197+
rans::calculateMaxBufferSize(vec.size(), reinterpret_cast<const o2::rans::LiteralEncoder64<VTP(vec)>*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec))
198+
199+
sz += ESTSIZE(cc.bcIncROF, CTF::BLCbcIncROF);
200+
sz += ESTSIZE(cc.orbitIncROF, CTF::BLCorbitIncROF);
201+
sz += ESTSIZE(cc.ndigROF, CTF::BLCndigROF);
202+
sz += ESTSIZE(cc.ndiaROF, CTF::BLCndiaROF);
203+
sz += ESTSIZE(cc.timeFrameInc, CTF::BLCtimeFrameInc);
204+
sz += ESTSIZE(cc.timeTDCInc, CTF::BLCtimeTDCInc);
205+
sz += ESTSIZE(cc.stripID, CTF::BLCstripID);
206+
sz += ESTSIZE(cc.chanInStrip, CTF::BLCchanInStrip);
207+
sz += ESTSIZE(cc.tot, CTF::BLCtot);
208+
sz += ESTSIZE(cc.pattMap, CTF::BLCpattMap);
209+
// clang-format on
210+
211+
LOG(INFO) << "Estimated output size is " << sz << " bytes";
212+
return sz;
213+
}

‎Detectors/TPC/reconstruction/include/TPCReconstruction/CTFCoder.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ class CTFCoder : public o2::ctf::CTFCoderBase
4545
void decode(const CTF::base& ec, VEC& buff);
4646

4747
void createCoders(const std::string& dictPath, o2::ctf::CTFCoderBase::OpType op);
48+
size_t estimateCompressedSize(const CompressedClusters& ccl);
4849

4950
static size_t constexpr Alignment = 16;
5051
static size_t estimateSize(CompressedClusters& c);
@@ -103,6 +104,10 @@ void CTFCoder::encode(VEC& buff, const CompressedClusters& ccl)
103104
MD::EENCODE //nSliceRowClusters
104105
};
105106

107+
// book output size with some margin
108+
auto szIni = estimateCompressedSize(ccl);
109+
buff.resize(szIni);
110+
106111
auto ec = CTF::create(buff);
107112
ec->setHeader(reinterpret_cast<const CompressedClustersCounters&>(ccl));
108113
ec->getANSHeader().majorVersion = 0;

0 commit comments

Comments
 (0)