@@ -103,9 +103,6 @@ class CTFWriterSpec : public o2::framework::Task
103103 template <typename C>
104104 void storeDictionary (DetID det, CTFHeader& header);
105105 void storeDictionaries ();
106- void prepareDictionaryTreeAndFile (DetID det);
107- void closeDictionaryTreeAndFile (CTFHeader& header);
108- std::string dictionaryFileName (const std::string& detName = " " );
109106 void closeTFTreeAndFile ();
110107 void prepareTFTreeAndFile (const o2::header::DataHeader* dh);
111108 size_t estimateCTFSize (ProcessingContext& pc);
@@ -118,19 +115,21 @@ class CTFWriterSpec : public o2::framework::Task
118115 bool mFinalized = false ;
119116 bool mWriteCTF = true ;
120117 bool mCreateDict = false ;
121- bool mDictPerDetector = false ;
122118 bool mCreateRunEnvDir = true ;
123119 bool mStoreMetaFile = false ;
124120 int mVerbosity = 0 ;
125121 int mSaveDictAfter = 0 ; // if positive and mWriteCTF==true, save dictionary after each mSaveDictAfter TFs processed
126122 int mFlagMinDet = 1 ; // append list of detectors to LHC period if their number is <= mFlagMinDet
123+ uint32_t mPrevDictTimeStamp = 0 ; // timestamp of the previously stored dictionary
124+ uint32_t mDictTimeStamp = 0 ; // timestamp of the currently stored dictionary
127125 uint64_t mRun = 0 ;
128126 size_t mMinSize = 0 ; // if > 0, accumulate CTFs in the same tree until the total size exceeds this minimum
129127 size_t mMaxSize = 0 ; // if > MinSize, and accumulated size will exceed this value, stop accumulation (even if mMinSize is not reached)
130128 size_t mChkSize = 0 ; // if > 0 and fallback storage provided, reserve this size per CTF file in production on primary storage
131129 size_t mAccCTFSize = 0 ; // so far accumulated size (if any)
132130 size_t mCurrCTFSize = 0 ; // size of currently processed CTF
133131 size_t mNCTF = 0 ; // total number of CTFs written
132+ size_t mNCTFPrevDict = 0 ; // total number of CTFs used for previous dictionary version
134133 size_t mNAccCTF = 0 ; // total number of CTFs accumulated in the current file
135134 size_t mCTFAutoSave = 0 ; // if > 0, autosave after so many TFs
136135 size_t mNCTFFiles = 0 ; // total number of CTF files written
@@ -151,7 +150,6 @@ class CTFWriterSpec : public o2::framework::Task
151150 int mLockFD = -1 ;
152151 std::unique_ptr<TFile> mCTFFileOut ;
153152 std::unique_ptr<TTree> mCTFTreeOut ;
154- std::unique_ptr<o2::dataformats::FileMetaData> mCTFFileMetaData ;
155153
156154 std::unique_ptr<TFile> mDictFileOut ; // file to store dictionary
157155 std::unique_ptr<TTree> mDictTreeOut ; // tree to store dictionary
@@ -181,7 +179,6 @@ CTFWriterSpec::CTFWriterSpec(DetID::mask_t dm, uint64_t r, const std::string& ou
181179// ___________________________________________________________________
182180void CTFWriterSpec::init (InitContext& ic)
183181{
184- mDictPerDetector = ic.options ().get <bool >(" dict-per-det" );
185182 // auto outmode = ic.options().get<std::string>("output-type"); // RS FIXME once global/local options clash is solved, --output-type will become device option
186183 auto outmode = mOutputType ;
187184 if (outmode == " ctf" ) {
@@ -230,18 +227,11 @@ void CTFWriterSpec::init(InitContext& ic)
230227 o2::utils::createDirectoriesIfAbsent (LOCKFileDir);
231228
232229 if (mCreateDict ) { // make sure that there is no local dictonary
233- for (int id = 0 ; id < DetID::nDetectors; id++) {
234- DetID det (id);
235- if (isPresent (det)) {
236- auto dictName = dictionaryFileName (det.getName ());
237- if (std::filesystem::exists (dictName)) {
238- throw std::runtime_error (o2::utils::Str::concat_string (" CTF dictionary creation is requested but " , dictName, " already exists, remove it!" ));
239- }
240- if (!mDictPerDetector ) {
241- break ; // no point in checking further
242- }
243- }
230+ std::string dictFileName = fmt::format (" {}{}.root" , mDictDir , o2::base::NameConf::CTFDICT);
231+ if (std::filesystem::exists (dictFileName)) {
232+ throw std::runtime_error (o2::utils::Str::concat_string (" CTF dictionary creation is requested but " , dictFileName, " already exists, remove it!" ));
244233 }
234+ o2::utils::createDirectoriesIfAbsent (mDictDir );
245235 }
246236}
247237
@@ -269,7 +259,6 @@ size_t CTFWriterSpec::processDet(o2::framework::ProcessingContext& pc, DetID det
269259 if (!mHeaders [det]) { // store 1st header
270260 mHeaders [det] = ctfImage.cloneHeader ();
271261 auto & hb = *static_cast <o2::ctf::CTFDictHeader*>(mHeaders [det].get ());
272- hb.dictTimeStamp = uint32_t (std::time (nullptr ));
273262 hb.det = det;
274263 }
275264 for (int ib = 0 ; ib < C::getNBlocks (); ib++) {
@@ -291,28 +280,38 @@ size_t CTFWriterSpec::processDet(o2::framework::ProcessingContext& pc, DetID det
291280template <typename C>
292281void CTFWriterSpec::storeDictionary (DetID det, CTFHeader& header)
293282{
283+ // create vector whose data contains dictionary in CTF format (EncodedBlock)
294284 if (!isPresent (det) || !mFreqsAccumulation [det].size ()) {
295285 return ;
296286 }
297- prepareDictionaryTreeAndFile (det);
298- // create vector whose data contains dictionary in CTF format (EncodedBlock)
299287 auto dictBlocks = C::createDictionaryBlocks (mFreqsAccumulation [det], mFreqsMetaData [det]);
300288 auto & h = C::get (dictBlocks.data ())->getHeader ();
301289 h = *reinterpret_cast <typename std::remove_reference<decltype (h)>::type*>(mHeaders [det].get ());
302290 auto & hb = static_cast <o2::ctf::CTFDictHeader&>(h);
303291 hb = *static_cast <const o2::ctf::CTFDictHeader*>(mHeaders [det].get ());
292+ hb.dictTimeStamp = mDictTimeStamp ;
293+
294+ auto getFileName = [this , det, &hb](bool curr) {
295+ return fmt::format (" {}{}_{}_v{}.{}_{}_{}.root" , this ->mDictDir , o2::base::NameConf::CTFDICT, det.getName (), int (hb.majorVersion ), int (hb.minorVersion ),
296+ curr ? this ->mDictTimeStamp : this ->mPrevDictTimeStamp , curr ? this ->mNCTF : this ->mNCTFPrevDict );
297+ };
304298
305299 C::get (dictBlocks.data ())->print (o2::utils::Str::concat_string (" Storing dictionary for " , det.getName (), " : " ));
306- C::get (dictBlocks.data ())->appendToTree (*mDictTreeOut .get (), det.getName ()); // cast to EncodedBlock
307- // mFreqsAccumulation[det].clear();
308- // mFreqsMetaData[det].clear();
309- if (mDictPerDetector ) {
310- header.detectors .reset ();
300+ auto outName = getFileName (true );
301+ TFile flout (outName.c_str (), " recreate" );
302+ flout.WriteObject (&dictBlocks, o2::base::NameConf::CCDBOBJECT.data ());
303+ flout.WriteObject (&hb, fmt::format (" ctf_dict_header_{}" , det.getName ()).c_str ());
304+ flout.Close ();
305+ LOGP (info, " Saved {} with {} TFs to {}" , hb.asString (), mNCTF , outName);
306+ if (mPrevDictTimeStamp ) {
307+ auto outNamePrev = getFileName (false );
308+ if (std::filesystem::exists (outNamePrev)) {
309+ std::filesystem::remove (outNamePrev);
310+ LOGP (info, " Removed previous dictionary version {}" , outNamePrev);
311+ }
311312 }
313+ C::get (dictBlocks.data ())->appendToTree (*mDictTreeOut .get (), det.getName ()); // cast to EncodedBlock and attach to dictionaries tree
312314 header.detectors .set (det);
313- if (mDictPerDetector ) {
314- closeDictionaryTreeAndFile (header);
315- }
316315}
317316
318317// ___________________________________________________________________
@@ -495,16 +494,15 @@ void CTFWriterSpec::prepareTFTreeAndFile(const o2::header::DataHeader* dh)
495494 }
496495 if (mCreateRunEnvDir && !mEnvironmentID .empty ()) {
497496 ctfDir += fmt::format (" {}_{}/" , mEnvironmentID , mRun );
498- o2::utils::createDirectoriesIfAbsent (ctfDir);
499- LOG (info) << " Created {} directory for CTFs output" << ctfDir;
497+ if (!ctfDir.empty ()) {
498+ o2::utils::createDirectoriesIfAbsent (ctfDir);
499+ LOGP (info, " Created {} directory for CTFs output" , ctfDir);
500+ }
500501 }
501502 mCurrentCTFFileName = o2::base::NameConf::getCTFFileName (mRun , dh->firstTForbit , dh->tfCounter );
502503 mCurrentCTFFileNameFull = fmt::format (" {}{}" , ctfDir, mCurrentCTFFileName );
503504 mCTFFileOut .reset (TFile::Open (fmt::format (" {}{}" , mCurrentCTFFileNameFull , TMPFileEnding).c_str (), " recreate" )); // to prevent premature external usage, use temporary name
504505 mCTFTreeOut = std::make_unique<TTree>(std::string (o2::base::NameConf::CTFTREENAME).c_str (), " O2 CTF tree" );
505- if (mStoreMetaFile ) {
506- mCTFFileMetaData = std::make_unique<o2::dataformats::FileMetaData>();
507- }
508506
509507 mNCTFFiles ++;
510508 }
@@ -525,16 +523,17 @@ void CTFWriterSpec::closeTFTreeAndFile()
525523 }
526524 // write CTF file metaFile data
527525 if (mStoreMetaFile ) {
528- mCTFFileMetaData ->fillFileData (mCurrentCTFFileNameFull );
529- mCTFFileMetaData ->run = mRun ;
530- mCTFFileMetaData ->LHCPeriod = mLHCPeriod ;
531- mCTFFileMetaData ->type = " raw" ;
532- mCTFFileMetaData ->priority = " high" ;
526+ o2::dataformats::FileMetaData ctfMetaData;
527+ ctfMetaData.fillFileData (mCurrentCTFFileNameFull );
528+ ctfMetaData.run = mRun ;
529+ ctfMetaData.LHCPeriod = mLHCPeriod ;
530+ ctfMetaData.type = " raw" ;
531+ ctfMetaData.priority = " high" ;
533532 auto metaFileNameTmp = fmt::format (" {}{}.tmp" , mCTFMetaFileDir , mCurrentCTFFileName );
534533 auto metaFileName = fmt::format (" {}{}.done" , mCTFMetaFileDir , mCurrentCTFFileName );
535534 try {
536535 std::ofstream metaFileOut (metaFileNameTmp);
537- metaFileOut << * mCTFFileMetaData . get () ;
536+ metaFileOut << ctfMetaData ;
538537 metaFileOut << " TFOrbits: " ;
539538 for (size_t i = 0 ; i < mTFOrbits .size (); i++) {
540539 metaFileOut << fmt::format (" {}{}" , i ? " , " : " " , mTFOrbits [i]);
@@ -545,7 +544,6 @@ void CTFWriterSpec::closeTFTreeAndFile()
545544 } catch (std::exception const & e) {
546545 LOG (error) << " Failed to store CTF meta data file " << metaFileName << " , reason: " << e.what ();
547546 }
548- mCTFFileMetaData .reset ();
549547 }
550548 } catch (std::exception const & e) {
551549 LOG (error) << " Failed to finalize CTF file " << mCurrentCTFFileNameFull << " , reason: " << e.what ();
@@ -557,39 +555,18 @@ void CTFWriterSpec::closeTFTreeAndFile()
557555 }
558556}
559557
560- // ___________________________________________________________________
561- void CTFWriterSpec::prepareDictionaryTreeAndFile (DetID det)
562- {
563- if (mDictPerDetector ) {
564- if (mDictTreeOut ) {
565- mDictTreeOut ->SetEntries (1 );
566- mDictTreeOut ->Write ();
567- mDictTreeOut .reset ();
568- mDictFileOut .reset ();
569- }
570- }
571- if (!mDictTreeOut ) {
572- mDictFileOut .reset (TFile::Open (dictionaryFileName (det.getName ()).c_str (), " recreate" ));
573- mDictTreeOut = std::make_unique<TTree>(std::string (o2::base::NameConf::CTFDICT).c_str (), " O2 CTF dictionary" );
574- }
575- }
576-
577- // ___________________________________________________________________
578- std::string CTFWriterSpec::dictionaryFileName (const std::string& detName)
579- {
580- if (mDictPerDetector ) {
581- if (detName.empty ()) {
582- throw std::runtime_error (" Per-detector dictionary files are requested but detector name is not provided" );
583- }
584- return o2::utils::Str::concat_string (mDictDir , detName, ' _' , o2::base::NameConf::CTFDICT, " .root" );
585- } else {
586- return o2::utils::Str::concat_string (mDictDir , o2::base::NameConf::CTFDICT, " .root" );
587- }
588- }
589-
590558// ___________________________________________________________________
591559void CTFWriterSpec::storeDictionaries ()
592560{
561+ // monolitic dictionary in tree format
562+ mDictTimeStamp = uint32_t (std::time (nullptr ));
563+ auto getFileName = [this ](bool curr) {
564+ return fmt::format (" {}{}_{}_{}.root" , this ->mDictDir , o2::base::NameConf::CTFDICT, curr ? this ->mDictTimeStamp : this ->mPrevDictTimeStamp , curr ? this ->mNCTF : this ->mNCTFPrevDict );
565+ };
566+ auto dictFileName = getFileName (true );
567+ mDictFileOut .reset (TFile::Open (dictFileName.c_str (), " recreate" ));
568+ mDictTreeOut = std::make_unique<TTree>(std::string (o2::base::NameConf::CTFDICT).c_str (), " O2 CTF dictionary" );
569+
593570 CTFHeader header{mRun , uint32_t (mNCTF )};
594571 storeDictionary<o2::itsmft::CTF>(DetID::ITS, header);
595572 storeDictionary<o2::itsmft::CTF>(DetID::MFT, header);
@@ -607,25 +584,27 @@ void CTFWriterSpec::storeDictionaries()
607584 storeDictionary<o2::zdc::CTF>(DetID::ZDC, header);
608585 storeDictionary<o2::hmpid::CTF>(DetID::HMP, header);
609586 storeDictionary<o2::ctp::CTF>(DetID::CTP, header);
610-
611- // close remnants
612- if ( mDictTreeOut ) {
613- closeDictionaryTreeAndFile (header );
614- }
615- LOG (info) << " Saved CTF dictionary after " << mNCTF << " TFs processed " ;
616- }
617-
618- // ___________________________________________________________________
619- void CTFWriterSpec::closeDictionaryTreeAndFile (CTFHeader& header)
620- {
621- if ( mDictTreeOut ) {
622- mDictFileOut -> cd ();
623- appendToTree (* mDictTreeOut . get (), " CTFHeader " , header );
624- mDictTreeOut -> SetEntries ( 1 );
625- mDictTreeOut -> Write ( mDictTreeOut -> GetName (), TObject:: kSingleKey );
626- mDictTreeOut . reset ( );
627- mDictFileOut . reset ();
587+ mDictFileOut -> cd ();
588+ appendToTree (* mDictTreeOut . get (), " CTFHeader " , header);
589+ mDictTreeOut -> SetEntries ( 1 );
590+ mDictTreeOut -> Write ( mDictTreeOut -> GetName (), TObject:: kSingleKey );
591+ mDictTreeOut . reset ();
592+ mDictFileOut . reset () ;
593+ std::string dictFileNameLnk = fmt::format ( " {}{}.root " , mDictDir , o2::base::NameConf::CTFDICT);
594+ if ( std::filesystem::exists (dictFileNameLnk)) {
595+ std::filesystem::remove (dictFileNameLnk);
596+ }
597+ std::filesystem::create_symlink (dictFileName, dictFileNameLnk);
598+ LOGP (info, " Saved CTF dictionaries tree with {} TFs to {} and linked to {} " , mNCTF , dictFileName, dictFileNameLnk);
599+ if ( mPrevDictTimeStamp ) {
600+ auto dictFileNamePrev = getFileName ( false );
601+ if ( std::filesystem::exists (dictFileNamePrev)) {
602+ std::filesystem::remove (dictFileNamePrev );
603+ LOGP (info, " Removed previous dictionary version {} " , dictFileNamePrev );
604+ }
628605 }
606+ mNCTFPrevDict = mNCTF ;
607+ mPrevDictTimeStamp = mDictTimeStamp ;
629608}
630609
631610// ___________________________________________________________________
@@ -729,7 +708,6 @@ DataProcessorSpec getCTFWriterSpec(DetID::mask_t dets, uint64_t run, const std::
729708 {" save-ctf-after" , VariantType::Int, 0 , {" if > 0, autosave CTF tree with multiple CTFs after every N CTFs" }},
730709 {" save-dict-after" , VariantType::Int, 0 , {" if > 0, in dictionary generation mode save it dictionary after certain number of TFs processed" }},
731710 {" ctf-dict-dir" , VariantType::String, " none" , {" CTF dictionary directory, must exist" }},
732- {" dict-per-det" , VariantType::Bool, false , {" create dictionary file per detector" }},
733711 {" output-dir" , VariantType::String, " none" , {" CTF output directory, must exist" }},
734712 {" output-dir-alt" , VariantType::String, " /dev/null" , {" Alternative CTF output directory, must exist (if not /dev/null)" }},
735713 {" meta-output-dir" , VariantType::String, " /dev/null" , {" CTF metadata output directory, must exist (if not /dev/null)" }},
0 commit comments