Add option to write only fraction of TFs to CTF

shahor02 · davidrohr · commit 54930862153f · 2022-07-01T09:33:06.000+02:00
with --ctf-rejection &lt;N&gt; one can request only fraction of TFs written to the CTF:
N&gt;0 : percentage to reject randomly (if &gt;99: reject all)
N&lt;-1: store only 1 out |N| TFs (with timeslice%N==0)
diff --git a/Detectors/CTF/workflow/src/CTFWriterSpec.cxx b/Detectors/CTF/workflow/src/CTFWriterSpec.cxx
@@ -51,6 +51,7 @@
 #include <vector>
 #include <TFile.h>
 #include <TTree.h>
+#include <TRandom.h>
 #include <filesystem>
 #include <ctime>
 #include <sys/stat.h>
@@ -121,6 +122,7 @@ class CTFWriterSpec : public o2::framework::Task
   bool mCreateDict = false;
   bool mCreateRunEnvDir = true;
   bool mStoreMetaFile = false;
+  bool mRejectCurrentTF = false;
   int mReportInterval = -1;
   int mVerbosity = 0;
   int mSaveDictAfter = 0;          // if positive and mWriteCTF==true, save dictionary after each mSaveDictAfter TFs processed
@@ -137,6 +139,7 @@ class CTFWriterSpec : public o2::framework::Task
   size_t mCTFAutoSave = 0;           // if > 0, autosave after so many TFs
   size_t mNCTFFiles = 0;             // total number of CTF files written
   int mMaxCTFPerFile = 0;            // max CTFs per files to store
+  int mRejRate = 0;                  // CTF rejection rule (>0: percentage to reject randomly, <0: reject if timeslice%|value|!=0)
   std::vector<uint32_t> mTFOrbits{}; // 1st orbits of TF accumulated in current file
   o2::framework::DataTakingContext mDataTakingContext{};
   o2::framework::TimingInfo mTimingInfo{};
@@ -220,6 +223,13 @@ void CTFWriterSpec::init(InitContext& ic)
   mMinSize = ic.options().get<int64_t>("min-file-size");
   mMaxSize = ic.options().get<int64_t>("max-file-size");
   mMaxCTFPerFile = ic.options().get<int>("max-ctf-per-file");
+  mRejRate = ic.options().get<int>("ctf-rejection");
+  if (mRejRate > 0) {
+    LOGP(info, "Will reject{} {}% of TFs", mRejRate < 100 ? " randomly" : "", mRejRate < 100 ? mRejRate : 100);
+  } else if (mRejRate < -1) {
+    LOGP(info, "Will reject all but each {}-th TF slice", -mRejRate);
+  }
+
   if (mWriteCTF) {
     if (mMinSize > 0) {
       LOG(info) << "Multiple CTFs will be accumulated in the tree/file until its size exceeds " << mMinSize << " bytes";
@@ -269,7 +279,7 @@ size_t CTFWriterSpec::processDet(o2::framework::ProcessingContext& pc, DetID det
   auto ctfBuffer = pc.inputs().get<gsl::span<o2::ctf::BufferType>>(det.getName());
   const auto ctfImage = C::getImage(ctfBuffer.data());
   ctfImage.print(o2::utils::Str::concat_string(det.getName(), ": "), mVerbosity);
-  if (mWriteCTF) {
+  if (mWriteCTF && !mRejectCurrentTF) {
     sz = ctfImage.appendToTree(*tree, det.getName());
     header.detectors.set(det);
   } else {
@@ -374,12 +384,11 @@ void CTFWriterSpec::run(ProcessingContext& pc)
   auto cput = mTimer.CpuTime();
   mTimer.Start(false);
   updateTimeDependentParams(pc);
-
+  mRejectCurrentTF = (mRejRate > 0 && int(gRandom->Rndm() * 100) < mRejRate) || (mRejRate < -1 && mTimingInfo.timeslice % (-mRejRate));
   mCurrCTFSize = estimateCTFSize(pc);
-  if (mWriteCTF) {
+  if (mWriteCTF && !mRejectCurrentTF) {
     prepareTFTreeAndFile();
   }
-
   // create header
   CTFHeader header{mTimingInfo.runNumber, mTimingInfo.creation, mTimingInfo.firstTFOrbit, mTimingInfo.tfCounter};
   size_t szCTF = 0;
@@ -406,7 +415,7 @@ void CTFWriterSpec::run(ProcessingContext& pc)
 
   mTimer.Stop();
 
-  if (mWriteCTF) {
+  if (mWriteCTF && !mRejectCurrentTF) {
     szCTF += appendToTree(*mCTFTreeOut.get(), "CTFHeader", header);
     mAccCTFSize += szCTF;
     mCTFTreeOut->SetEntries(++mNAccCTF);
@@ -429,7 +438,7 @@ void CTFWriterSpec::run(ProcessingContext& pc)
       mCTFTreeOut->AutoSave("override");
     }
   } else {
-    LOG(info) << "TF#" << mNCTF << " CTF writing is disabled, size was " << szCTF << " bytes";
+    LOG(info) << "TF#" << mNCTF << " {" << header << "} CTF writing is disabled, size was " << szCTF << " bytes";
   }
 
   mNCTF++;
@@ -701,6 +710,7 @@ DataProcessorSpec getCTFWriterSpec(DetID::mask_t dets, uint64_t run, const std::
             {"min-file-size", VariantType::Int64, 0l, {"accumulate CTFs until given file size reached"}},
             {"max-file-size", VariantType::Int64, 0l, {"if > 0, try to avoid exceeding given file size, also used for space check"}},
             {"max-ctf-per-file", VariantType::Int, 0, {"if > 0, avoid storing more than requested CTFs per file"}},
+            {"ctf-rejection", VariantType::Int, 0, {">0: percentage to reject randomly, <0: reject if timeslice%|value|!=0"}},
             {"ignore-partition-run-dir", VariantType::Bool, false, {"Do not creare partition-run directory in output-dir"}}}};
 }
 
