TPC SCD calib: Ues time stamps for filenames and some fixes for merging slots

martenole · martenole · commit be9daac2fae4 · 2022-12-09T14:04:54.000+01:00
diff --git a/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/include/TPCInterpolationWorkflow/TPCResidualAggregatorSpec.h b/Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/include/TPCInterpolationWorkflow/TPCResidualAggregatorSpec.h
@@ -73,6 +73,8 @@ class ResidualAggregatorDevice : public o2::framework::Task
       storeMetaFile = false;
     }
 
+    LOGP(info, "Creating aggregator with {} entries per voxel minimum. Output file writing enabled: {}, meta file writing enabled: {}",
+         minEnt, writeOutput, storeMetaFile);
     mAggregator = std::make_unique<o2::tpc::ResidualAggregator>(minEnt);
     if (writeOutput) {
       mAggregator->setOutputDir(outputDir);
@@ -110,6 +112,10 @@ class ResidualAggregatorDevice : public o2::framework::Task
     updateTimeDependentParams(pc);
     std::chrono::duration<double, std::milli> ccdbUpdateTime = std::chrono::high_resolution_clock::now() - runStartTime;
 
+    // assume that the orbit reset time (given here in ms) can change within a run
+    auto orbitResetTime = o2::base::GRPGeomHelper::instance().getOrbitResetTimeMS();
+
+    // we always require the unbinned residuals and the associated track references
     auto residualsData = pc.inputs().get<gsl::span<o2::tpc::UnbinnedResid>>("unbinnedRes");
     auto trackRefs = pc.inputs().get<gsl::span<o2::tpc::TrackDataCompact>>("trackRefs");
 
@@ -133,7 +139,7 @@ class ResidualAggregatorDevice : public o2::framework::Task
 
     o2::base::TFIDInfoHelper::fillTFIDInfo(pc, mAggregator->getCurrentTFInfo());
     LOG(info) << "Processing TF " << mAggregator->getCurrentTFInfo().tfCounter << " with " << trkData->size() << " tracks and " << residualsData.size() << " unbinned residuals associated to them";
-    mAggregator->process(residualsData, trackRefs, trkDataPtr, lumi);
+    mAggregator->process(residualsData, trackRefs, orbitResetTime, trkDataPtr, lumi);
     std::chrono::duration<double, std::milli> runDuration = std::chrono::high_resolution_clock::now() - runStartTime;
     LOGP(info, "Duration for run method: {} ms. From this taken for time dependent param update: {} ms",
          std::chrono::duration_cast<std::chrono::milliseconds>(runDuration).count(),
diff --git a/Detectors/TPC/calibration/SpacePoints/include/SpacePoints/ResidualAggregator.h b/Detectors/TPC/calibration/SpacePoints/include/SpacePoints/ResidualAggregator.h
@@ -49,7 +49,7 @@ struct ResidualsContainer {
   void fillStatisticsBranches();
   uint64_t getNEntries() const { return nResidualsTotal; }
 
-  void fill(const o2::dataformats::TFIDInfo& ti, const gsl::span<const UnbinnedResid> resid, const gsl::span<const o2::tpc::TrackDataCompact> trkRefsIn, const gsl::span<const o2::tpc::TrackData>* trkDataIn, const o2::ctp::LumiInfo* lumiInput);
+  void fill(const o2::dataformats::TFIDInfo& ti, const gsl::span<const UnbinnedResid> resid, const gsl::span<const o2::tpc::TrackDataCompact> trkRefsIn, long orbitResetTime, const gsl::span<const o2::tpc::TrackData>* trkDataIn, const o2::ctp::LumiInfo* lumiInput);
   void merge(ResidualsContainer* prev);
   void print();
   void writeToFile(bool closeFileAfterwards);
@@ -59,7 +59,6 @@ struct ResidualsContainer {
   std::array<std::vector<TrackResiduals::LocalResid>*, SECTORSPERSIDE * SIDES> residualsPtr{};
   std::array<std::vector<TrackResiduals::VoxStats>, SECTORSPERSIDE * SIDES> stats{}; ///< voxel statistics sent to the aggregator
   std::array<std::vector<TrackResiduals::VoxStats>*, SECTORSPERSIDE * SIDES> statsPtr{};
-  uint32_t runNumber;                                                        ///< run number (required for meta data file)
   std::vector<uint32_t> tfOrbits, *tfOrbitsPtr{&tfOrbits};                   ///< first TF orbit
   std::vector<uint32_t> sumOfResiduals, *sumOfResidualsPtr{&sumOfResiduals}; ///< sum of residuals for each TF
   std::vector<o2::ctp::LumiInfo> lumi, *lumiPtr{&lumi};                      ///< luminosity information from CTP per TF
@@ -68,9 +67,6 @@ struct ResidualsContainer {
   std::vector<TrackDataCompact> trackInfo, *trackInfoPtr{&trackInfo};        ///< allows to obtain track type for each binned residual downstream
 
   std::string fileName{"o2tpc_residuals"};
-  std::string treeNameResiduals{"resid"};
-  std::string treeNameStats{"stats"};
-  std::string treeNameRecords{"records"};
   std::unique_ptr<TFile> fileOut{nullptr};
   std::unique_ptr<TTree> treeOutResidualsUnbinned{nullptr};
   std::unique_ptr<TTree> treeOutTrackData{nullptr};
@@ -86,11 +82,13 @@ struct ResidualsContainer {
   int autosaveInterval{0};            ///< if > 0, then the output written to file for every n-th TF
 
   // additional info
+  long orbitReset{0};                               ///< current orbit reset time in ms
+  uint32_t firstTForbit{0};                         ///< stored for the first seen TF to allow conversion to time stamp
   TFType firstSeenTF{o2::calibration::INFINITE_TF}; ///< the first TF which was added to this container
   TFType lastSeenTF{0};                             ///< the last TF which was added to this container
   uint64_t nResidualsTotal{0};                      ///< the total number of residuals for this container
 
-  ClassDefNV(ResidualsContainer, 4);
+  ClassDefNV(ResidualsContainer, 5);
 };
 
 class ResidualAggregator final : public o2::calibration::TimeSlotCalibration<ResidualsContainer>
diff --git a/Detectors/TPC/calibration/SpacePoints/src/ResidualAggregator.cxx b/Detectors/TPC/calibration/SpacePoints/src/ResidualAggregator.cxx
@@ -27,12 +27,14 @@ using namespace o2::tpc;
 ResidualsContainer::~ResidualsContainer()
 {
   // trees must be deleted before the file is closed, otherwise segfaults
+  LOGP(debug, "Deleting ResidualsContainer with {} entries. File name is {}", getNEntries(), fileName);
   treeOutResidualsUnbinned.reset();
   treeOutTrackData.reset();
   treeOutResiduals.reset();
   treeOutStats.reset();
   treeOutRecords.reset();
   if (fileOut) {
+    LOG(debug) << "Removing output file of discarded slot";
     // this slot was not finalized, need to close and remove the file
     fileOut->Close();
     fileOut.reset();
@@ -52,26 +54,30 @@ ResidualsContainer::ResidualsContainer(const ResidualsContainer& rhs)
 
 ResidualsContainer::ResidualsContainer(ResidualsContainer&& rhs)
 {
+  LOGP(debug, "Move operator called for rhs with {} entries", rhs.nResidualsTotal);
   trackResiduals = rhs.trackResiduals;
   fileOut = std::move(rhs.fileOut);
   fileName = std::move(rhs.fileName);
   treeOutResidualsUnbinned = std::move(rhs.treeOutResidualsUnbinned);
   treeOutTrackData = std::move(rhs.treeOutTrackData);
   treeOutResiduals = std::move(rhs.treeOutResiduals);
   treeOutStats = std::move(rhs.treeOutStats);
+  treeOutRecords = std::move(rhs.treeOutRecords);
   for (int iSec = 0; iSec < SECTORSPERSIDE * SIDES; ++iSec) {
     residuals[iSec] = std::move(rhs.residuals[iSec]);
     stats[iSec] = std::move(rhs.stats[iSec]);
   }
-  runNumber = rhs.runNumber;
   tfOrbits = std::move(rhs.tfOrbits);
   sumOfResiduals = std::move(rhs.sumOfResiduals);
   lumi = std::move(rhs.lumi);
   unbinnedRes = std::move(rhs.unbinnedRes);
   trackInfo = std::move(rhs.trackInfo);
   trkData = std::move(rhs.trkData);
+  orbitReset = rhs.orbitReset;
+  firstTForbit = rhs.firstTForbit;
   firstSeenTF = rhs.firstSeenTF;
   lastSeenTF = rhs.lastSeenTF;
+  nResidualsTotal = rhs.nResidualsTotal;
 }
 
 void ResidualsContainer::init(const TrackResiduals* residualsEngine, std::string outputDir, bool wFile, bool wBinnedResid, bool wUnbinnedResid, bool wTrackData, int autosave, int compression)
@@ -98,9 +104,9 @@ void ResidualsContainer::init(const TrackResiduals* residualsEngine, std::string
     treeOutTrackData->Branch("trk", &trkDataPtr);
   }
   if (writeBinnedResid) {
-    treeOutResiduals = std::make_unique<TTree>(treeNameResiduals.c_str(), "TPC binned residuals");
-    treeOutStats = std::make_unique<TTree>(treeNameStats.c_str(), "Voxel statistics mean position and nEntries");
-    treeOutRecords = std::make_unique<TTree>(treeNameRecords.c_str(), "Statistics per TF slot");
+    treeOutResiduals = std::make_unique<TTree>("resid", "TPC binned residuals");
+    treeOutStats = std::make_unique<TTree>("stats", "Voxel statistics mean position and nEntries");
+    treeOutRecords = std::make_unique<TTree>("records", "Statistics per TF slot");
     for (int iSec = 0; iSec < SECTORSPERSIDE * SIDES; ++iSec) {
       residualsPtr[iSec] = &residuals[iSec];
       statsPtr[iSec] = &stats[iSec];
@@ -122,6 +128,7 @@ void ResidualsContainer::init(const TrackResiduals* residualsEngine, std::string
     treeOutRecords->Branch("sumOfResiduals", &sumOfResidualsPtr);
     treeOutRecords->Branch("lumi", &lumiPtr);
   }
+  LOG(debug) << "Done initializing residuals container for file named " << fileName;
 }
 
 void ResidualsContainer::fillStatisticsBranches()
@@ -135,7 +142,7 @@ void ResidualsContainer::fillStatisticsBranches()
   }
 }
 
-void ResidualsContainer::fill(const o2::dataformats::TFIDInfo& ti, const gsl::span<const UnbinnedResid> resid, const gsl::span<const o2::tpc::TrackDataCompact> trkRefsIn, const gsl::span<const o2::tpc::TrackData>* trkDataIn, const o2::ctp::LumiInfo* lumiInput)
+void ResidualsContainer::fill(const o2::dataformats::TFIDInfo& ti, const gsl::span<const UnbinnedResid> resid, const gsl::span<const o2::tpc::TrackDataCompact> trkRefsIn, long orbitResetTime, const gsl::span<const o2::tpc::TrackData>* trkDataIn, const o2::ctp::LumiInfo* lumiInput)
 {
   // receives large vector of unbinned residuals and fills the sector-wise vectors
   // with binned residuals and statistics
@@ -145,6 +152,8 @@ void ResidualsContainer::fill(const o2::dataformats::TFIDInfo& ti, const gsl::sp
     lastSeenTF = ti.tfCounter;
   }
   if (ti.tfCounter < firstSeenTF) {
+    orbitReset = orbitResetTime;
+    firstTForbit = ti.firstTForbit;
     firstSeenTF = ti.tfCounter;
   }
   for (const auto& residIn : resid) {
@@ -206,7 +215,6 @@ void ResidualsContainer::fill(const o2::dataformats::TFIDInfo& ti, const gsl::sp
     treeOutResidualsUnbinned->Fill();
     unbinnedRes.clear();
   }
-  runNumber = ti.runNumber;
   tfOrbits.push_back(ti.firstTForbit);
   if (lumiInput) {
     lumi.push_back(*lumiInput);
@@ -255,7 +263,7 @@ void ResidualsContainer::merge(ResidualsContainer* prev)
 {
   // the previous slot is merged to this one and afterwards
   // the previous one will be deleted
-  LOG(info) << "Merging previous slot into current one";
+  LOGP(debug, "Merging previous slot with {} entries into current one with {} entries", prev->getNEntries(), getNEntries());
   if (writeBinnedResid) {
     for (int iSec = 0; iSec < SECTORSPERSIDE * SIDES; ++iSec) {
       // merge statistics
@@ -275,8 +283,9 @@ void ResidualsContainer::merge(ResidualsContainer* prev)
         stat.nEntries += statPrev.nEntries;
       }
       // prepare merging of residuals
-      prev->treeOutResiduals->SetBranchAddress(Form("sec%d", iSec), &residualsPtr);
+      prev->treeOutResiduals->SetBranchAddress(Form("sec%d", iSec), &residualsPtr[iSec]);
     }
+    prev->treeOutResiduals->SetBranchAddress("trackInfo", &trackInfoPtr);
     // We append the entries of the tree of the following slot to the
     // previous slot and afterwards move the merged tree to this slot.
     // This way the order of the entries is preserved
@@ -305,6 +314,12 @@ void ResidualsContainer::merge(ResidualsContainer* prev)
   treeOutTrackData = std::move(prev->treeOutTrackData);
   treeOutResidualsUnbinned = std::move(prev->treeOutResidualsUnbinned);
 
+  // since we want to continue using the TTrees of the previous slot, we must
+  // avoid that ROOT deletes them when the TFile of the previous slot is erased
+  treeOutResiduals->SetDirectory(fileOut.get());
+  treeOutTrackData->SetDirectory(fileOut.get());
+  treeOutResidualsUnbinned->SetDirectory(fileOut.get());
+
   nResidualsTotal += prev->nResidualsTotal;
 
   // append the current vector to the vector of the previous container and afterwards swap them,
@@ -317,6 +332,7 @@ void ResidualsContainer::merge(ResidualsContainer* prev)
   std::swap(prev->lumi, lumi);
 
   firstSeenTF = prev->firstSeenTF;
+  LOGP(debug, "Done with the merge. Current slot has {} entries", getNEntries());
 }
 
 void ResidualsContainer::print()
@@ -334,8 +350,9 @@ ResidualAggregator::~ResidualAggregator()
 
 bool ResidualAggregator::hasEnoughData(const Slot& slot) const
 {
-  LOG(debug) << "There are " << slot.getContainer()->getNEntries() << " entries currently. Min entries prt voxel: " << mMinEntries;
+  LOG(debug) << "There are " << slot.getContainer()->getNEntries() << " entries currently. Min entries per voxel: " << mMinEntries;
   auto entriesPerVoxel = slot.getContainer()->getNEntries() / (mTrackResiduals.getNVoxelsPerSector() * SECTORSPERSIDE * SIDES);
+  LOGP(debug, "Slot has {} entries per voxel, at least {} are required", entriesPerVoxel, mMinEntries);
   return entriesPerVoxel >= mMinEntries;
 }
 
@@ -350,13 +367,17 @@ void ResidualAggregator::finalizeSlot(Slot& slot)
   auto finalizeStartTime = std::chrono::high_resolution_clock::now();
   auto cont = slot.getContainer();
   cont->print();
-  if (!mWriteOutput) {
-    LOG(info) << "Skip writing output, since file output is disabled";
+  if (!mWriteOutput || cont->getNEntries() == 0) {
+    LOGP(info, "Skip writing output with {} entries, since file output is disabled or slot is empty", cont->getNEntries());
     return;
   }
   cont->writeToFile(true);
 
-  auto fileName = fmt::format("o2tpc_residuals_{}_{}_{}_{}.root", slot.getTFStart(), slot.getTFEnd(), cont->firstSeenTF, cont->lastSeenTF);
+  long orbitOffsetStart = (cont->firstSeenTF - slot.getTFStart()) * o2::base::GRPGeomHelper::getNHBFPerTF();
+  long orbitOffsetEnd = (slot.getTFEnd() - cont->firstSeenTF) * o2::base::GRPGeomHelper::getNHBFPerTF();
+  long timeStartMS = cont->orbitReset + (cont->firstTForbit - orbitOffsetStart) * o2::constants::lhc::LHCOrbitMUS * 1.e-3;
+  long timeEndMS = cont->orbitReset + (cont->firstTForbit + orbitOffsetEnd) * o2::constants::lhc::LHCOrbitMUS * 1.e-3;
+  auto fileName = fmt::format("o2tpc_residuals_{}_{}_{}_{}.root", timeStartMS, timeEndMS, slot.getTFStart(), slot.getTFEnd());
   auto fileNameWithPath = mOutputDir + fileName;
   std::filesystem::rename(o2::utils::Str::concat_string(mOutputDir, cont->fileName, ".part"), fileNameWithPath);
   if (mStoreMetaData) {
