Skip to content

Commit 1b5d98d

Browse files
aphecetcheaphecetche
authored
Raw: make the output directory creation more robust. (#6458)
* Raw: make the output directory creation more robust. Turns out that `std::filesystem::create_directories` on some platforms/compilers does not return true even when actually creating the directory. * assertOutputDirectory: change logic and add comment * put the ending brace in the right place...
1 parent 907083a commit 1b5d98d

8 files changed

Lines changed: 63 additions & 71 deletions

File tree

‎Detectors/FIT/FDD/simulation/src/digit2raw.cxx‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,20 +104,13 @@ void digi2raw(const std::string& inpName, const std::string& outDir, bool filePe
104104
wr.useRDHVersion(rdhV);
105105
wr.setDontFillEmptyHBF(noEmptyHBF);
106106

107+
o2::raw::assertOutputDirectory(outDir);
108+
107109
std::string outDirName(outDir);
108110
if (outDirName.back() != '/') {
109111
outDirName += '/';
110112
}
111113

112-
// if needed, create output directory
113-
if (!std::filesystem::exists(outDirName)) {
114-
if (!std::filesystem::create_directories(outDirName)) {
115-
LOG(FATAL) << "could not create output directory " << outDirName;
116-
} else {
117-
LOG(INFO) << "created output directory " << outDirName;
118-
}
119-
}
120-
121114
m2r.readDigits(outDirName, inpName);
122115
wr.writeConfFile(wr.getOrigin().str, "RAWDATA", o2::utils::Str::concat_string(outDirName, wr.getOrigin().str, "raw.cfg"));
123116
//LOG(INFO)<<o2::utils::Str::concat_string(outDirName, wr.getOrigin().str)<<"\n";

‎Detectors/FIT/FT0/simulation/src/digi2raw.cxx‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,18 +106,12 @@ void digi2raw(const std::string& inpName, const std::string& outDir, int verbosi
106106
wr.useRDHVersion(rdhV);
107107
wr.setDontFillEmptyHBF(noEmptyHBF);
108108

109+
o2::raw::assertOutputDirectory(outDir);
110+
109111
std::string outDirName(outDir);
110112
if (outDirName.back() != '/') {
111113
outDirName += '/';
112114
}
113-
// if needed, create output directory
114-
if (!std::filesystem::exists(outDirName)) {
115-
if (!std::filesystem::create_directories(outDirName)) {
116-
LOG(FATAL) << "could not create output directory " << outDirName;
117-
} else {
118-
LOG(INFO) << "created output directory " << outDirName;
119-
}
120-
}
121115

122116
m2r.readDigits(outDirName, inpName);
123117
wr.writeConfFile(wr.getOrigin().str, "RAWDATA", o2::utils::Str::concat_string(outDirName, wr.getOrigin().str, "raw.cfg"));

‎Detectors/FIT/FV0/simulation/src/digit2raw.cxx‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,18 +106,12 @@ void digit2raw(const std::string& inpName, const std::string& outDir, int verbos
106106
wr.useRDHVersion(rdhV);
107107
wr.setDontFillEmptyHBF(noEmptyHBF);
108108

109+
o2::raw::assertOutputDirectory(outDir);
110+
109111
std::string outDirName(outDir);
110112
if (outDirName.back() != '/') {
111113
outDirName += '/';
112114
}
113-
// if needed, create output directory
114-
if (!std::filesystem::exists(outDirName)) {
115-
if (!std::filesystem::create_directories(outDirName)) {
116-
LOG(FATAL) << "could not create output directory " << outDirName;
117-
} else {
118-
LOG(INFO) << "created output directory " << outDirName;
119-
}
120-
}
121115

122116
m2r.convertDigitsToRaw(outDirName, inpName);
123117
wr.writeConfFile(wr.getOrigin().str, "RAWDATA", o2::utils::Str::concat_string(outDirName, wr.getOrigin().str, "raw.cfg"));

‎Detectors/Raw/include/DetectorsRaw/RawFileWriter.h‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -79,22 +79,22 @@ class RawFileWriter
7979
/// Single GBT link helper
8080
struct LinkData {
8181
static constexpr int MarginToFlush = 10 * sizeof(RDHAny); // flush superpage if free space left <= this margin
82-
RDHAny rdhCopy; // RDH with the running info of the last RDH seen
83-
IR updateIR; // IR at which new HBF needs to be created
84-
int lastRDHoffset = -1; // position of last RDH in the link buffer
85-
bool startOfRun = true; // to signal if this is the 1st HBF of the run or not
86-
uint8_t packetCounter = 0; // running counter
87-
uint16_t pageCnt = 0; // running counter
88-
LinkSubSpec_t subspec = 0; // subspec according to DataDistribution
89-
bool discardData = false; // discard data if true (e.g. desired max IR reached)
82+
RDHAny rdhCopy; // RDH with the running info of the last RDH seen
83+
IR updateIR; // IR at which new HBF needs to be created
84+
int lastRDHoffset = -1; // position of last RDH in the link buffer
85+
bool startOfRun = true; // to signal if this is the 1st HBF of the run or not
86+
uint8_t packetCounter = 0; // running counter
87+
uint16_t pageCnt = 0; // running counter
88+
LinkSubSpec_t subspec = 0; // subspec according to DataDistribution
89+
bool discardData = false; // discard data if true (e.g. desired max IR reached)
9090
//
9191
size_t nTFWritten = 0; // number of TFs written
9292
size_t nRDHWritten = 0; // number of RDHs written
9393
size_t nBytesWritten = 0; // number of bytes written
9494
//
95-
std::string fileName{}; // file name associated with this link
96-
std::vector<char> buffer; // buffer to accumulate superpage data
97-
RawFileWriter* writer = nullptr; // pointer on the parent writer
95+
std::string fileName{}; // file name associated with this link
96+
std::vector<char> buffer; // buffer to accumulate superpage data
97+
RawFileWriter* writer = nullptr; // pointer on the parent writer
9898

9999
PayloadCache cacheBuffer; // used for caching in case of async. data input
100100
std::unique_ptr<TTree> cacheTree; // tree to store the cache
@@ -141,7 +141,6 @@ class RawFileWriter
141141
nRDHWritten++;
142142
return pushBack(reinterpret_cast<const char*>(&rdh), sizeof(RDHAny), false);
143143
}
144-
145144
};
146145
//=====================================================================================
147146
// If addData was called with given IR for at least 1 link, then it should be called for all links, even it with empty payload
@@ -402,13 +401,13 @@ class RawFileWriter
402401
int mVerbosity = 0;
403402
o2::header::DataOrigin mOrigin = o2::header::gDataOriginInvalid;
404403
int mUseRDHVersion = RDHUtils::getVersion<o2::header::RAWDataHeader>(); // by default, use default version
405-
int mSuperPageSize = 1024 * 1024; // super page size
406-
bool mStartTFOnNewSPage = true; // every TF must start on a new SPage
407-
bool mDontFillEmptyHBF = false; // skipp adding empty HBFs (uness it must have TF flag)
408-
bool mAddSeparateHBFStopPage = true; // HBF stop is added on a separate CRU page
409-
bool mUseRDHStop = true; // detector uses STOP in RDH
410-
bool mCRUDetector = true; // Detector readout via CRU ( RORC if false)
411-
bool mApplyCarryOverToLastPage = false; // call CarryOver method also for last chunk and overwrite modified trailer
404+
int mSuperPageSize = 1024 * 1024; // super page size
405+
bool mStartTFOnNewSPage = true; // every TF must start on a new SPage
406+
bool mDontFillEmptyHBF = false; // skipp adding empty HBFs (uness it must have TF flag)
407+
bool mAddSeparateHBFStopPage = true; // HBF stop is added on a separate CRU page
408+
bool mUseRDHStop = true; // detector uses STOP in RDH
409+
bool mCRUDetector = true; // Detector readout via CRU ( RORC if false)
410+
bool mApplyCarryOverToLastPage = false; // call CarryOver method also for last chunk and overwrite modified trailer
412411

413412
//>> caching --------------
414413
bool mCachingStage = false; // signal that current data should be cached
@@ -425,7 +424,14 @@ class RawFileWriter
425424
bool mDoLazinessCheck = true;
426425

427426
ClassDefNV(RawFileWriter, 1);
428-
}; // namespace raw
427+
};
428+
429+
/** Ensure (i.e. create if needed) directory
430+
* @param outDirName : output path to be asserted
431+
*
432+
* Log a FATAL if the directory does not exist and can not be created
433+
*/
434+
void assertOutputDirectory(std::string_view outDirName);
429435

430436
} // namespace raw
431437
} // namespace o2

‎Detectors/Raw/src/RawFileWriter.cxx‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "DetectorsRaw/HBFUtils.h"
2323
#include "CommonConstants/Triggers.h"
2424
#include "Framework/Logger.h"
25+
#include <filesystem>
2526

2627
using namespace o2::raw;
2728
using IR = o2::InteractionRecord;
@@ -451,7 +452,7 @@ void RawFileWriter::LinkData::addPreformattedCRUPage(const gsl::span<char> data)
451452
throw std::runtime_error("preformatted payload exceeds max size");
452453
}
453454
if (int(buffer.size()) - lastRDHoffset > sizeof(RDHAny)) { // we must start from empty page
454-
addHBFPage(); // start new CRU page
455+
addHBFPage(); // start new CRU page
455456
}
456457
pushBack(&data[0], data.size());
457458
}
@@ -646,11 +647,11 @@ void RawFileWriter::LinkData::fillEmptyHBHs(const IR& ir, bool dataAdded)
646647
if (writer->mVerbosity > 2) {
647648
LOG(INFO) << "Adding HBF " << ir << " for " << describe();
648649
}
649-
closeHBFPage(); // close current HBF: add RDH with stop and update counters
650-
RDHUtils::setTriggerType(rdhCopy, 0); // reset to avoid any detector specific flags in the dummy HBFs
650+
closeHBFPage(); // close current HBF: add RDH with stop and update counters
651+
RDHUtils::setTriggerType(rdhCopy, 0); // reset to avoid any detector specific flags in the dummy HBFs
651652
writer->mHBFUtils.updateRDH<RDHAny>(rdhCopy, ir, false); // update HBF orbit/bc and trigger flags
652-
openHBFPage(rdhCopy); // open new HBF
653-
updateIR = ir + 1; // new Trigger in RORC detector will be generated at >= this IR
653+
openHBFPage(rdhCopy); // open new HBF
654+
updateIR = ir + 1; // new Trigger in RORC detector will be generated at >= this IR
654655
}
655656
}
656657

@@ -726,3 +727,23 @@ void RawFileWriter::DetLazinessCheck::completeLinks(RawFileWriter* wr, const IR&
726727
}
727728
clear();
728729
}
730+
731+
void o2::raw::assertOutputDirectory(std::string_view outDirName)
732+
{
733+
if (!std::filesystem::exists(outDirName)) {
734+
#if defined(__clang__)
735+
// clang `create_directories` implementation is misbehaving and can
736+
// return false even if the directory is actually successfully created
737+
// so we work around that "feature" by not checking the
738+
// return value at all but using a second call to `exists`
739+
std::filesystem::create_directories(outDirName);
740+
if (!std::filesystem::exists(outDirName)) {
741+
LOG(FATAL) << "could not create output directory " << outDirName;
742+
}
743+
#else
744+
if (!std::filesystem::create_directories(outDirName)) {
745+
LOG(FATAL) << "could not create output directory " << outDirName;
746+
}
747+
#endif
748+
}
749+
}

‎Detectors/TPC/workflow/src/convertDigitsToRawZS.cxx‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,7 @@ void convertDigitsToZSfinal(std::string_view digitsFile, std::string_view output
9797
// if needed, create output directory
9898
if (!std::filesystem::exists(outDir)) {
9999
if (createParentDir) {
100-
if (!std::filesystem::create_directories(outDir)) {
101-
LOG(FATAL) << "could not create output directory " << outDir;
102-
} else {
103-
LOG(INFO) << "created output directory " << outDir;
104-
}
100+
o2::raw::assertOutputDirectory(outDir);
105101
} else {
106102
LOGP(error, "Requested output directory '{}' does not exists, consider removing '-n'", outDir);
107103
exit(1);

‎Detectors/TRD/simulation/src/trap2raw.cxx‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -129,18 +129,12 @@ void trap2raw(const std::string& inpDigitsName, const std::string& inpTrackletsN
129129
wr.useRDHVersion(rdhV);
130130
wr.setDontFillEmptyHBF(noEmptyHBF);
131131

132+
o2::raw::assertOutputDirectory(outDir);
133+
132134
std::string outDirName(outDir);
133135
if (outDirName.back() != '/') {
134136
outDirName += '/';
135137
}
136-
// if needed, create output directory
137-
if (!std::filesystem::exists(outDirName)) {
138-
if (!std::filesystem::create_directories(outDirName)) {
139-
LOG(FATAL) << "could not create output directory " << outDirName;
140-
} else {
141-
LOG(INFO) << "created output directory " << outDirName;
142-
}
143-
}
144138

145139
mc2raw.setTrackletHCHeader(trackletHCHeader);
146140
LOG(info) << "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%;";

‎Detectors/ZDC/simulation/src/digi2raw.cxx‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -153,18 +153,12 @@ void digi2raw(const std::string& inpName, const std::string& outDir, int verbosi
153153
wr.setSuperPageSize(superPageSizeInB);
154154
wr.useRDHVersion(rdhV);
155155

156+
o2::raw::assertOutputDirectory(outDir);
157+
156158
std::string outDirName(outDir);
157159
if (outDirName.back() != '/') {
158160
outDirName += '/';
159161
}
160-
// if needed, create output directory
161-
if (!std::filesystem::exists(outDirName)) {
162-
if (!std::filesystem::create_directories(outDirName)) {
163-
LOG(FATAL) << "could not create output directory " << outDirName;
164-
} else {
165-
LOG(INFO) << "created output directory " << outDirName;
166-
}
167-
}
168162

169163
d2r.setModuleConfig(moduleConfig);
170164
d2r.setSimCondition(simCondition);

0 commit comments

Comments
 (0)