Skip to content

Commit 67c6aac

Browse files
davidrohrdavidrohr
committed
GPU TPC QC: Add option to run on a fraction of the time frames and general improvements
1 parent 3841358 commit 67c6aac

8 files changed

Lines changed: 56 additions & 8 deletions

File tree

‎GPU/GPUTracking/Base/GPUReconstruction.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ int GPUReconstruction::InitPhaseBeforeDevice()
262262
mProcessingSettings.trackletSelectorSlices = 1;
263263
}
264264
}
265-
if (mProcessingSettings.createO2Output > 1 && mProcessingSettings.runQA) {
265+
if (mProcessingSettings.createO2Output > 1 && mProcessingSettings.runQA && mProcessingSettings.qcRunFraction == 100.f) {
266266
mProcessingSettings.createO2Output = 1;
267267
}
268268
if (!mProcessingSettings.createO2Output || !IsGPU()) {

‎GPU/GPUTracking/Definitions/GPUSettingsList.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ AddOption(doublePipelineClusterizer, bool, true, "", 0, "Include the input data
166166
AddOption(prefetchTPCpageScan, char, 0, "", 0, "Prefetch Data for TPC page scan in CPU cache")
167167
AddOption(runMC, bool, false, "", 0, "Process MC labels")
168168
AddOption(runQA, int, 0, "qa", 'q', "Enable tracking QA (negative number to provide bitmask for QA tasks)", message("Running QA: %s"), def(1))
169+
AddOption(qcRunFraction, float, 100.f, "", 0, "Percentage of events to process with QC")
169170
AddOption(outputSharedClusterMap, bool, false, "", 0, "Ship optional shared cluster map as output for further use")
170171
AddOption(disableTPCNoisyPadFilter, bool, false, "", 0, "Disables all TPC noisy pad filters (Not the normal noise filter!)")
171172
AddOption(createO2Output, char, 2, "", 0, "Create Track output in O2 format (2 = skip non-O2 output in GPU track format (reverts to =1 if QA is requested))")

‎GPU/GPUTracking/Global/GPUChainTracking.cxx‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,9 @@ int GPUChainTracking::RunChain()
612612
return 1;
613613
}
614614
}
615+
if (needQA && GetProcessingSettings().qcRunFraction != 100.f) {
616+
mFractionalQAEnabled = (rand() % 10000) < (unsigned int)(GetProcessingSettings().qcRunFraction * 100);
617+
}
615618
if (GetProcessingSettings().debugLevel >= 6) {
616619
*mDebugFile << "\n\nProcessing event " << mRec->getNEventsProcessed() << std::endl;
617620
}
@@ -716,10 +719,13 @@ int GPUChainTracking::RunChainFinalize()
716719
#endif
717720

718721
const bool needQA = GPUQA::QAAvailable() && (GetProcessingSettings().runQA || (GetProcessingSettings().eventDisplay && mIOPtrs.nMCInfosTPC));
719-
if (needQA) {
722+
if (needQA && (GetProcessingSettings().qcRunFraction == 100.f || mFractionalQAEnabled)) {
720723
mRec->getGeneralStepTimer(GeneralStep::QA).Start();
721724
mQA->RunQA(!GetProcessingSettings().runQA);
722725
mRec->getGeneralStepTimer(GeneralStep::QA).Stop();
726+
if (GetProcessingSettings().debugLevel == 0) {
727+
GPUInfo("Total QA runtime: %d us", (int)(mRec->getGeneralStepTimer(GeneralStep::QA).GetElapsedTime() * 1000000));
728+
}
723729
}
724730

725731
if (GetProcessingSettings().showOutputStat) {

‎GPU/GPUTracking/Global/GPUChainTracking.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ class GPUChainTracking : public GPUChain, GPUReconstructionHelpers::helperDelega
199199

200200
const GPUSettingsDisplay* mConfigDisplay = nullptr; // Abstract pointer to Standalone Display Configuration Structure
201201
const GPUSettingsQA* mConfigQA = nullptr; // Abstract pointer to Standalone QA Configuration Structure
202+
bool mFractionalQAEnabled = false;
202203

203204
protected:
204205
struct GPUTrackingFlatObjects : public GPUProcessor {

‎GPU/GPUTracking/Global/GPUChainTrackingMerger.cxx‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "GPUChainTracking.h"
1616
#include "GPULogging.h"
1717
#include "GPUO2DataTypes.h"
18+
#include "GPUQA.h"
1819
#include "utils/strtag.h"
1920
#include <fstream>
2021

@@ -225,6 +226,7 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
225226
}
226227
if (doGPU && !doGPUall) {
227228
TransferMemoryResourcesToHost(RecoStep::TPCMerging, &Merger, 0);
229+
GPUError("foo1");
228230
SynchronizeStream(0);
229231
}
230232

@@ -247,7 +249,15 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
247249
RecordMarker(&mEvents->single, 0);
248250
if (!GetProcessingSettings().fullMergerOnGPU) {
249251
TransferMemoryResourceLinkToHost(RecoStep::TPCMerging, Merger.MemoryResOutput(), outputStream, nullptr, &mEvents->single);
250-
} else if (GetProcessingSettings().keepDisplayMemory || GetProcessingSettings().createO2Output <= 1) {
252+
} else if (GetProcessingSettings().keepDisplayMemory || GetProcessingSettings().createO2Output <= 1 || mFractionalQAEnabled) {
253+
if (!(GetProcessingSettings().keepDisplayMemory || GetProcessingSettings().createO2Output <= 1)) {
254+
size_t size = mRec->Res(Merger.MemoryResOutput()).Size() + GPUCA_MEMALIGN;
255+
void* buffer = mQA->AllocateScratchBuffer(size);
256+
void* bufferEnd = Merger.SetPointersOutput(buffer);
257+
if ((char*)bufferEnd - (char*)buffer > size) {
258+
throw std::runtime_error("QA Scratch buffer exceeded");
259+
}
260+
}
251261
GPUMemCpy(RecoStep::TPCMerging, Merger.OutputTracks(), MergerShadowAll.OutputTracks(), Merger.NOutputTracks() * sizeof(*Merger.OutputTracks()), outputStream, 0, nullptr, &mEvents->single);
252262
if (param().par.dodEdx) {
253263
GPUMemCpy(RecoStep::TPCMerging, Merger.OutputTracksdEdx(), MergerShadowAll.OutputTracksdEdx(), Merger.NOutputTracks() * sizeof(*Merger.OutputTracksdEdx()), outputStream, 0, nullptr, &mEvents->single);
@@ -264,6 +274,7 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
264274
}
265275
if (GetProcessingSettings().keepDisplayMemory && !GetProcessingSettings().keepAllMemory) {
266276
TransferMemoryResourcesToHost(RecoStep::TPCMerging, &Merger, -1, true);
277+
GPUError("foo2");
267278
}
268279

269280
mRec->ReturnVolatileDeviceMemory();
@@ -296,6 +307,7 @@ int GPUChainTracking::RunTPCTrackingMerger(bool synchronizeOutput)
296307
if (GetProcessingSettings().runMC && mIOPtrs.clustersNative && mIOPtrs.clustersNative->clustersMCTruth) {
297308
AllocateRegisteredMemory(Merger.MemoryResOutputO2MC(), mSubOutputControls[GPUTrackingOutputs::getIndex(&GPUTrackingOutputs::tpcTracksO2Labels)]);
298309
TransferMemoryResourcesToHost(RecoStep::TPCMerging, &Merger, -1, true);
310+
GPUError("foo3");
299311
runKernel<GPUTPCGMO2Output, GPUTPCGMO2Output::mc>(GetGridAuto(0, GPUReconstruction::krnlDeviceType::CPU), krnlRunRangeNone, krnlEventNone);
300312
} else if (doGPUall) {
301313
RecordMarker(&mEvents->single, 0);

‎GPU/GPUTracking/qa/GPUQA.cxx‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -727,6 +727,10 @@ int GPUQA::InitQA(int tasks)
727727
mHist1Dd = new std::vector<TH1D>;
728728
mQATasks = tasks;
729729

730+
if (mTracking->GetProcessingSettings().qcRunFraction != 100.f && mQATasks != taskClusterCounts) {
731+
throw std::runtime_error("QA with qcRunFraction only supported for taskClusterCounts");
732+
}
733+
730734
if (mTracking) {
731735
mClNative = mTracking->mIOPtrs.clustersNative;
732736
}
@@ -1792,6 +1796,7 @@ void GPUQA::RunQA(bool matchOnly, const std::vector<o2::tpc::TrackTPC>* tracksEx
17921796
}
17931797
GPUInfo("Wrote %s,%d clusters in total, %d left, %d to be removed", fname, dumpClTot, dumpClLeft, dumpClRem);
17941798
}
1799+
mTrackingScratchBuffer.clear();
17951800
}
17961801

17971802
void GPUQA::GetName(char* fname, int k)
@@ -2718,15 +2723,22 @@ void GPUQA::PrintClusterCount(int mode, int& num, const char* name, unsigned lon
27182723
// do nothing, just count num
27192724
} else if (mode == 1) {
27202725
char name2[128];
2721-
sprintf(name2, "clusterCount%d", num);
2726+
sprintf(name2, "clusterCount%d_", num);
2727+
char* ptr = name2 + strlen(name2);
2728+
for (unsigned int i = 0; i < strlen(name); i++) {
2729+
if ((name[i] >= 'a' && name[i] <= 'z') || (name[i] >= 'A' && name[i] <= 'Z') || (name[i] >= '0' && name[i] <= '9')) {
2730+
*(ptr++) = name[i];
2731+
}
2732+
}
2733+
*ptr = 0;
27222734
createHist(mHistClusterCount[num], name2, name, 1000, 0, mConfig.histMaxNClusters, 1000, 0, 100);
27232735
} else if (mode == 0) {
27242736
if (normalization && mConfig.enableLocalOutput) {
27252737
printf("\t%35s: %'12llu (%6.2f%%)\n", name, n, 100.f * n / normalization);
27262738
}
27272739
if (mConfig.clusterRejectionHistograms) {
27282740
float ratio = 100.f * n / std::max(normalization, 1llu);
2729-
mHistClusterCount[num]->Fill(n, ratio, 1);
2741+
mHistClusterCount[num]->Fill(normalization, ratio, 1);
27302742
}
27312743
}
27322744
num++;
@@ -2748,6 +2760,7 @@ int GPUQA::DoClusterCounts(unsigned long long int* attachClusterCounts, int mode
27482760
PrintClusterCount(mode, num, "Protected", mClusterCounts.nProt, mClusterCounts.nTotal);
27492761
PrintClusterCount(mode, num, "Unattached", mClusterCounts.nUnattached, mClusterCounts.nTotal);
27502762
PrintClusterCount(mode, num, "Removed (Strategy A)", mClusterCounts.nTotal - mClusterCounts.nUnattached - mClusterCounts.nProt, mClusterCounts.nTotal);
2763+
PrintClusterCount(mode, num, "Removed (Strategy B)", mClusterCounts.nTotal - mClusterCounts.nProt, mClusterCounts.nTotal);
27512764
}
27522765

27532766
PrintClusterCount(mode, num, "Merged Loopers (Afterburner)", mClusterCounts.nMergedLooper, mClusterCounts.nTotal);
@@ -2768,3 +2781,9 @@ int GPUQA::DoClusterCounts(unsigned long long int* attachClusterCounts, int mode
27682781
}
27692782
return num;
27702783
}
2784+
2785+
void* GPUQA::AllocateScratchBuffer(size_t nBytes)
2786+
{
2787+
mTrackingScratchBuffer.resize((nBytes + sizeof(mTrackingScratchBuffer[0]) - 1) / sizeof(mTrackingScratchBuffer[0]));
2788+
return mTrackingScratchBuffer.data();
2789+
}

‎GPU/GPUTracking/qa/GPUQA.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ class GPUQA
5353
bool clusterRemovable(int attach, bool prot) const { return false; }
5454
void DumpO2MCData(const char* filename) const {}
5555
int ReadO2MCData(const char* filename) { return 1; }
56+
void* AllocateScratchBuffer(size_t nBytes) { return nullptr; }
5657
static bool QAAvailable() { return false; }
5758
static bool IsInitialized() { return false; }
5859
};
@@ -116,6 +117,7 @@ class GPUQA
116117
const std::vector<TH1D>& getHistograms1Dd() const { return *mHist1Dd; }
117118
void resetHists();
118119
int loadHistograms(std::vector<TH1F>& i1, std::vector<TH2F>& i2, std::vector<TH1D>& i3, int tasks = -1);
120+
void* AllocateScratchBuffer(size_t nBytes);
119121

120122
static constexpr int N_CLS_HIST = 8;
121123
static constexpr int N_CLS_TYPE = 3;
@@ -310,6 +312,8 @@ class GPUQA
310312
std::vector<std::vector<bool>> mGoodTracks;
311313
std::vector<std::vector<bool>> mGoodHits;
312314

315+
std::vector<unsigned long int> mTrackingScratchBuffer;
316+
313317
static std::vector<TColor*> mColors;
314318
static int initColors();
315319

‎prodtests/full-system-test/dpl-workflow.sh‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,13 @@ fi
107107

108108
has_processing_step ENTROPY_ENCODER && has_detector_ctf TPC && GPU_OUTPUT+=",compressed-clusters-ctf"
109109

110+
if workflow_has_parameter QC && has_detector_qc TPC; then
111+
GPU_OUTPUT+=",qa"
112+
[[ -z $TPC_TRACKING_QC_RUN_FRACTION ]] && TPC_TRACKING_QC_RUN_FRACTION=1
113+
GPU_CONFIG_KEY+="GPU_QA.clusterRejectionHistograms=1;GPU_proc.qcRunFraction=$TPC_TRACKING_QC_RUN_FRACTION;"
114+
[[ $HOSTMEMSIZE == "0" && $TPC_TRACKING_QC_RUN_FRACTION == "100" ]] && HOSTMEMSIZE=$(( 5 << 30 ))
115+
fi
116+
110117
if [[ -z $DISABLE_ROOT_OUTPUT ]]; then
111118
# enable only if root output is written, because it slows down the processing
112119
GPU_OUTPUT+=",send-clusters-per-sector"
@@ -150,9 +157,7 @@ fi
150157

151158
if [[ $GPUTYPE != "CPU" ]]; then
152159
GPU_CONFIG_KEY+="GPU_proc.forceMemoryPoolSize=$GPUMEMSIZE;"
153-
if [[ $HOSTMEMSIZE == "0" ]]; then
154-
HOSTMEMSIZE=$(( 1 << 30 ))
155-
fi
160+
[[ $HOSTMEMSIZE == "0" ]] && HOSTMEMSIZE=$(( 1 << 30 ))
156161
fi
157162

158163
if [[ $HOSTMEMSIZE != "0" ]]; then

0 commit comments

Comments
 (0)