Skip to content

Commit 6a97524

Browse files
davidrohrdavidrohr
committed
GPU: Add yet another tpcCompression gather mode
1 parent 54d893a commit 6a97524

5 files changed

Lines changed: 55 additions & 13 deletions

File tree

‎GPU/GPUTracking/Base/GPUReconstruction.cxx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,9 @@ void* GPUReconstruction::AllocateVolatileDeviceMemory(size_t size)
601601
if (mVolatileMemoryStart == nullptr) {
602602
mVolatileMemoryStart = mDeviceMemoryPool;
603603
}
604+
if (size == 0) {
605+
return nullptr; // Future GPU memory allocation is volatile
606+
}
604607
char* retVal;
605608
GPUProcessor::computePointerWithAlignment(mDeviceMemoryPool, retVal, size);
606609
if (mDeviceMemoryPool > mDeviceMemoryPoolEnd) {

‎GPU/GPUTracking/Base/GPUSettingsList.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ AddOption(nStreams, int, 8, "", 0, "Number of GPU streams / command queues")
108108
AddOption(trackletConstructorInPipeline, int, -1, "", 0, "Run tracklet constructor in the pipeline")
109109
AddOption(trackletSelectorInPipeline, int, -1, "", 0, "Run tracklet selector in the pipeline")
110110
AddOption(mergerSortTracks, int, -1, "", 0, "Sort track indizes for GPU track fit")
111-
AddOption(tpcCompressionGatherMode, int, -1, "", 0, "TPC Compressed Clusters Gather Mode")
111+
AddOption(tpcCompressionGatherMode, int, -1, "", 0, "TPC Compressed Clusters Gather Mode (0: DMA transfer gather gpu to host, 1: serial DMA to host and gather by copy on CPU, 2. gather via GPU kernal DMA access, 3. gather on GPU via kernel, dma afterwards")
112112
AddOption(tpcCompressionGatherModeKernel, int, -1, "", 0, "TPC Compressed Clusters Gather Mode Kernel")
113113
AddOption(runMC, bool, false, "", 0, "Process MC labels")
114114
AddOption(ompKernels, bool, true, "", 0, "Parallelize with OMP inside kernels instead of over slices")

‎GPU/GPUTracking/DataCompression/GPUTPCCompression.cxx‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ using namespace GPUCA_NAMESPACE::gpu;
1919

2020
void GPUTPCCompression::InitializeProcessor() {}
2121

22+
void* GPUTPCCompression::SetPointersOutputGPU(void* mem)
23+
{
24+
SetPointersCompressedClusters(mem, *mOutputA, mOutputA->nAttachedClusters, mOutputA->nTracks, mOutputA->nUnattachedClusters, true);
25+
return mem;
26+
}
27+
2228
void* GPUTPCCompression::SetPointersOutputHost(void* mem)
2329
{
2430
computePointerWithoutAlignment(mem, mOutputFlat);
@@ -29,7 +35,7 @@ void* GPUTPCCompression::SetPointersOutputHost(void* mem)
2935
void* GPUTPCCompression::SetPointersScratch(void* mem)
3036
{
3137
computePointerWithAlignment(mem, mClusterStatus, mMaxClusters);
32-
if (mRec->GetProcessingSettings().tpcCompressionGatherMode == 2) {
38+
if (mRec->GetProcessingSettings().tpcCompressionGatherMode >= 2) {
3339
computePointerWithAlignment(mem, mAttachedClusterFirstIndex, mMaxTracks);
3440
}
3541
if (mRec->GetProcessingSettings().tpcCompressionGatherMode != 1) {
@@ -50,7 +56,7 @@ void* GPUTPCCompression::SetPointersOutput(void* mem)
5056
template <class T>
5157
void GPUTPCCompression::SetPointersCompressedClusters(void*& mem, T& c, unsigned int nClA, unsigned int nTr, unsigned int nClU, bool reducedClA)
5258
{
53-
computePointerWithAlignment(mem, c.qTotU, nClU);
59+
computePointerWithAlignment(mem, c.qTotU, nClU); // Do not reorder, qTotU ist used as first address in GPUChainTracking::RunTPCCompression
5460
computePointerWithAlignment(mem, c.qMaxU, nClU);
5561
computePointerWithAlignment(mem, c.flagsU, nClU);
5662
computePointerWithAlignment(mem, c.padDiffU, nClU);
@@ -87,17 +93,22 @@ void* GPUTPCCompression::SetPointersMemory(void* mem)
8793
{
8894
computePointerWithAlignment(mem, mMemory);
8995
computePointerWithAlignment(mem, mOutput);
96+
mOutputA = mOutput;
9097
return mem;
9198
}
9299

93100
void GPUTPCCompression::RegisterMemoryAllocation()
94101
{
95102
AllocateAndInitializeLate();
96103
mMemoryResOutputHost = mRec->RegisterMemoryAllocation(this, &GPUTPCCompression::SetPointersOutputHost, GPUMemoryResource::MEMORY_OUTPUT_FLAG | GPUMemoryResource::MEMORY_HOST | GPUMemoryResource::MEMORY_CUSTOM, "TPCCompressionOutputHost");
97-
if (mRec->GetProcessingSettings().tpcCompressionGatherMode != 2) {
98-
mRec->RegisterMemoryAllocation(this, &GPUTPCCompression::SetPointersOutput, GPUMemoryResource::MEMORY_OUTPUT | GPUMemoryResource::MEMORY_STACK, "TPCCompressionOutput");
104+
if (mRec->GetProcessingSettings().tpcCompressionGatherMode == 3) {
105+
mMemoryResOutputGPU = mRec->RegisterMemoryAllocation(this, &GPUTPCCompression::SetPointersOutputGPU, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_GPU | GPUMemoryResource::MEMORY_CUSTOM | GPUMemoryResource::MEMORY_STACK, "TPCCompressionOutputGPU");
106+
}
107+
unsigned int stackScratch = (mRec->GetProcessingSettings().tpcCompressionGatherMode != 3) ? GPUMemoryResource::MEMORY_STACK : 0;
108+
if (mRec->GetProcessingSettings().tpcCompressionGatherMode < 2) {
109+
mRec->RegisterMemoryAllocation(this, &GPUTPCCompression::SetPointersOutput, GPUMemoryResource::MEMORY_OUTPUT | stackScratch, "TPCCompressionOutput");
99110
}
100-
mRec->RegisterMemoryAllocation(this, &GPUTPCCompression::SetPointersScratch, GPUMemoryResource::MEMORY_SCRATCH | GPUMemoryResource::MEMORY_STACK, "TPCCompressionScratch");
111+
mRec->RegisterMemoryAllocation(this, &GPUTPCCompression::SetPointersScratch, GPUMemoryResource::MEMORY_SCRATCH | stackScratch, "TPCCompressionScratch");
101112
mRec->RegisterMemoryAllocation(this, &GPUTPCCompression::SetPointersMemory, GPUMemoryResource::MEMORY_PERMANENT, "TPCCompressionMemory");
102113
}
103114

‎GPU/GPUTracking/DataCompression/GPUTPCCompression.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ class GPUTPCCompression : public GPUProcessor
4949
void RegisterMemoryAllocation();
5050
void SetMaxData(const GPUTrackingInOutPointers& io);
5151

52+
void* SetPointersOutputGPU(void* mem);
5253
void* SetPointersOutputHost(void* mem);
5354
void* SetPointersOutputPtrs(void* mem);
5455
void* SetPointersOutput(void* mem);
@@ -79,6 +80,7 @@ class GPUTPCCompression : public GPUProcessor
7980

8081
o2::tpc::CompressedClustersPtrs mPtrs;
8182
o2::tpc::CompressedClusters* mOutput = nullptr;
83+
o2::tpc::CompressedClusters* mOutputA = nullptr; // Always points to host buffer
8284
o2::tpc::CompressedClustersFlat* mOutputFlat = nullptr;
8385

8486
memory* mMemory = nullptr;
@@ -96,6 +98,7 @@ class GPUTPCCompression : public GPUProcessor
9698
GPUd() static void truncateSignificantBits(T& val, unsigned int nBits, unsigned int max);
9799

98100
short mMemoryResOutputHost = -1;
101+
short mMemoryResOutputGPU = -1;
99102
};
100103

101104
template <class T>

‎GPU/GPUTracking/Global/GPUChainTracking.cxx‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,10 @@ bool GPUChainTracking::ValidateSettings()
306306
return false;
307307
}
308308
}
309+
if (!(GetRecoStepsGPU() & GPUDataTypes::RecoStep::TPCCompression) && (ProcessingSettings().tpcCompressionGatherMode == 1 || ProcessingSettings().tpcCompressionGatherMode == 3)) {
310+
GPUError("Invalid tpcCompressionGatherMode for compression on CPU");
311+
return false;
312+
}
309313
return true;
310314
}
311315

@@ -2096,6 +2100,10 @@ int GPUChainTracking::RunTPCCompression()
20962100
}
20972101
}
20982102
}
2103+
2104+
if (ProcessingSettings().tpcCompressionGatherMode == 3) {
2105+
mRec->AllocateVolatileDeviceMemory(0); // make future device memory allocation volatile
2106+
}
20992107
SetupGPUProcessor(&Compressor, true);
21002108
new (Compressor.mMemory) GPUTPCCompression::memory;
21012109

@@ -2124,6 +2132,12 @@ int GPUChainTracking::RunTPCCompression()
21242132
O->nComppressionModes = param().rec.tpcCompressionModes;
21252133
size_t outputSize = AllocateRegisteredMemory(Compressor.mMemoryResOutputHost, mOutputCompressedClusters);
21262134
Compressor.mOutputFlat->set(outputSize, *Compressor.mOutput);
2135+
void* hostFlatPtr = Compressor.mOutput->qTotU; // First array as allocated in GPUTPCCompression::SetPointersCompressedClusters
2136+
size_t copySize = 0;
2137+
if (ProcessingSettings().tpcCompressionGatherMode == 3) {
2138+
CompressorShadow.mOutputA = Compressor.mOutput;
2139+
copySize = AllocateRegisteredMemory(Compressor.mMemoryResOutputGPU); // We overwrite Compressor.mOutput with the allocated output pointers on the GPU
2140+
}
21272141
const o2::tpc::CompressedClustersPtrs* P = nullptr;
21282142
HighResTimer* gatherTimer = nullptr;
21292143
int outputStream = 0;
@@ -2132,12 +2146,14 @@ int GPUChainTracking::RunTPCCompression()
21322146
outputStream = mRec->NStreams() - 2;
21332147
}
21342148

2135-
if (ProcessingSettings().tpcCompressionGatherMode == 2) {
2136-
void* devicePtr = mRec->getGPUPointer(Compressor.mOutputFlat);
2137-
if (devicePtr != Compressor.mOutputFlat) {
2138-
CompressedClustersPtrs& ptrs = *Compressor.mOutput; // We need to update the ptrs with the gpu-mapped version of the host address space
2139-
for (unsigned int i = 0; i < sizeof(ptrs) / sizeof(void*); i++) {
2140-
reinterpret_cast<char**>(&ptrs)[i] = reinterpret_cast<char**>(&ptrs)[i] + (reinterpret_cast<char*>(devicePtr) - reinterpret_cast<char*>(Compressor.mOutputFlat));
2149+
if (ProcessingSettings().tpcCompressionGatherMode >= 2) {
2150+
if (ProcessingSettings().tpcCompressionGatherMode == 2) {
2151+
void* devicePtr = mRec->getGPUPointer(Compressor.mOutputFlat);
2152+
if (devicePtr != Compressor.mOutputFlat) {
2153+
CompressedClustersPtrs& ptrs = *Compressor.mOutput; // We need to update the ptrs with the gpu-mapped version of the host address space
2154+
for (unsigned int i = 0; i < sizeof(ptrs) / sizeof(void*); i++) {
2155+
reinterpret_cast<char**>(&ptrs)[i] = reinterpret_cast<char**>(&ptrs)[i] + (reinterpret_cast<char*>(devicePtr) - reinterpret_cast<char*>(Compressor.mOutputFlat));
2156+
}
21412157
}
21422158
}
21432159
TransferMemoryResourcesToGPU(myStep, &Compressor, outputStream);
@@ -2163,7 +2179,10 @@ int GPUChainTracking::RunTPCCompression()
21632179
GPUError("Invalid compression kernel selected.");
21642180
return 1;
21652181
}
2166-
2182+
if (ProcessingSettings().tpcCompressionGatherMode == 3) {
2183+
RecordMarker(&mEvents->stream[outputStream], outputStream);
2184+
GPUMemCpy(myStep, hostFlatPtr, Compressor.mOutput->qTotU, copySize, outputStream, false);
2185+
}
21672186
} else {
21682187
char direction = 0;
21692188
if (ProcessingSettings().tpcCompressionGatherMode == 0) {
@@ -2215,6 +2234,12 @@ int GPUChainTracking::RunTPCCompression()
22152234
gatherTimer->Stop();
22162235
}
22172236
mIOPtrs.tpcCompressedClusters = Compressor.mOutputFlat;
2237+
if (ProcessingSettings().tpcCompressionGatherMode == 3) {
2238+
SynchronizeEvents(&mEvents->stream[outputStream]);
2239+
ReleaseEvent(&mEvents->stream[outputStream]);
2240+
mRec->ReturnVolatileDeviceMemory();
2241+
}
2242+
22182243
if (mPipelineFinalizationCtx == nullptr) {
22192244
SynchronizeStream(outputStream);
22202245
} else {

0 commit comments

Comments
 (0)