GPU: Add option to force size of GPU / Host memory pool

davidrohr · MichaelLettrich · commit deb0a781ce01 · 2019-07-24T11:15:47.000+02:00
diff --git a/GPU/GPUTracking/Base/GPUReconstruction.cxx b/GPU/GPUTracking/Base/GPUReconstruction.cxx
@@ -120,9 +120,18 @@ int GPUReconstruction::Init()
   mDeviceProcessingSettings.nThreads = 1;
 #endif
 
+  mDeviceMemorySize = mHostMemorySize = 0;
   for (unsigned int i = 0; i < mChains.size(); i++) {
     mChains[i]->RegisterPermanentMemoryAndProcessors();
+    size_t memGpu, memHost;
+    mChains[i]->MemorySize(memGpu, memHost);
+    mDeviceMemorySize += memGpu;
+    mHostMemorySize += memHost;
   }
+  if (mDeviceProcessingSettings.forceMemoryPoolSize) {
+    mDeviceMemorySize = mHostMemorySize = mDeviceProcessingSettings.forceMemoryPoolSize;
+  }
+
   for (unsigned int i = 0; i < mProcessors.size(); i++) {
     (mProcessors[i].proc->*(mProcessors[i].RegisterMemoryAllocation))();
   }
diff --git a/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx b/GPU/GPUTracking/Base/GPUReconstructionCPU.cxx
@@ -113,8 +113,10 @@ int GPUReconstructionCPU::GetThread()
 int GPUReconstructionCPU::InitDevice()
 {
   if (mDeviceProcessingSettings.memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {
-    mHostMemoryPermanent = mHostMemoryBase = operator new(GPUCA_HOST_MEMORY_SIZE);
-    mHostMemorySize = GPUCA_HOST_MEMORY_SIZE;
+    if (mDeviceMemorySize > mHostMemorySize) {
+      mHostMemorySize = mDeviceMemorySize;
+    }
+    mHostMemoryPermanent = mHostMemoryBase = operator new(mHostMemorySize);
     ClearAllocatedMemory();
   }
   SetThreadCounts();
diff --git a/GPU/GPUTracking/Base/GPUReconstructionDeviceBase.cxx b/GPU/GPUTracking/Base/GPUReconstructionDeviceBase.cxx
@@ -242,15 +242,6 @@ int GPUReconstructionDeviceBase::InitDevice()
     return (1);
   }
 
-  mDeviceMemorySize = 0;
-  mHostMemorySize = 0;
-  for (unsigned int i = 0; i < mChains.size(); i++) {
-    size_t memGpu, memHost;
-    mChains[i]->MemorySize(memGpu, memHost);
-    mDeviceMemorySize += memGpu;
-    mHostMemorySize += memHost;
-  }
-
   int retVal = InitDevice_Runtime();
   if (retVal) {
     GPUImportant("GPU Tracker initialization failed");
diff --git a/GPU/GPUTracking/Base/GPUSettings.cxx b/GPU/GPUTracking/Base/GPUSettings.cxx
@@ -79,4 +79,5 @@ void GPUSettingsDeviceProcessing::SetDefaults()
   nStreams = 8;
   trackletConstructorInPipeline = true;
   trackletSelectorInPipeline = false;
+  forceMemoryPoolSize = 0;
 }
diff --git a/GPU/GPUTracking/Base/GPUSettings.h b/GPU/GPUTracking/Base/GPUSettings.h
@@ -134,6 +134,7 @@ struct GPUSettingsDeviceProcessing {
   int nStreams;                       // Number of parallel GPU streams
   bool trackletConstructorInPipeline; // Run tracklet constructor in pileline like the preceeding tasks instead of as one big block
   bool trackletSelectorInPipeline;    // Run tracklet selector in pipeline, requres also tracklet constructor in pipeline
+  size_t forceMemoryPoolSize;         // Override size of memory pool to be allocated on GPU / Host
 };
 } // namespace gpu
 } // namespace GPUCA_NAMESPACE
diff --git a/GPU/GPUTracking/Standalone/qconfigoptions.h b/GPU/GPUTracking/Standalone/qconfigoptions.h
@@ -129,6 +129,7 @@ AddOptionSet(nways, int, 1, "1Way", 0, "Use 3-way track-fit")
 AddOption(nwaysouter, bool, false, "OuterParam", 0, "Create OuterParam")
 AddOption(dzdr, float, 2.5f, "DzDr", 0, "Use dZ/dR search window instead of vertex window")
 AddOption(cont, bool, false, "continuous", 0, "Process continuous timeframe data")
+AddOption(forceMemorySize, unsigned long long int, 0, "forceMemorySize", 0, "Force size of allocated GPU / page locked host memory", min(0ull))
 AddOption(outputcontrolmem, unsigned long long int, 0, "outputMemory", 0, "Use predefined output buffer of this size", min(0ull), message("Using %lld bytes as output memory"))
 AddOption(affinity, int, -1, "cpuAffinity", 0, "Pin CPU affinity to this CPU core", min(-1), message("Setting affinity to restrict on CPU %d"))
 AddOption(fifo, bool, false, "fifoScheduler", 0, "Use FIFO realtime scheduler", message("Setting FIFO scheduler: %s"))
diff --git a/GPU/GPUTracking/Standalone/standalone.cxx b/GPU/GPUTracking/Standalone/standalone.cxx
@@ -285,6 +285,7 @@ int SetupReconstruction()
     devProc.nThreads = configStandalone.OMPThreads;
   }
   devProc.deviceNum = configStandalone.cudaDevice;
+  devProc.forceMemoryPoolSize = configStandalone.forceMemorySize;
   devProc.debugLevel = configStandalone.DebugLevel;
   devProc.runQA = configStandalone.qa;
   devProc.runCompressionStatistics = configStandalone.compressionStat;

Original file line number	Diff line number	Diff line change
`@@ -113,8 +113,10 @@ int GPUReconstructionCPU::GetThread()`
`113`	`113`	`int GPUReconstructionCPU::InitDevice()`
`114`	`114`	`{`
`115`	`115`	`if (mDeviceProcessingSettings.memoryAllocationStrategy == GPUMemoryResource::ALLOCATION_GLOBAL) {`
`116`		`- mHostMemoryPermanent = mHostMemoryBase = operator new(GPUCA_HOST_MEMORY_SIZE);`
`117`		`- mHostMemorySize = GPUCA_HOST_MEMORY_SIZE;`
	`116`	`+ if (mDeviceMemorySize > mHostMemorySize) {`
	`117`	`+ mHostMemorySize = mDeviceMemorySize;`
	`118`	`+ }`
	`119`	`+ mHostMemoryPermanent = mHostMemoryBase = operator new(mHostMemorySize);`
`118`	`120`	`ClearAllocatedMemory();`
`119`	`121`	`}`
`120`	`122`	`SetThreadCounts();`
Original file line number	Diff line number	Diff line change
`@@ -79,4 +79,5 @@ void GPUSettingsDeviceProcessing::SetDefaults()`
`79`	`79`	`nStreams = 8;`
`80`	`80`	`trackletConstructorInPipeline = true;`
`81`	`81`	`trackletSelectorInPipeline = false;`
	`82`	`+ forceMemoryPoolSize = 0;`
`82`	`83`	`}`
Original file line number	Diff line number	Diff line change
`@@ -285,6 +285,7 @@ int SetupReconstruction()`
`285`	`285`	`devProc.nThreads = configStandalone.OMPThreads;`
`286`	`286`	`}`
`287`	`287`	`devProc.deviceNum = configStandalone.cudaDevice;`
	`288`	`+ devProc.forceMemoryPoolSize = configStandalone.forceMemorySize;`
`288`	`289`	`devProc.debugLevel = configStandalone.DebugLevel;`
`289`	`290`	`devProc.runQA = configStandalone.qa;`
`290`	`291`	`devProc.runCompressionStatistics = configStandalone.compressionStat;`