P4 to Git Change 1809277 by gandryey@gera-win10 on 2019/06/11 17:34:13

SWDEV-180872 - Runtime support changes for Cooperative Group Features
	- Initial implementation of the core functionality. Disabled by default. Use GPU_ENABLE_COOP_GROUPS=1 to enable the feature.
	- Runtime uses device queue for cooperative executions with a synchronization on the launched queue.
	- The current implementation is pure runtime change and it can work if only one app uses this feature. No ROCr/KFD support was added or tested
	- Only inline assembler was tested

Affected files ...

... //depot/stg/opencl/drivers/opencl/api/hip/hip_device.cpp#20 edit
... //depot/stg/opencl/drivers/opencl/api/hip/hip_device_runtime.cpp#15 edit
... //depot/stg/opencl/drivers/opencl/api/hip/hip_hcc.def.in#15 edit
... //depot/stg/opencl/drivers/opencl/api/hip/hip_hcc.map.in#17 edit
... //depot/stg/opencl/drivers/opencl/api/hip/hip_module.cpp#28 edit
... //depot/stg/opencl/drivers/opencl/api/hip/hip_platform.cpp#32 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#338 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpudevice.cpp#606 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpudevice.hpp#171 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palblit.cpp#31 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palblit.hpp#9 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#142 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#39 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palschedcl.cpp#6 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#135 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#61 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.cpp#32 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.hpp#12 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#127 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#37 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocschedcl.cpp#3 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#75 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.hpp#23 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#94 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.hpp#92 edit
... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#311 edit

rocclr/runtime/device/rocm/rocdevice.cpp

@@ -148,6 +148,7 @@ Device::Device(hsa_agent_t bkendDevice)
     , pro_device_(nullptr)
     , pro_ena_(false)
     , freeMem_(0)
+    , hsa_exclusive_gpu_access_(false)
     , numOfVgpus_(0) {
   group_segment_.handle = 0;
   system_segment_.handle = 0;
@@ -585,6 +586,7 @@ bool Device::init() {
 }
 
 extern const char* SchedulerSourceCode;
+extern const char* GwsInitSourceCode;
 
 void Device::tearDown() {
   NullDevice::tearDown();
@@ -648,6 +650,9 @@ bool Device::create(bool sramEccEnabled) {
 #if defined(WITH_LIGHTNING_COMPILER) || defined(USE_COMGR_LIBRARY)
   std::string sch = SchedulerSourceCode;
   if (settings().useLightning_) {
+    if (info().cooperativeGroups_) {
+      sch.append(GwsInitSourceCode);
+    }
     scheduler = sch.c_str();
   }
 #ifndef USE_COMGR_LIBRARY
@@ -781,6 +786,38 @@ device::Program* NullDevice::createProgram(amd::option::Options* options) {
   return program;
 }
 
+bool Device::AcquireExclusiveGpuAccess() {
+  // Lock the virtual GPU list
+  vgpusAccess().lock();
+
+  // Find all available virtual GPUs and lock them
+  // from the execution of commands
+  for (uint idx = 0; idx < vgpus().size(); ++idx) {
+    vgpus()[idx]->execution().lock();
+    // Make sure a wait is done
+    vgpus()[idx]->releaseGpuMemoryFence();
+  }
+  if (!hsa_exclusive_gpu_access_) {
+    // @todo call rocr
+    hsa_exclusive_gpu_access_ = true;
+  }
+  return true;
+}
+
+void Device::ReleaseExclusiveGpuAccess(VirtualGPU& vgpu) const {
+  // Make sure the operation is done
+  vgpu.releaseGpuMemoryFence();
+
+  // Find all available virtual GPUs and unlock them
+  // for the execution of commands
+  for (uint idx = 0; idx < vgpus().size(); ++idx) {
+    vgpus()[idx]->execution().unlock();
+  }
+
+  // Unock the virtual GPU list
+  vgpusAccess().unlock();
+}
+
 device::Program* Device::createProgram(amd::option::Options* options) {
   device::Program* program;
   if (settings().useLightning_) {
@@ -1339,6 +1376,8 @@ bool Device::populateOCLDeviceConstants() {
     //TODO: set to true once thread trace support is available
     info_.threadTraceEnable_ = false;
     info_.pcieDeviceId_ = deviceInfo_.pciDeviceId_;
+    info_.cooperativeGroups_ = GPU_ENABLE_COOP_GROUPS;
+    info_.cooperativeMultiDeviceGroups_ = GPU_ENABLE_COOP_GROUPS;
   }
 
   info_.maxPipePacketSize_ = info_.maxMemAllocSize_;
@@ -1356,8 +1395,12 @@ bool Device::populateOCLDeviceConstants() {
 }
 
 device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) {
+  amd::ScopedLock lock(vgpusAccess());
+
   bool profiling = (queue != nullptr) && queue->properties().test(CL_QUEUE_PROFILING_ENABLE);
 
+  profiling |= (queue == nullptr) ? true : false;
+
   // Initialization of heap and other resources occur during the command
   // queue creation time.
   VirtualGPU* virtualDevice = new VirtualGPU(*this);