P4 to Git Change 1367810 by gandryey@gera-w8 on 2017/01/31 19:02:27

SWDEV-112171 - [ROCm CQE][OCLonLC][QR][G] System hangs/Failures observed with few WF conf tests, due to CL#1364923
	- Make HSA copy call lock protected, due to a possible race condition and HW hangs in integer_ops long_math test

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.cpp#10 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#39 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#15 edit

rocclr/runtime/device/rocm/rocblit.cpp

@@ -616,6 +616,9 @@ bool DmaBlitManager::hsaCopy(
     bool                enableCopyRect,
     bool                flushDMA) const
 {
+    // todo integerops long_math test exposes
+    // a HW hang without lock protection. note: Runtime kernel copy works fine
+    amd::ScopedLock k(dev().hsaCopyOps());
     address src = reinterpret_cast<address>(srcMemory.getDeviceMemory());
     address dst = reinterpret_cast<address>(dstMemory.getDeviceMemory());
 
@@ -663,6 +666,8 @@ bool DmaBlitManager::hsaCopy(
 bool DmaBlitManager::hsaCopyStaged(
     const_address hostSrc, address hostDst, size_t size, address staging, bool hostToDev) const
 {
+    amd::ScopedLock k(dev().hsaCopyOps());
+
     // No allocation is necessary for Full Profile
     hsa_status_t status;
     if (dev().agent_profile() == HSA_PROFILE_FULL) {

rocclr/runtime/device/rocm/rocdevice.cpp

@@ -189,6 +189,7 @@ Device::Device(hsa_agent_t bkendDevice)
     , xferQueue_(nullptr)
     , xferRead_(nullptr)
     , xferWrite_(nullptr)
+    , hsaCopyOps_(nullptr)
     , numOfVgpus_(0)
 {
     group_segment_.handle = 0;
@@ -207,6 +208,7 @@ Device::~Device()
     }
     delete mapCache_;
     delete mapCacheOps_;
+    delete hsaCopyOps_;
 
     // Destroy temporary buffers for read/write
     delete xferRead_;
@@ -626,6 +628,11 @@ Device::create()
         return false;
     }
 
+    hsaCopyOps_ = new amd::Monitor("HSA copy Lock", true);
+    if (nullptr == hsaCopyOps_) {
+        return false;
+    }
+
     mapCache_ = new std::vector<amd::Memory*>();
     if (mapCache_ == NULL) {
         return false;

rocclr/runtime/device/rocm/rocdevice.hpp

@@ -411,6 +411,8 @@ public:
         amd::Memory* mem    //!< Pointer to AMD memory object
         ) const;
 
+    amd::Monitor& hsaCopyOps() const { return *hsaCopyOps_; }
+
 private:
     static hsa_ven_amd_loader_1_00_pfn_t amd_loader_ext_table;
 
@@ -439,6 +441,10 @@ private:
     XferBuffers*    xferRead_;      //!< Transfer buffers read
     XferBuffers*    xferWrite_;     //!< Transfer buffers write
 
+    //! todo it shouldn't require a lock proteciton,
+    //! but currently it causes a HW hang in MT environment
+    amd::Monitor*   hsaCopyOps_;    //!< HSA copy is protected under lock
+
 public:
     amd::Atomic<uint>   numOfVgpus_; //!< Virtual gpu unique index
 };  // class roc::Device