P4 to Git Change 1476364 by gandryey@gera-w8 on 2017/10/30 14:40:10

SWDEV-79445 - Add logic to handle persistent memory separately from the host memory path
	- Some of the optimizations applied to the host memory can't be used for persistent

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocblit.cpp#20 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocmemory.cpp#30 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocmemory.hpp#10 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocvirtual.cpp#45 edit


[ROCm/clr commit: c2105d8f16]

projects/clr/rocclr/runtime/device/rocm/rocblit.cpp

@@ -197,7 +197,8 @@ bool DmaBlitManager::writeBuffer(const void* srcHost, device::Memory& dstMemory,
                                  const amd::Coord3D& origin, const amd::Coord3D& size,
                                  bool entire) const {
   // Use host copy if memory has direct access
-  if (setup_.disableWriteBuffer_ || dstMemory.isHostMemDirectAccess()) {
+  if (setup_.disableWriteBuffer_ || dstMemory.isHostMemDirectAccess() ||
+      gpuMem(dstMemory).IsPersistentDirectMap()) {
     return HostBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire);
   } else {
     size_t dstSize = size[0];
@@ -283,7 +284,8 @@ bool DmaBlitManager::writeBufferRect(const void* srcHost, device::Memory& dstMem
                                      const amd::BufferRect& bufRect, const amd::Coord3D& size,
                                      bool entire) const {
   // Use host copy if memory has direct access
-  if (setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess()) {
+  if (setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess() ||
+      gpuMem(dstMemory).IsPersistentDirectMap()) {
     return HostBlitManager::writeBufferRect(srcHost, dstMemory, hostRect, bufRect, size, entire);
   } else {
     Memory& xferBuf = dev().xferWrite().acquire();
@@ -1631,7 +1633,8 @@ bool KernelBlitManager::writeBuffer(const void* srcHost, device::Memory& dstMemo
   bool result = false;
 
   // Use host copy if memory has direct access
-  if (setup_.disableWriteBuffer_ || dstMemory.isHostMemDirectAccess()) {
+  if (setup_.disableWriteBuffer_ || dstMemory.isHostMemDirectAccess() ||
+      gpuMem(dstMemory).IsPersistentDirectMap()) {
     result = HostBlitManager::writeBuffer(srcHost, dstMemory, origin, size, entire);
     synchronize();
     return result;
@@ -1679,7 +1682,8 @@ bool KernelBlitManager::writeBufferRect(const void* srcHost, device::Memory& dst
   bool result = false;
 
   // Use host copy if memory has direct access
-  if (setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess()) {
+  if (setup_.disableWriteBufferRect_ || dstMemory.isHostMemDirectAccess() ||
+      gpuMem(dstMemory).IsPersistentDirectMap()) {
     result = HostBlitManager::writeBufferRect(srcHost, dstMemory, hostRect, bufRect, size, entire);
     synchronize();
     return result;

projects/clr/rocclr/runtime/device/rocm/rocmemory.cpp

@@ -33,6 +33,7 @@ Memory::Memory(const roc::Device& dev, amd::Memory& owner)
       deviceMemory_(nullptr),
       kind_(MEMORY_KIND_NORMAL),
       amdImageDesc_(nullptr),
+      persistent_host_ptr_(nullptr),
       pinnedMemory_(nullptr) {}
 
 Memory::Memory(const roc::Device& dev, size_t size)
@@ -41,6 +42,7 @@ Memory::Memory(const roc::Device& dev, size_t size)
       deviceMemory_(nullptr),
       kind_(MEMORY_KIND_NORMAL),
       amdImageDesc_(nullptr),
+      persistent_host_ptr_(nullptr),
       pinnedMemory_(nullptr) {}
 
 Memory::~Memory() {
@@ -93,15 +95,16 @@ void* Memory::allocMapTarget(const amd::Coord3D& origin, const amd::Coord3D& reg
 
   incIndMapCount();
   // If the device backing storage is direct accessible, use it.
-  const cl_mem_flags memFlags = owner()->getMemFlags();
-  if (isHostMemDirectAccess() || (memFlags & CL_MEM_USE_PERSISTENT_MEM_AMD)) {
+  if (isHostMemDirectAccess()) {
     if (owner()->getHostMem() != nullptr) {
       return (static_cast<char*>(owner()->getHostMem()) + origin[0]);
     }
 
     return (static_cast<char*>(deviceMemory_) + origin[0]);
   }
-
+  if (IsPersistentDirectMap()) {
+    return (static_cast<char*>(persistent_host_ptr_) + origin[0]);
+  }
   // Otherwise, check for host memory.
   void* hostMem = owner()->getHostMem();
   if (hostMem != nullptr) {
@@ -150,8 +153,7 @@ void* Memory::cpuMap(device::VirtualDevice& vDev, uint flags, uint startLayer, u
 
   assert(mapTarget != nullptr);
 
-  const cl_mem_flags memFlags = owner()->getMemFlags();
-  if (!isHostMemDirectAccess() && !(memFlags & CL_MEM_USE_PERSISTENT_MEM_AMD)) {
+  if (!isHostMemDirectAccess() && !IsPersistentDirectMap()) {
     if (!vDev.blitMgr().readBuffer(*this, mapTarget, amd::Coord3D(0), amd::Coord3D(size()), true)) {
       decIndMapCount();
       return nullptr;
@@ -162,8 +164,7 @@ void* Memory::cpuMap(device::VirtualDevice& vDev, uint flags, uint startLayer, u
 }
 
 void Memory::cpuUnmap(device::VirtualDevice& vDev) {
-  const cl_mem_flags memFlags = owner()->getMemFlags();
-  if (!isHostMemDirectAccess() && !(memFlags & CL_MEM_USE_PERSISTENT_MEM_AMD)) {
+  if (!isHostMemDirectAccess() && !IsPersistentDirectMap()) {
     if (!vDev.blitMgr().writeBuffer(mapMemory_->getHostMem(), *this, amd::Coord3D(0),
                                     amd::Coord3D(size()), true)) {
       LogError("[OCL] Fail sync the device memory on cpuUnmap");
@@ -650,7 +651,7 @@ bool Buffer::create() {
     if (deviceMemory_ == nullptr) {
       return false;
     }
-    owner()->setHostMem(host_ptr);
+    persistent_host_ptr_ = host_ptr;
     return true;
   }
 #endif

projects/clr/rocclr/runtime/device/rocm/rocmemory.hpp

@@ -85,6 +85,11 @@ class Memory : public device::Memory {
 
   size_t version() const { return version_; }
 
+  bool IsPersistentDirectMap() const {
+    return ((owner() != nullptr) && (owner()->getMemFlags() & CL_MEM_USE_PERSISTENT_MEM_AMD)); }
+
+  void* PersistentHostPtr() const { return persistent_host_ptr_; }
+
  protected:
   bool allocateMapMemory(size_t allocationSize);
 
@@ -110,6 +115,8 @@ class Memory : public device::Memory {
 
   hsa_amd_image_descriptor_t* amdImageDesc_;
 
+  void* persistent_host_ptr_;  //!< Host accessible pointer for persistent memory
+
  private:
   // Disable copy constructor
   Memory(const Memory&);

projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp

@@ -1188,6 +1188,8 @@ void VirtualGPU::submitMapMemory(amd::MapMemoryCommand& cmd) {
       // Add memory to VA cache, so rutnime can detect direct access to VA
       dev().addVACache(devMemory);
     }
+  } else if (devMemory->IsPersistentDirectMap()) {
+    // Persistent memory - NOP map
   } else if (mapFlag & (CL_MAP_READ | CL_MAP_WRITE)) {
     bool result = false;
     roc::Memory* hsaMemory = static_cast<roc::Memory*>(devMemory);
@@ -1266,6 +1268,8 @@ void VirtualGPU::submitUnmapMemory(amd::UnmapMemoryCommand& cmd) {
       // Remove memory from VA cache
       dev().removeVACache(devMemory);
     }
+  } else if (devMemory->IsPersistentDirectMap()) {
+    // Persistent memory - NOP unmap
   } else if (mapInfo->isUnmapWrite()) {
     // Commit the changes made by the user.
     if (!devMemory->isHostMemDirectAccess()) {