Enabled allocation of pseudo fine grain memory where memory ordering is per point to point connection

Atomic memory operations on these memory buffers are not guaranteed
to be visible at system scope

Change-Id: I4cccde114632071a000384502a83bc191e77e85b

runtime/hsa-runtime/CMakeLists.txt

@@ -87,7 +87,7 @@ if (ROCM_CCACHE_BUILD)
 endif() # if (ROCM_CCACHE_BUILD)
 
 ## Get version strings
-get_version ( "1.6.0" )
+get_version ( "1.7.0" )
 if ( ${ROCM_PATCH_VERSION} )
   set ( VERSION_PATCH ${ROCM_PATCH_VERSION})
 endif()

runtime/hsa-runtime/core/inc/memory_region.h

@@ -89,6 +89,7 @@ class MemoryRegion : public Checked<0x9C961F19EE175BB3> {
     AllocateDoubleMap = (1 << 2),   // Map twice VA allocation to backing store
     AllocateDirect = (1 << 3),      // Bypass fragment cache.
     AllocateIPC = (1 << 4),         // System memory that can be IPC-shared
+    AllocatePCIeRW = (1 << 5),      // Enforce pseudo fine grain/RW memory
   };
 
   typedef uint32_t AllocateFlags;

runtime/hsa-runtime/core/runtime/amd_memory_region.cpp

@@ -188,6 +188,9 @@ hsa_status_t MemoryRegion::AllocateImpl(size_t& size, AllocateFlags alloc_flags,
   if (IsSystem() && (alloc_flags & AllocateIPC))
       kmt_alloc_flags.ui32.NonPaged = 1;
 
+  // Allocate pseudo fine grain memory
+  kmt_alloc_flags.ui32.CoarseGrain = (alloc_flags & AllocatePCIeRW ? 0 : kmt_alloc_flags.ui32.CoarseGrain);
+
   // Only allow using the suballocator for ordinary VRAM.
   if (IsLocalMemory()) {
     bool subAllocEnabled = !core::Runtime::runtime_singleton_->flag().disable_fragment_alloc();
@@ -451,8 +454,7 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo(
   // without regard to type of requesting device (CPU / GPU)
   // Return disallowed by default if framebuffer is fine grained
   // and requesting device is connected via xGMI link
-  // Return never allowed if framebuffer is fine grained and
-  // requesting device is connected via PCIe link
+
   if (IsLocalMemory()) {
 
     // Return disallowed by default if memory is coarse
@@ -461,12 +463,6 @@ hsa_amd_memory_pool_access_t MemoryRegion::GetAccessInfo(
       return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT;
     }
 
-    // Determine if pool is pseudo fine-grained due to env flag
-    // Return disallowed by default
-    if (core::Runtime::runtime_singleton_->flag().fine_grain_pcie()) {
-      return HSA_AMD_MEMORY_POOL_ACCESS_DISALLOWED_BY_DEFAULT;
-    }
-
     // Return disallowed by default if memory is fine
     // grained and link type is xGMI.
     if (agent.HiveId() == owner()->HiveId()) {

runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp

@@ -686,7 +686,7 @@ hsa_status_t hsa_amd_memory_pool_allocate(hsa_amd_memory_pool_t memory_pool, siz
   TRY;
   IS_OPEN();
 
-  if (size == 0 || ptr == NULL || flags != 0) {
+  if (size == 0 || ptr == NULL || (flags > HSA_AMD_MEMORY_POOL_PCIE_FLAG)) {
     return HSA_STATUS_ERROR_INVALID_ARGUMENT;
   }
 
@@ -697,8 +697,11 @@ hsa_status_t hsa_amd_memory_pool_allocate(hsa_amd_memory_pool_t memory_pool, siz
     return (hsa_status_t)HSA_STATUS_ERROR_INVALID_MEMORY_POOL;
   }
 
-  return core::Runtime::runtime_singleton_->AllocateMemory(
-      mem_region, size, core::MemoryRegion::AllocateRestrict, ptr);
+  MemoryRegion::AllocateFlags alloc_flag = core::MemoryRegion::AllocateRestrict;
+
+  if (flags == HSA_AMD_MEMORY_POOL_PCIE_FLAG) alloc_flag |= core::MemoryRegion::AllocatePCIeRW;
+
+  return core::Runtime::runtime_singleton_->AllocateMemory(mem_region, size, alloc_flag, ptr);
   CATCH;
 }
 

runtime/hsa-runtime/inc/hsa_ext_amd.h

@@ -1000,6 +1000,24 @@ typedef enum {
   HSA_AMD_MEMORY_POOL_INFO_ALLOC_MAX_SIZE = 16,
 } hsa_amd_memory_pool_info_t;
 
+/**
+ * @brief Memory pool flag used to specify allocation directives
+ *
+ */
+typedef enum hsa_amd_memory_pool_flag_s {
+  /**
+   * Allocates memory that conforms to standard HSA memory consistency model
+   */
+  HSA_AMD_MEMORY_POOL_STANDARD_FLAG = 0,
+  /**
+   * Allocates fine grain memory type where memory ordering is per point to point
+   * connection. Atomic memory operations on these memory buffers are not
+   * guaranteed to be visible at system scope.
+   */
+  HSA_AMD_MEMORY_POOL_PCIE_FLAG = 1,
+
+} hsa_amd_memory_pool_flag_t;
+
 /**
  * @brief Get the current value of an attribute of a memory pool.
  *
@@ -1068,7 +1086,7 @@ hsa_status_t HSA_API hsa_amd_agent_iterate_memory_pools(
  * ::HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE in @p memory_pool.
  *
  * @param[in] flags A bit-field that is used to specify allocation
- * directives. Reserved parameter, must be 0.
+ * directives.
  *
  * @param[out] ptr Pointer to the location where to store the base virtual
  * address of