SWDEV-240804 - Add coarse grain memory support

Add an extension to memory advise to disable cache coherency for
better performance

Change-Id: I283703d81d9c36ddfa2c8fffa15eef60e2195056

rocclr/device/device.hpp

@@ -97,10 +97,11 @@ enum MemoryAdvice : uint32_t {
   UnsetReadMostly = 2,        ///< Undo the effect of hipMemAdviseSetReadMostly
   SetPreferredLocation = 3,   ///< Set the preferred location for the data as the specified device
   UnsetPreferredLocation = 4, ///< Clear the preferred location for the data
-  SetAccessedBy = 5,          ///< Data will be accessed by the specified device,
-                              ///< so prevent page faults as much as possible
-  UnsetAccessedBy = 6         ///< Let the Unified Memory subsystem decide on
-                              ///< the page faulting policy for the specified device
+  SetAccessedBy = 5,          ///< Data will be accessed by the specified device, reducing
+                              ///< the amount of page faults
+  UnsetAccessedBy = 6,        ///< HMM decides on the page faulting policy for the specified device
+  SetCoarseGrain = 100,       ///< Change cache policy to improve performance (disables coherency)
+  UnsetCoarseGrain = 101      ///< Restore coherent cache policy at the cost of some performance
 };
 
 enum MemRangeAttribute : uint32_t {

rocclr/device/rocm/rocdevice.cpp

@@ -2270,6 +2270,12 @@ bool Device::SetSvmAttributesInt(const void* dev_ptr, size_t count,
         // @note: 0 may cause a failure on old runtimes
         attr.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE, 0});
         break;
+      case amd::MemoryAdvice::SetCoarseGrain:
+        attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED});
+        break;
+      case amd::MemoryAdvice::UnsetCoarseGrain:
+        attr.push_back({HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG, HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED});
+        break;
       default:
         return false;
       break;