Add fine grain vram pool.

Part 1 of 2.
Enables fine grain vram over PCIe based on env flag.
Part 2 will extend to XGMI.

Change-Id: I8ad506e004b398d56d462b0200274eae2293a461


[ROCm/ROCR-Runtime commit: c56d86100b]

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -302,6 +302,9 @@ void GpuAgent::InitRegionList() {
 
           if (region->IsLocalMemory()) {
             local_region_ = region;
+            // Expose VRAM as uncached/fine grain over PCIe (if enabled) or XGMI.
+            if (core::Runtime::runtime_singleton_->flag().fine_grain_pcie())
+              regions_.push_back(new MemoryRegion(true, false, this, mem_props[mem_idx]));
           }
           break;
         }

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp

@@ -279,7 +279,8 @@ hsa_status_t MemoryRegion::GetInfo(hsa_region_info_t attribute,
           break;
         case HSA_HEAPTYPE_FRAME_BUFFER_PRIVATE:
         case HSA_HEAPTYPE_FRAME_BUFFER_PUBLIC:
-          *((uint32_t*)value) = HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED;
+          *((uint32_t*)value) = fine_grain() ? HSA_REGION_GLOBAL_FLAG_FINE_GRAINED
+                                             : HSA_REGION_GLOBAL_FLAG_COARSE_GRAINED;
           break;
         default:
           *((uint32_t*)value) = 0;

projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h

@@ -103,6 +103,9 @@ class Flag {
 
     var = os::GetEnvVar("HSA_REV_COPY_DIR");
     rev_copy_dir_ = (var == "1") ? true : false;
+
+    var = os::GetEnvVar("HSA_FORCE_FINE_GRAIN_PCIE");
+    fine_grain_pcie_ = (var == "1") ? true : false;
   }
 
   bool check_flat_scratch() const { return check_flat_scratch_; }
@@ -125,6 +128,8 @@ class Flag {
 
   bool rev_copy_dir() const { return rev_copy_dir_; }
 
+  bool fine_grain_pcie() const { return fine_grain_pcie_; }
+
   std::string enable_sdma() const { return enable_sdma_; }
 
   std::string visible_gpus() const { return visible_gpus_; }
@@ -146,6 +151,7 @@ class Flag {
   bool report_tool_load_failures_;
   bool disable_fragment_alloc_;
   bool rev_copy_dir_;
+  bool fine_grain_pcie_;
 
   std::string enable_sdma_;