rocr/driver: add memory residency management interface in driver

This commit introduces MakeMemoryResident and MakeMemoryUnresident
functions to KfdDriver and XdnaDriver classes.

- Added implementations in amd_kfd_driver.cpp
- Added stubs in amd_xdna_driver.cpp returning HSA_STATUS_ERROR
- Updated header files amd_kfd_driver.h and amd_xdna_driver.h
- Removed MakeKfdMemoryResident/Unresident from amd_memory_region.cpp

Signed-off-by: Honglei Huang <Honglei1.Huang@amd.com>

runtime/hsa-runtime/core/inc/amd_kfd_driver.h

@@ -132,6 +132,10 @@ public:
   hsa_status_t AvailableMemory(uint32_t node_id, uint64_t* available_size) const override;
   hsa_status_t RegisterMemory(void* ptr, uint64_t size, HsaMemFlags mem_flags) const override;
   hsa_status_t DeregisterMemory(void* ptr) const override;
+  hsa_status_t MakeMemoryResident(const void* mem, size_t size, uint64_t* alternate_va,
+                                  const HsaMemMapFlags* mem_flags, uint32_t num_nodes,
+                                  const uint32_t* nodes) const override;
+  hsa_status_t MakeMemoryUnresident(const void* mem) const override;
 
   hsa_status_t OpenSMI(uint32_t node_id, int* fd) const override;
 

runtime/hsa-runtime/core/inc/amd_memory_region.h

@@ -77,13 +77,6 @@ class MemoryRegion : public core::MemoryRegion {
     return reinterpret_cast<MemoryRegion*>(region.handle);
   }
 
-  /// @brief Pin memory.
-  static bool MakeKfdMemoryResident(size_t num_node, const uint32_t* nodes, const void* ptr,
-                                    size_t size, uint64_t* alternate_va, HsaMemMapFlags map_flag);
-
-  /// @brief Unpin memory.
-  static bool MakeKfdMemoryUnresident(const void* ptr);
-
   MemoryRegion(bool fine_grain, bool kernarg, bool full_profile, bool extended_scope_fine_grain,
                bool user_visible, core::Agent* owner, const HsaMemoryProperties& mem_props);
 

runtime/hsa-runtime/core/inc/amd_xdna_driver.h

@@ -246,6 +246,10 @@ public:
   hsa_status_t AvailableMemory(uint32_t node_id, uint64_t* available_size) const override;
   hsa_status_t RegisterMemory(void* ptr, uint64_t size, HsaMemFlags mem_flags) const override;
   hsa_status_t DeregisterMemory(void* ptr) const override;
+  hsa_status_t MakeMemoryResident(const void* mem, size_t size, uint64_t* alternate_va,
+                                  const HsaMemMapFlags* mem_flags, uint32_t num_nodes,
+                                  const uint32_t* nodes) const override;
+  hsa_status_t MakeMemoryUnresident(const void* mem) const override;
 
   hsa_status_t IsModelEnabled(bool* enable) const override;
 

runtime/hsa-runtime/core/inc/driver.h

@@ -326,6 +326,24 @@ public:
   /// @return HSA_STATUS_SUCCESS if deregister memory successfully.
   virtual hsa_status_t DeregisterMemory(void* ptr) const = 0;
 
+  /// @brief Make the memory is resident and can be accessed by GPU
+  /// @param[in] mem address of memory to be made resident
+  /// @param[in] size size of memory
+  /// @param[out] alternate_va alternate virtual address
+  /// @param[in] mem_flags memory flags can be null
+  /// @param[in] num_nodes number of nodes to be used can be 0 if not used
+  /// @param[in] nodes nodes to be used can be null
+  /// @return HSA_STATUS_SUCCESS if the driver successfully makes the memory
+  virtual hsa_status_t MakeMemoryResident(const void* mem, size_t size, uint64_t* alternate_va,
+                                          const HsaMemMapFlags* mem_flags = nullptr,
+                                          uint32_t num_nodes = 0,
+                                          const uint32_t* nodes = nullptr) const = 0;
+
+  /// @brief Releases the residency of the memory
+  /// @param[in] mem address of memory to be made unresident
+  /// @return HSA_STATUS_SUCCESS if the driver successfully makes the memory
+  virtual hsa_status_t MakeMemoryUnresident(const void* mem) const = 0;
+
   /// Unique identifier for supported kernel-mode drivers.
   const DriverType kernel_driver_type_;