SWDEV-298985 - hipMemPrefetchAsync should prefetch the data to the specified destination device

Pass the device agent specified by the user to the ROCr api instead of passing the device agent attached to the specified stream

Change-Id: I86c98935b9dc404eaa6d47ccdd082a8c3678fb36


[ROCm/clr commit: 169cc857fd]

projects/clr/rocclr/device/rocm/rocvirtual.cpp

@@ -1578,7 +1578,7 @@ void VirtualGPU::submitSvmPrefetchAsync(amd::SvmPrefetchAsyncCommand& cmd) {
     // Find the requested agent for the transfer
     hsa_agent_t agent = (cmd.cpu_access() ||
         (dev().settings().hmmFlags_ & Settings::Hmm::EnableSystemMemory)) ?
-        dev().getCpuAgent() : gpu_device();
+        dev().getCpuAgent() : (static_cast<const roc::Device*>(cmd.device()))->getBackendDevice();
 
     // Initiate a prefetch command
     hsa_status_t status = hsa_amd_svm_prefetch_async(

projects/clr/rocclr/platform/command.hpp

@@ -1509,17 +1509,19 @@ class CopyMemoryP2PCommand : public CopyMemoryCommand {
 
 /*! \brief      Prefetch command for SVM memory
  *
- *  \details    Prefetches SVM memory into the current device or CPU
+ *  \details    Prefetches SVM memory into the destination device or CPU
  */
 class SvmPrefetchAsyncCommand : public Command {
   const void* dev_ptr_;   //!< Device pointer to memory for prefetch
   size_t count_;          //!< the size for prefetch
-  bool   cpu_access_;     //!< Prefetch data into CPU location
+  bool cpu_access_;       //!< Prefetch data into CPU location
+  amd::Device* dev_;      //!< Destination device to prefetch to
 
  public:
   SvmPrefetchAsyncCommand(HostQueue& queue, const EventWaitList& eventWaitList,
-                          const void* dev_ptr, size_t count, bool cpu_access)
-      : Command(queue, 1, eventWaitList), dev_ptr_(dev_ptr), count_(count), cpu_access_(cpu_access) {}
+                          const void* dev_ptr, size_t count, amd::Device* dev, bool cpu_access)
+      : Command(queue, 1, eventWaitList), dev_ptr_(dev_ptr), count_(count),
+        dev_(dev), cpu_access_(cpu_access) {}
 
   virtual void submit(device::VirtualDevice& device) { device.submitSvmPrefetchAsync(*this); }
 
@@ -1527,6 +1529,7 @@ class SvmPrefetchAsyncCommand : public Command {
 
   const void* dev_ptr() const { return dev_ptr_; }
   size_t count() const { return count_; }
+  amd::Device* device() const { return dev_; }
   size_t cpu_access() const { return cpu_access_; }
 };