SWDEV-254363 - Checks to make sure kernel is launched on the same device as it was launched.

Change-Id: I2f273a70b1a5fc0e9fc9c6144eabd32466ff4e59

rocclr/hip_code_object.cpp

@@ -160,22 +160,26 @@ DynCO::~DynCO() {
   delete fb_info_;
 }
 
-hipError_t DynCO::getDeviceVar(DeviceVar** dvar, std::string var_name, int device_id) {
+hipError_t DynCO::getDeviceVar(DeviceVar** dvar, std::string var_name) {
   amd::ScopedLock lock(dclock_);
 
+  CheckDeviceIdMatch();
+
   auto it = vars_.find(var_name);
   if (it == vars_.end()) {
     DevLogPrintfError("Cannot find the Var: %s ", var_name.c_str());
     return hipErrorNotFound;
   }
 
-  it->second->getDeviceVar(dvar, device_id, module());
+  it->second->getDeviceVar(dvar, device_id_, module());
   return hipSuccess;
 }
 
 hipError_t DynCO::getDynFunc(hipFunction_t* hfunc, std::string func_name) {
   amd::ScopedLock lock(dclock_);
 
+  CheckDeviceIdMatch();
+
   auto it = functions_.find(func_name);
   if (it == functions_.end()) {
     DevLogPrintfError("Cannot find the function: %s ", func_name.c_str());

rocclr/hip_code_object.hpp

@@ -74,7 +74,7 @@ class DynCO : public CodeObject {
   amd::Monitor dclock_{"Guards Dynamic Code object", true};
 
 public:
-  DynCO() {}
+  DynCO() : device_id_(ihipGetDevice()) {}
   virtual ~DynCO();
 
   //LoadsCodeObject and its data
@@ -83,9 +83,17 @@ public:
 
   //Gets GlobalVar/Functions from a dynamically loaded code object
   hipError_t getDynFunc(hipFunction_t* hfunc, std::string func_name);
-  hipError_t getDeviceVar(DeviceVar** dvar, std::string var_name, int deviceId);
+  hipError_t getDeviceVar(DeviceVar** dvar, std::string var_name);
+
+  // Device ID Check to check if module is launched in the same device it was loaded.
+  inline void CheckDeviceIdMatch() {
+    if (device_id_ != ihipGetDevice()) {
+      guarantee(false && "Device mismatch from where this module is loaded");
+    }
+  }
 
 private:
+  int device_id_;
   FatBinaryInfo* fb_info_;
 
   //Maps for vars/funcs, could be keyed in with std::string name

rocclr/hip_module.cpp

@@ -98,7 +98,7 @@ hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, hipModule_t h
   HIP_INIT_API(hipModuleGetGlobal, dptr, bytes, hmod, name);
 
   /* Get address and size for the global symbol */
-  if (hipSuccess != PlatformState::instance().getDynGlobalVar(name, ihipGetDevice(), hmod, dptr, bytes)) {
+  if (hipSuccess != PlatformState::instance().getDynGlobalVar(name, hmod, dptr, bytes)) {
     DevLogPrintfError("Cannot find global Var: %s for module: 0x%x at device: %d \n",
                        name, hmod, ihipGetDevice());
     HIP_RETURN(hipErrorNotFound);

rocclr/hip_platform.cpp

@@ -759,7 +759,7 @@ hipError_t PlatformState::getDynFunc(hipFunction_t* hfunc, hipModule_t hmod,
   return it->second->getDynFunc(hfunc, func_name);
 }
 
-hipError_t PlatformState::getDynGlobalVar(const char* hostVar, int deviceId, hipModule_t hmod,
+hipError_t PlatformState::getDynGlobalVar(const char* hostVar, hipModule_t hmod,
                                           hipDeviceptr_t* dev_ptr, size_t* size_ptr) {
   amd::ScopedLock lock(lock_);
 
@@ -770,7 +770,7 @@ hipError_t PlatformState::getDynGlobalVar(const char* hostVar, int deviceId, hip
   }
 
   hip::DeviceVar* dvar = nullptr;
-  IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar, deviceId));
+  IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar));
   *dev_ptr = dvar->device_ptr();
   *size_ptr = dvar->size();
 
@@ -784,8 +784,8 @@ hipError_t PlatformState::registerTexRef(textureReference* texRef, hipModule_t h
   return hipSuccess;
 }
 
-hipError_t PlatformState::getDynTexGlobalVar(textureReference* texRef, int deviceId,
-                                             hipDeviceptr_t* dev_ptr, size_t* size_ptr) {
+hipError_t PlatformState::getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr,
+                                             size_t* size_ptr) {
   amd::ScopedLock lock(lock_);
 
   auto tex_it = texRef_map_.find(texRef);
@@ -801,7 +801,7 @@ hipError_t PlatformState::getDynTexGlobalVar(textureReference* texRef, int devic
   }
 
   hip::DeviceVar* dvar = nullptr;
-  IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, tex_it->second.second, deviceId));
+  IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, tex_it->second.second));
   *dev_ptr = dvar->device_ptr();
   *size_ptr = dvar->size();
 
@@ -818,7 +818,7 @@ hipError_t PlatformState::getDynTexRef(const char* hostVar, hipModule_t hmod, te
   }
 
   hip::DeviceVar* dvar = nullptr;
-  IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar, ihipGetDevice()));
+  IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar));
 
   dvar->shadowVptr = new texture<char>();
   *texRef =  reinterpret_cast<textureReference*>(dvar->shadowVptr);

rocclr/hip_platform.hpp

@@ -48,12 +48,13 @@ public:
   hipError_t unloadModule(hipModule_t hmod);
 
   hipError_t getDynFunc(hipFunction_t *hfunc, hipModule_t hmod, const char* func_name);
-  hipError_t getDynGlobalVar(const char* hostVar, int deviceId, hipModule_t hmod, 
-                             hipDeviceptr_t* dev_ptr, size_t* size_ptr);
+  hipError_t getDynGlobalVar(const char* hostVar, hipModule_t hmod, hipDeviceptr_t* dev_ptr,
+                             size_t* size_ptr);
   hipError_t getDynTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef);
 
   hipError_t registerTexRef(textureReference* texRef, hipModule_t hmod, std::string name);
-  hipError_t getDynTexGlobalVar(textureReference* texRef, int deviceId, hipDeviceptr_t* dev_ptr, size_t* size_ptr);
+  hipError_t getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr,
+                                size_t* size_ptr);
 
   /* Singleton instance */
   static PlatformState& instance() {

rocclr/hip_texture.cpp

@@ -805,8 +805,7 @@ hipError_t hipTexRefSetArray(textureReference* texRef,
   hipDeviceptr_t refDevPtr = nullptr;
   size_t refDevSize = 0;
 
-  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, ihipGetDevice(),
-                                                                 &refDevPtr, &refDevSize));
+  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize));
   assert(refDevSize == sizeof(textureReference));
 
   // Any previous address or HIP array state associated with the texture reference is superseded by this function.
@@ -881,8 +880,7 @@ hipError_t hipTexRefSetAddress(size_t* ByteOffset,
 
   hipDeviceptr_t refDevPtr = nullptr;
   size_t refDevSize = 0;
-  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, ihipGetDevice(),
-                                                                 &refDevPtr, &refDevSize));
+  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize));
   assert(refDevSize == sizeof(textureReference));
 
   // Any previous address or HIP array state associated with the texture reference is superseded by this function.
@@ -926,8 +924,7 @@ hipError_t hipTexRefSetAddress2D(textureReference* texRef,
 
   hipDeviceptr_t refDevPtr = nullptr;
   size_t refDevSize = 0;
-  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, ihipGetDevice(),
-                                                                 &refDevPtr, &refDevSize));
+  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize));
   assert(refDevSize == sizeof(textureReference));
 
   // Any previous address or HIP array state associated with the texture reference is superseded by this function.
@@ -1204,8 +1201,7 @@ hipError_t hipTexRefSetMipmappedArray(textureReference* texRef,
 
   hipDeviceptr_t refDevPtr = nullptr;
   size_t refDevSize = 0;
-  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, ihipGetDevice(),
-                                                                 &refDevPtr, &refDevSize));
+  HIP_RETURN_ONFAIL(PlatformState::instance().getDynTexGlobalVar(texRef, &refDevPtr, &refDevSize));
   assert(refDevSize == sizeof(textureReference));
 
   // Any previous address or HIP array state associated with the texture reference is superseded by this function.