Make hsa_agent_t handles use pointer to device::Device

For roc devices create hsa_agent_t handles using a pointer to the
device::Device. This ensures each device has a different hsa_agent_t
handle. This may be necessary to ensure the loader symbol lookup will
search only symbols for the correct device.

Change-Id: Iee6dd40d68bf22a02ce8c75cbe5ac8f5a0d9e418


[ROCm/clr commit: 76c371d78a]

projects/clr/rocclr/device/device.hpp

@@ -1476,6 +1476,25 @@ class Device : public RuntimeObject {
   //! Returns TRUE if the device is available for computations
   bool isOnline() const { return online_; }
 
+  //! Return a non-zero uint64_t value that uniquely identifies the device.
+  //! This can be used when a scalar value handle to the device is require.
+  static uint64_t toHandle(const Device *device) {
+    static_assert(reinterpret_cast<uint64_t>(static_cast<const Device*>(nullptr)) == 0,
+                  "nullptr value is not 0");
+    static_assert(sizeof(device) <= sizeof(uint64_t), "Handle size does not match pointer size");
+    return device ? reinterpret_cast<uint64_t>(device) : 0;
+  }
+
+  //! Return the device corresponding to a handle returned by Device::handle,
+  //! or nullptr if the handle is 0. This can be used when a scalar value
+  //! handle for a device is provided.
+  static const Device* fromHhandle(uint64_t handle) {
+    static_assert(reinterpret_cast<uint64_t>(static_cast<const Device*>(nullptr)) == 0,
+                  "nullptr value is not 0");
+    static_assert(sizeof(handle) <= sizeof(uint64_t), "Handle size does not match pointer size");
+    return handle ? reinterpret_cast<const Device*>(handle) : nullptr;
+  }
+
   //! Returns device settings
   const device::Settings& settings() const { return *settings_; }
 

projects/clr/rocclr/device/gpu/gpuprogram.cpp

@@ -1619,8 +1619,7 @@ bool HSAILProgram::linkImpl(amd::option::Options* options) {
     }
   }
   // ACL_TYPE_CG stage is not performed for offline compilation
-  hsa_agent_t agent;
-  agent.handle = 1;
+  hsa_agent_t agent = {amd::Device::toHandle(&(device()))};
   if (hsaLoad) {
     executable_ = loader_->CreateExecutable(HSA_PROFILE_FULL, NULL);
     if (executable_ == NULL) {

projects/clr/rocclr/device/pal/palkernel.cpp

@@ -438,10 +438,9 @@ bool LightningKernel::init() {
   }
 
   // Copy codeobject of this kernel from the program CPU segment
-  hsa_agent_t agent;
-  agent.handle = 1;
+  hsa_agent_t agent = {amd::Device::toHandle(&(device()))};
 
-  auto sym = prog().GetSymbol(symbolName().c_str(), const_cast<hsa_agent_t*>(&agent));
+  auto sym = prog().GetSymbol(symbolName().c_str(), &agent);
 
   if (!setKernelCode(sym, &akc_)) {
     return false;
@@ -452,13 +451,10 @@ bool LightningKernel::init() {
 
     // handle device enqueue
     if (!RuntimeHandle().empty()) {
-      hsa_agent_t agent;
-      agent.handle = 1;
       amd::hsa::loader::Symbol* rth_symbol;
 
       // Get the runtime handle symbol GPU address
-      rth_symbol = prog().GetSymbol(const_cast<char*>(RuntimeHandle().c_str()),
-                                    const_cast<hsa_agent_t*>(&agent));
+      rth_symbol = prog().GetSymbol(RuntimeHandle().c_str(), &agent);
       uint64_t symbol_address;
       rth_symbol->GetInfo(HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, &symbol_address);
 

projects/clr/rocclr/device/pal/palprogram.cpp

@@ -262,8 +262,7 @@ bool HSAILProgram::setKernels(amd::option::Options* options, void* binary, size_
   }
 
   // ACL_TYPE_CG stage is not performed for offline compilation
-  hsa_agent_t agent;
-  agent.handle = 1;
+
   executable_ = loader_->CreateExecutable(HSA_PROFILE_FULL, nullptr);
   if (executable_ == nullptr) {
     buildLog_ += "Error: Executable for AMD HSA Code Object isn't created.\n";
@@ -273,6 +272,7 @@ bool HSAILProgram::setKernels(amd::option::Options* options, void* binary, size_
   hsa_code_object_t code_object;
   code_object.handle = reinterpret_cast<uint64_t>(binary);
 
+  hsa_agent_t agent = {amd::Device::toHandle(&(device()))};
   hsa_status_t status = executable_->LoadCodeObject(agent, code_object, nullptr);
   if (status != HSA_STATUS_SUCCESS) {
     buildLog_ += "Error: AMD HSA Code Object loading failed.\n";
@@ -409,17 +409,17 @@ bool HSAILProgram::defineGlobalVar(const char* name, void* dptr) {
     return false;
   }
 
-  hsa_status_t hsa_status = HSA_STATUS_SUCCESS;
-  hsa_agent_t agent;
+  hsa_agent_t agent = {amd::Device::toHandle(&(device()))};
 
-  agent.handle = 1;
-  hsa_status = executable_->DefineAgentExternalVariable(name, agent, HSA_VARIABLE_SEGMENT_GLOBAL, dptr);
+  hsa_status_t hsa_status =
+      executable_->DefineAgentExternalVariable(name, agent, HSA_VARIABLE_SEGMENT_GLOBAL, dptr);
   if (HSA_STATUS_SUCCESS != hsa_status) {
     buildLog_ += "Could not define Program External Variable";
     buildLog_ += "\n";
+    return false;
   }
 
-  return (hsa_status == HSA_STATUS_SUCCESS);
+  return true;
 }
 
 bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_pptr, size_t* bytes,
@@ -432,7 +432,6 @@ bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_p
   size_t offset = 0;
   uint32_t flags = 0;
   amd::Memory* parent = nullptr;
-  hsa_agent_t agent;
   hsa_symbol_kind_t type;
   hsa_status_t status = HSA_STATUS_SUCCESS;
   amd::hsa::loader::Symbol* symbol = nullptr;
@@ -443,8 +442,9 @@ bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_p
     return false;
   }
 
+  hsa_agent_t agent = {amd::Device::toHandle(&(device()))};
+
   /* Retrieve the Symbol obj from global name*/
-  agent.handle = 1;
   symbol = executable_->GetSymbol(global_name, &agent);
   if (!symbol) {
     buildLog_ += "Error: Getting Global Var Symbol";
@@ -790,6 +790,8 @@ bool LightningProgram::setKernels(amd::option::Options* options, void* binary, s
   hsa_code_object_t code_object;
   code_object.handle = reinterpret_cast<uint64_t>(binary);
 
+  hsa_agent_t agent = {amd::Device::toHandle(&(device()))};
+
   hsa_status_t status = executable_->LoadCodeObject(agent, code_object, nullptr);
   if (status != HSA_STATUS_SUCCESS) {
     buildLog_ += "Error: AMD HSA Code Object loading failed.\n";