From 2476472bcbc94346e5633a11533e36615578100a Mon Sep 17 00:00:00 2001 From: Tony Tye Date: Sun, 10 Jan 2021 10:33:58 +0000 Subject: [PATCH] Make hsa_agent_t handles use pointer to device::Device For roc devices create hsa_agent_t handles using a pointer to the device::Device. This ensures each device has a different hsa_agent_t handle. This may be necessary to ensure the loader symbol lookup will search only symbols for the correct device. Change-Id: Iee6dd40d68bf22a02ce8c75cbe5ac8f5a0d9e418 [ROCm/clr commit: 76c371d78ab6bc31b29b3f991b5de4e84a75fc78] --- projects/clr/rocclr/device/device.hpp | 19 ++++++++++++++++++ projects/clr/rocclr/device/gpu/gpuprogram.cpp | 3 +-- projects/clr/rocclr/device/pal/palkernel.cpp | 10 +++------- projects/clr/rocclr/device/pal/palprogram.cpp | 20 ++++++++++--------- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/projects/clr/rocclr/device/device.hpp b/projects/clr/rocclr/device/device.hpp index e5d4e55fb2..6b1d281616 100644 --- a/projects/clr/rocclr/device/device.hpp +++ b/projects/clr/rocclr/device/device.hpp @@ -1476,6 +1476,25 @@ class Device : public RuntimeObject { //! Returns TRUE if the device is available for computations bool isOnline() const { return online_; } + //! Return a non-zero uint64_t value that uniquely identifies the device. + //! This can be used when a scalar value handle to the device is require. + static uint64_t toHandle(const Device *device) { + static_assert(reinterpret_cast(static_cast(nullptr)) == 0, + "nullptr value is not 0"); + static_assert(sizeof(device) <= sizeof(uint64_t), "Handle size does not match pointer size"); + return device ? reinterpret_cast(device) : 0; + } + + //! Return the device corresponding to a handle returned by Device::handle, + //! or nullptr if the handle is 0. This can be used when a scalar value + //! handle for a device is provided. + static const Device* fromHhandle(uint64_t handle) { + static_assert(reinterpret_cast(static_cast(nullptr)) == 0, + "nullptr value is not 0"); + static_assert(sizeof(handle) <= sizeof(uint64_t), "Handle size does not match pointer size"); + return handle ? reinterpret_cast(handle) : nullptr; + } + //! Returns device settings const device::Settings& settings() const { return *settings_; } diff --git a/projects/clr/rocclr/device/gpu/gpuprogram.cpp b/projects/clr/rocclr/device/gpu/gpuprogram.cpp index bd95dc8368..d57f07624d 100644 --- a/projects/clr/rocclr/device/gpu/gpuprogram.cpp +++ b/projects/clr/rocclr/device/gpu/gpuprogram.cpp @@ -1619,8 +1619,7 @@ bool HSAILProgram::linkImpl(amd::option::Options* options) { } } // ACL_TYPE_CG stage is not performed for offline compilation - hsa_agent_t agent; - agent.handle = 1; + hsa_agent_t agent = {amd::Device::toHandle(&(device()))}; if (hsaLoad) { executable_ = loader_->CreateExecutable(HSA_PROFILE_FULL, NULL); if (executable_ == NULL) { diff --git a/projects/clr/rocclr/device/pal/palkernel.cpp b/projects/clr/rocclr/device/pal/palkernel.cpp index f12bd9077f..8dc74a1982 100644 --- a/projects/clr/rocclr/device/pal/palkernel.cpp +++ b/projects/clr/rocclr/device/pal/palkernel.cpp @@ -438,10 +438,9 @@ bool LightningKernel::init() { } // Copy codeobject of this kernel from the program CPU segment - hsa_agent_t agent; - agent.handle = 1; + hsa_agent_t agent = {amd::Device::toHandle(&(device()))}; - auto sym = prog().GetSymbol(symbolName().c_str(), const_cast(&agent)); + auto sym = prog().GetSymbol(symbolName().c_str(), &agent); if (!setKernelCode(sym, &akc_)) { return false; @@ -452,13 +451,10 @@ bool LightningKernel::init() { // handle device enqueue if (!RuntimeHandle().empty()) { - hsa_agent_t agent; - agent.handle = 1; amd::hsa::loader::Symbol* rth_symbol; // Get the runtime handle symbol GPU address - rth_symbol = prog().GetSymbol(const_cast(RuntimeHandle().c_str()), - const_cast(&agent)); + rth_symbol = prog().GetSymbol(RuntimeHandle().c_str(), &agent); uint64_t symbol_address; rth_symbol->GetInfo(HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, &symbol_address); diff --git a/projects/clr/rocclr/device/pal/palprogram.cpp b/projects/clr/rocclr/device/pal/palprogram.cpp index d6afeef44f..2a4f0c51ea 100644 --- a/projects/clr/rocclr/device/pal/palprogram.cpp +++ b/projects/clr/rocclr/device/pal/palprogram.cpp @@ -262,8 +262,7 @@ bool HSAILProgram::setKernels(amd::option::Options* options, void* binary, size_ } // ACL_TYPE_CG stage is not performed for offline compilation - hsa_agent_t agent; - agent.handle = 1; + executable_ = loader_->CreateExecutable(HSA_PROFILE_FULL, nullptr); if (executable_ == nullptr) { buildLog_ += "Error: Executable for AMD HSA Code Object isn't created.\n"; @@ -273,6 +272,7 @@ bool HSAILProgram::setKernels(amd::option::Options* options, void* binary, size_ hsa_code_object_t code_object; code_object.handle = reinterpret_cast(binary); + hsa_agent_t agent = {amd::Device::toHandle(&(device()))}; hsa_status_t status = executable_->LoadCodeObject(agent, code_object, nullptr); if (status != HSA_STATUS_SUCCESS) { buildLog_ += "Error: AMD HSA Code Object loading failed.\n"; @@ -409,17 +409,17 @@ bool HSAILProgram::defineGlobalVar(const char* name, void* dptr) { return false; } - hsa_status_t hsa_status = HSA_STATUS_SUCCESS; - hsa_agent_t agent; + hsa_agent_t agent = {amd::Device::toHandle(&(device()))}; - agent.handle = 1; - hsa_status = executable_->DefineAgentExternalVariable(name, agent, HSA_VARIABLE_SEGMENT_GLOBAL, dptr); + hsa_status_t hsa_status = + executable_->DefineAgentExternalVariable(name, agent, HSA_VARIABLE_SEGMENT_GLOBAL, dptr); if (HSA_STATUS_SUCCESS != hsa_status) { buildLog_ += "Could not define Program External Variable"; buildLog_ += "\n"; + return false; } - return (hsa_status == HSA_STATUS_SUCCESS); + return true; } bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_pptr, size_t* bytes, @@ -432,7 +432,6 @@ bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_p size_t offset = 0; uint32_t flags = 0; amd::Memory* parent = nullptr; - hsa_agent_t agent; hsa_symbol_kind_t type; hsa_status_t status = HSA_STATUS_SUCCESS; amd::hsa::loader::Symbol* symbol = nullptr; @@ -443,8 +442,9 @@ bool HSAILProgram::createGlobalVarObj(amd::Memory** amd_mem_obj, void** device_p return false; } + hsa_agent_t agent = {amd::Device::toHandle(&(device()))}; + /* Retrieve the Symbol obj from global name*/ - agent.handle = 1; symbol = executable_->GetSymbol(global_name, &agent); if (!symbol) { buildLog_ += "Error: Getting Global Var Symbol"; @@ -790,6 +790,8 @@ bool LightningProgram::setKernels(amd::option::Options* options, void* binary, s hsa_code_object_t code_object; code_object.handle = reinterpret_cast(binary); + hsa_agent_t agent = {amd::Device::toHandle(&(device()))}; + hsa_status_t status = executable_->LoadCodeObject(agent, code_object, nullptr); if (status != HSA_STATUS_SUCCESS) { buildLog_ += "Error: AMD HSA Code Object loading failed.\n";