From 1aae64e2516f91ced9319ee0f645a8aa6dd1928d Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Wed, 21 Apr 2021 14:49:43 -0500 Subject: [PATCH] Revert "Revert SVM and XNACK support." This reverts commit da41352a93de1eec1a842e573536d87fede6ada9. Conflicts: opensrc/hsa-runtime/core/util/flag.h Change-Id: I16daf41588e6139126d66af54b0693de2e7e39f3 [ROCm/ROCR-Runtime commit: 77046a1aaa11e242a9668187424ff06dfe9d60c2] --- .../core/common/hsa_table_interface.cpp | 21 + .../hsa-runtime/core/inc/amd_gpu_agent.h | 3 +- .../hsa-runtime/core/inc/hsa_ext_amd_impl.h | 15 + .../runtime/hsa-runtime/core/inc/runtime.h | 38 ++ .../core/runtime/amd_gpu_agent.cpp | 6 +- .../hsa-runtime/core/runtime/amd_topology.cpp | 45 +- .../core/runtime/hsa_api_trace.cpp | 3 + .../hsa-runtime/core/runtime/hsa_ext_amd.cpp | 32 ++ .../hsa-runtime/core/runtime/runtime.cpp | 506 ++++++++++++++++++ .../runtime/hsa-runtime/core/util/flag.h | 15 + .../runtime/hsa-runtime/hsacore.so.def | 3 + .../runtime/hsa-runtime/inc/hsa.h | 16 +- .../runtime/hsa-runtime/inc/hsa_api_trace.h | 3 + .../runtime/hsa-runtime/inc/hsa_ext_amd.h | 156 ++++++ 14 files changed, 852 insertions(+), 10 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp index e48f24c54f..0ff7b5f8b9 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/common/hsa_table_interface.cpp @@ -1151,6 +1151,27 @@ hsa_status_t HSA_API hsa_amd_signal_value_pointer(hsa_signal_t signal, return amdExtTable->hsa_amd_signal_value_pointer_fn(signal, value_ptr); } +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_svm_attributes_set(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count) { + return amdExtTable->hsa_amd_svm_attributes_set_fn(ptr, size, attribute_list, attribute_count); +} + +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_svm_attributes_get(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count) { + return amdExtTable->hsa_amd_svm_attributes_get_fn(ptr, size, attribute_list, attribute_count); +} + +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_svm_prefetch_async(void* ptr, size_t size, hsa_agent_t agent, + uint32_t num_dep_signals, const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal) { + return amdExtTable->hsa_amd_svm_prefetch_async_fn(ptr, size, agent, num_dep_signals, dep_signals, completion_signal); +} + // Tools only table interfaces. namespace rocr { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index c00005ff0f..df1f4f2b95 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -173,7 +173,8 @@ class GpuAgent : public GpuAgentInt { // @param [in] node Node id. Each CPU in different socket will get distinct // id. // @param [in] node_props Node property. - GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props); + // @param [in] xnack_mode XNACK mode of device. + GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xnack_mode); // @brief GPU agent destructor. ~GpuAgent(); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h index 510e369601..9954b8fc7a 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/hsa_ext_amd_impl.h @@ -246,6 +246,21 @@ hsa_status_t hsa_amd_deregister_deallocation_callback( hsa_status_t hsa_amd_signal_value_pointer(hsa_signal_t signal, volatile hsa_signal_value_t** value_ptr); +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_svm_attributes_set(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count); + +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_svm_attributes_get(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count); + +// Mirrors Amd Extension Apis +hsa_status_t HSA_API hsa_amd_svm_prefetch_async(void* ptr, size_t size, hsa_agent_t agent, + uint32_t num_dep_signals, const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal); + } // namespace amd } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h index 634224c5ff..d232e6f955 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/runtime.h @@ -292,6 +292,15 @@ class Runtime { hsa_status_t IPCDetach(void* ptr); + hsa_status_t SetSvmAttrib(void* ptr, size_t size, hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count); + + hsa_status_t GetSvmAttrib(void* ptr, size_t size, hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count); + + hsa_status_t SvmPrefetch(void* ptr, size_t size, hsa_agent_t agent, uint32_t num_dep_signals, + const hsa_signal_t* dep_signals, hsa_signal_t completion_signal); + const std::vector& cpu_agents() { return cpu_agents_; } const std::vector& gpu_agents() { return gpu_agents_; } @@ -395,6 +404,28 @@ class Runtime { std::vector arg_; }; + struct PrefetchRange; + typedef std::map prefetch_map_t; + + struct PrefetchOp { + void* base; + size_t size; + uint32_t node_id; + int remaining_deps; + hsa_signal_t completion; + std::vector dep_signals; + prefetch_map_t::iterator prefetch_map_entry; + }; + + struct PrefetchRange { + PrefetchRange() {} + PrefetchRange(size_t Bytes, PrefetchOp* Op) : bytes(Bytes), op(Op) {} + size_t bytes; + PrefetchOp* op; + prefetch_map_t::iterator prev; + prefetch_map_t::iterator next; + }; + // Will be created before any user could call hsa_init but also could be // destroyed before incorrectly written programs call hsa_shutdown. static KernelMutex bootstrap_lock_; @@ -444,6 +475,9 @@ class Runtime { /// @retval Index in ::link_matrix_. uint32_t GetIndexLinkInfo(uint32_t node_id_from, uint32_t node_id_to); + /// @brief Get most recently issued SVM prefetch agent for the range in question. + Agent* GetSVMPrefetchAgent(void* ptr, size_t size); + // Mutex object to protect multithreaded access to ::allocation_map_, // KFD map/unmap, register/unregister, and access to hsaKmtQueryPointerInfo // registered & mapped arrays. @@ -485,6 +519,10 @@ class Runtime { // Contains the region, address, and size of previously allocated memory. std::map allocation_map_; + // Pending prefetch containers. + KernelMutex prefetch_lock_; + prefetch_map_t prefetch_map_; + // Allocator using ::system_region_ std::function system_allocator_; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 090d3d0a18..1d5205f527 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -77,7 +77,7 @@ extern HsaApiTable hsa_internal_api_table_; } // namespace core namespace AMD { -GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) +GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xnack_mode) : GpuAgentInt(node), properties_(node_props), current_coherency_type_(HSA_AMD_COHERENCY_TYPE_COHERENT), @@ -119,8 +119,8 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) rocr::core::IsaFeature xnack = rocr::core::IsaFeature::Unsupported; if (isa_base->IsXnackSupported()) { // TODO: This needs to be obtained form KFD once HMM implemented. - xnack = profile_ == HSA_PROFILE_FULL ? core::IsaFeature::Enabled - : core::IsaFeature::Disabled; + xnack = xnack_mode ? core::IsaFeature::Enabled + : core::IsaFeature::Disabled; } // Set instruction set architecture via node property, only on GPU device. diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp index 97d45d8e16..6b6a308583 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_topology.cpp @@ -68,6 +68,38 @@ namespace AMD { static const uint kKfdVersionMajor = 0; static const uint kKfdVersionMinor = 99; +// Query for user preference and use that to determine Xnack mode of ROCm system. +// Return true if Xnack mode is ON or false if OFF. Xnack mode of a system is +// orthogonal to devices that do not support Xnack mode. It is legal for a +// system with Xnack ON to have devices that do not support Xnack functionality. +bool BindXnackMode() { + // Get users' preference for Xnack mode of ROCm platform + HSAint32 mode; + mode = core::Runtime::runtime_singleton_->flag().xnack(); + bool config_xnack = + (core::Runtime::runtime_singleton_->flag().xnack() != Flag::XNACK_REQUEST::XNACK_UNCHANGED); + + // Indicate to driver users' preference for Xnack mode + // Call to driver can fail and is a supported feature + HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; + if (config_xnack) { + status = hsaKmtSetXNACKMode(mode); + if (status == HSAKMT_STATUS_SUCCESS) { + return mode; + } + } + + // Get Xnack mode of devices bound by driver. This could happen + // when a call to SET Xnack mode fails or user has no particular + // preference + status = hsaKmtGetXNACKMode((HSAint32*)&mode); + if(status != HSAKMT_STATUS_SUCCESS) { + debug_print("KFD does not support xnack mode query.\nROCr must assume xnack is disabled.\n"); + return false; + } + return mode; +} + CpuAgent* DiscoverCpu(HSAuint32 node_id, HsaNodeProperties& node_prop) { if (node_prop.NumCPUCores == 0) { return nullptr; @@ -79,14 +111,14 @@ CpuAgent* DiscoverCpu(HSAuint32 node_id, HsaNodeProperties& node_prop) { return cpu; } -GpuAgent* DiscoverGpu(HSAuint32 node_id, HsaNodeProperties& node_prop) { +GpuAgent* DiscoverGpu(HSAuint32 node_id, HsaNodeProperties& node_prop, bool xnack_mode) { GpuAgent* gpu = nullptr; if (node_prop.NumFComputeCores == 0) { // Ignore non GPUs. return nullptr; } try { - gpu = new GpuAgent(node_id, node_prop); + gpu = new GpuAgent(node_id, node_prop, xnack_mode); const HsaVersionInfo& kfd_version = core::Runtime::runtime_singleton_->KfdVersion(); @@ -212,7 +244,7 @@ void RegisterLinkInfo(uint32_t node_id, uint32_t num_link) { /** * Process the list of Gpus that are surfaced to user */ -static void SurfaceGpuList(std::vector& gpu_list) { +static void SurfaceGpuList(std::vector& gpu_list, bool xnack_mode) { // Process user visible Gpu devices int32_t invalidIdx = -1; int32_t list_sz = gpu_list.size(); @@ -229,7 +261,7 @@ static void SurfaceGpuList(std::vector& gpu_list) { // Instantiate a Gpu device. The IO links // of this node have already been registered assert((node_prop.NumFComputeCores != 0) && "Improper node used for GPU device discovery."); - DiscoverGpu(gpu_list[idx], node_prop); + DiscoverGpu(gpu_list[idx], node_prop, xnack_mode); } } @@ -313,8 +345,11 @@ void BuildTopology() { RegisterLinkInfo(node_id, node_prop.NumIOLinks); } + // Determine the Xnack mode to be bound for system + bool xnack_mode = BindXnackMode(); + // Instantiate ROCr objects to encapsulate Gpu devices - SurfaceGpuList(gpu_usr_list); + SurfaceGpuList(gpu_usr_list, xnack_mode); } bool Load() { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp index 9e50971b3d..605ec15aec 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_api_trace.cpp @@ -391,6 +391,9 @@ void HsaApiTable::UpdateAmdExts() { amd_ext_api.hsa_amd_register_deallocation_callback_fn = AMD::hsa_amd_register_deallocation_callback; amd_ext_api.hsa_amd_deregister_deallocation_callback_fn = AMD::hsa_amd_deregister_deallocation_callback; amd_ext_api.hsa_amd_signal_value_pointer_fn = AMD::hsa_amd_signal_value_pointer; + amd_ext_api.hsa_amd_svm_attributes_set_fn = AMD::hsa_amd_svm_attributes_set; + amd_ext_api.hsa_amd_svm_attributes_get_fn = AMD::hsa_amd_svm_attributes_get; + amd_ext_api.hsa_amd_svm_prefetch_async_fn = AMD::hsa_amd_svm_prefetch_async; } void LoadInitialHsaApiTable() { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index f4776eb990..66d7ff7278 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -1009,5 +1009,37 @@ hsa_status_t hsa_amd_runtime_queue_create_register(hsa_amd_runtime_queue_notifie CATCH; } +hsa_status_t hsa_amd_svm_attributes_set(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count) { + TRY; + IS_OPEN(); + return core::Runtime::runtime_singleton_->SetSvmAttrib(ptr, size, attribute_list, + attribute_count); + CATCH; +} + +hsa_status_t hsa_amd_svm_attributes_get(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count) { + TRY; + IS_OPEN(); + return core::Runtime::runtime_singleton_->GetSvmAttrib(ptr, size, attribute_list, + attribute_count); + CATCH; +} + +hsa_status_t hsa_amd_svm_prefetch_async(void* ptr, size_t size, hsa_agent_t agent, + uint32_t num_dep_signals, const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal) { + TRY; + IS_OPEN(); + // Validate inputs. + // if (core::g_use_interrupt_wait && (!core::InterruptSignal::IsType(signal))) + return core::Runtime::runtime_singleton_->SvmPrefetch(ptr, size, agent, num_dep_signals, + dep_signals, completion_signal); + CATCH; +} + } // namespace amd } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp index f622cf2ddc..aba70cb5e3 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -644,6 +644,18 @@ hsa_status_t Runtime::GetSystemInfo(hsa_system_info_t attribute, void* value) { *(const char**)value = STRING(ROCR_BUILD_ID); break; } + case HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED: { + // todo: Get HMM kernel support info. + *(bool*)value = true; + break; + } + case HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT: { + bool ret = true; + for(auto agent : gpu_agents_) + ret &= (agent->isa()->GetXnack() == IsaFeature::Enabled); + *(bool*)value = ret; + break; + } default: return HSA_STATUS_ERROR_INVALID_ARGUMENT; } @@ -1586,5 +1598,499 @@ void Runtime::InternalQueueCreateNotify(const hsa_queue_t* queue, hsa_agent_t ag internal_queue_create_notifier_(queue, agent, internal_queue_create_notifier_user_data_); } +hsa_status_t Runtime::SetSvmAttrib(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count) { + uint32_t set_attribs = 0; + std::vector agent_seen(agents_by_node_.size(), false); + + std::vector attribs; + attribs.reserve(attribute_count); + uint32_t set_flags = 0; + uint32_t clear_flags = 0; + + auto Convert = [&](uint64_t value) -> Agent* { + hsa_agent_t handle = {value}; + Agent* agent = Agent::Convert(handle); + if ((agent == nullptr) || !agent->IsValid()) + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_AGENT, + "Invalid agent handle in Runtime::SetSvmAttrib."); + return agent; + }; + + auto ConvertAllowNull = [&](uint64_t value) -> Agent* { + hsa_agent_t handle = {value}; + Agent* agent = Agent::Convert(handle); + if ((agent != nullptr) && (!agent->IsValid())) + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_AGENT, + "Invalid agent handle in Runtime::SetSvmAttrib."); + return agent; + }; + + auto ConfirmNew = [&](Agent* agent) { + if (agent_seen[agent->node_id()]) + throw AMD::hsa_exception( + HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS, + "Multiple attributes given for the same agent in Runtime::SetSvmAttrib."); + agent_seen[agent->node_id()] = true; + }; + + auto Check = [&](uint64_t attrib) { + if (set_attribs & (1 << attrib)) + throw AMD::hsa_exception(HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS, + "Attribute given multiple times in Runtime::SetSvmAttrib."); + set_attribs |= (1 << attrib); + }; + + auto kmtPair = [](uint32_t attrib, uint32_t value) { + HSA_SVM_ATTRIBUTE pair = {attrib, value}; + return pair; + }; + + for (uint32_t i = 0; i < attribute_count; i++) { + auto attrib = attribute_list[i].attribute; + auto value = attribute_list[i].value; + + switch (attrib) { + case HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG: { + Check(attrib); + switch (value) { + case HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED: + set_flags |= HSA_SVM_FLAG_COHERENT; + break; + case HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED: + clear_flags |= HSA_SVM_FLAG_COHERENT; + break; + default: + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_ARGUMENT, + "Invalid HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG value."); + } + break; + } + case HSA_AMD_SVM_ATTRIB_READ_ONLY: { + Check(attrib); + if (value) + set_flags |= HSA_SVM_FLAG_GPU_RO; + else + clear_flags |= HSA_SVM_FLAG_GPU_RO; + break; + } + case HSA_AMD_SVM_ATTRIB_HIVE_LOCAL: { + Check(attrib); + if (value) + set_flags |= HSA_SVM_FLAG_HIVE_LOCAL; + else + clear_flags |= HSA_SVM_FLAG_HIVE_LOCAL; + break; + } + case HSA_AMD_SVM_ATTRIB_MIGRATION_GRANULARITY: { + Check(attrib); + // Max migration size is 1GB. + if (value > 18) value = 18; + attribs.push_back(kmtPair(HSA_SVM_ATTR_GRANULARITY, value)); + break; + } + case HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION: { + Check(attrib); + Agent* agent = ConvertAllowNull(value); + if (agent == nullptr) + attribs.push_back(kmtPair(HSA_SVM_ATTR_PREFERRED_LOC, INVALID_NODEID)); + else + attribs.push_back(kmtPair(HSA_SVM_ATTR_PREFERRED_LOC, agent->node_id())); + break; + } + case HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE: { + Agent* agent = Convert(value); + ConfirmNew(agent); + if (agent->device_type() == Agent::kAmdCpuDevice) { + set_flags |= HSA_SVM_FLAG_HOST_ACCESS; + } else { + attribs.push_back(kmtPair(HSA_SVM_ATTR_ACCESS, agent->node_id())); + } + break; + } + case HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE: { + Agent* agent = Convert(value); + ConfirmNew(agent); + if (agent->device_type() == Agent::kAmdCpuDevice) { + set_flags |= HSA_SVM_FLAG_HOST_ACCESS; + } else { + attribs.push_back(kmtPair(HSA_SVM_ATTR_ACCESS_IN_PLACE, agent->node_id())); + } + break; + } + case HSA_AMD_SVM_ATTRIB_AGENT_NO_ACCESS: { + Agent* agent = Convert(value); + ConfirmNew(agent); + if (agent->device_type() == Agent::kAmdCpuDevice) { + clear_flags |= HSA_SVM_FLAG_HOST_ACCESS; + } else { + attribs.push_back(kmtPair(HSA_SVM_ATTR_NO_ACCESS, agent->node_id())); + } + break; + } + default: + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_ARGUMENT, + "Illegal or invalid attribute in Runtime::SetSvmAttrib"); + } + } + + // Merge CPU access properties - grant access if any CPU needs access. + // Probably wrong. + if (set_flags & HSA_SVM_FLAG_HOST_ACCESS) clear_flags &= ~HSA_SVM_FLAG_HOST_ACCESS; + + // Add flag updates + if (clear_flags) attribs.push_back(kmtPair(HSA_SVM_ATTR_CLR_FLAGS, clear_flags)); + if (set_flags) attribs.push_back(kmtPair(HSA_SVM_ATTR_SET_FLAGS, set_flags)); + + uint8_t* base = AlignDown((uint8_t*)ptr, 4096); + uint8_t* end = AlignUp((uint8_t*)ptr + size, 4096); + size_t len = end - base; + HSAKMT_STATUS error = hsaKmtSVMSetAttr(base, len, attribs.size(), &attribs[0]); + if (error != HSAKMT_STATUS_SUCCESS) + throw AMD::hsa_exception(HSA_STATUS_ERROR, "hsaKmtSVMSetAttr failed."); + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t Runtime::GetSvmAttrib(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count) { + std::vector attribs; + attribs.reserve(attribute_count); + + std::vector kmtIndices(attribute_count); + + bool getFlags = false; + + auto Convert = [&](uint64_t value) -> Agent* { + hsa_agent_t handle = {value}; + Agent* agent = Agent::Convert(handle); + if ((agent == nullptr) || !agent->IsValid()) + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_AGENT, + "Invalid agent handle in Runtime::GetSvmAttrib."); + return agent; + }; + + auto kmtPair = [](uint32_t attrib, uint32_t value) { + HSA_SVM_ATTRIBUTE pair = {attrib, value}; + return pair; + }; + + for (uint32_t i = 0; i < attribute_count; i++) { + auto& attrib = attribute_list[i].attribute; + auto& value = attribute_list[i].value; + + switch (attrib) { + case HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG: + case HSA_AMD_SVM_ATTRIB_READ_ONLY: + case HSA_AMD_SVM_ATTRIB_HIVE_LOCAL: { + getFlags = true; + kmtIndices[i] = -1; + break; + } + case HSA_AMD_SVM_ATTRIB_MIGRATION_GRANULARITY: { + kmtIndices[i] = attribs.size(); + attribs.push_back(kmtPair(HSA_SVM_ATTR_GRANULARITY, 0)); + break; + } + case HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION: { + kmtIndices[i] = attribs.size(); + attribs.push_back(kmtPair(HSA_SVM_ATTR_PREFERRED_LOC, 0)); + break; + } + case HSA_AMD_SVM_ATTRIB_PREFETCH_LOCATION: { + value = Agent::Convert(GetSVMPrefetchAgent(ptr, size)).handle; + kmtIndices[i] = -1; + break; + } + case HSA_AMD_SVM_ATTRIB_ACCESS_QUERY: { + Agent* agent = Convert(value); + if (agent->device_type() == Agent::kAmdCpuDevice) { + getFlags = true; + kmtIndices[i] = -1; + } else { + kmtIndices[i] = attribs.size(); + attribs.push_back(kmtPair(HSA_SVM_ATTR_ACCESS, agent->node_id())); + } + break; + } + default: + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_ARGUMENT, + "Illegal or invalid attribute in Runtime::SetSvmAttrib"); + } + } + + if (getFlags) attribs.push_back(kmtPair(HSA_SVM_ATTR_SET_FLAGS, 0)); + + uint8_t* base = AlignDown((uint8_t*)ptr, 4096); + uint8_t* end = AlignUp((uint8_t*)ptr + size, 4096); + size_t len = end - base; + if (attribs.size() != 0) { + HSAKMT_STATUS error = hsaKmtSVMGetAttr(base, len, attribs.size(), &attribs[0]); + if (error != HSAKMT_STATUS_SUCCESS) + throw AMD::hsa_exception(HSA_STATUS_ERROR, "hsaKmtSVMGetAttr failed."); + } + + for (uint32_t i = 0; i < attribute_count; i++) { + auto& attrib = attribute_list[i].attribute; + auto& value = attribute_list[i].value; + + switch (attrib) { + case HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG: { + if (attribs[attribs.size() - 1].value & HSA_SVM_FLAG_COHERENT) + value = HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED; + else + value = HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED; + break; + } + case HSA_AMD_SVM_ATTRIB_READ_ONLY: { + value = (attribs[attribs.size() - 1].value & HSA_SVM_FLAG_GPU_RO); + break; + } + case HSA_AMD_SVM_ATTRIB_HIVE_LOCAL: { + value = (attribs[attribs.size() - 1].value & HSA_SVM_FLAG_HIVE_LOCAL); + break; + } + case HSA_AMD_SVM_ATTRIB_MIGRATION_GRANULARITY: { + value = attribs[kmtIndices[i]].value; + break; + } + case HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION: { + uint64_t node = attribs[kmtIndices[i]].value; + Agent* agent = nullptr; + if (node != INVALID_NODEID) agent = agents_by_node_[node][0]; + value = Agent::Convert(agent).handle; + break; + } + case HSA_AMD_SVM_ATTRIB_PREFETCH_LOCATION: { + break; + } + case HSA_AMD_SVM_ATTRIB_ACCESS_QUERY: { + if (kmtIndices[i] == -1) { + if (attribs[attribs.size() - 1].value & HSA_SVM_FLAG_HOST_ACCESS) + attrib = HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE; + } else { + switch (attribs[kmtIndices[i]].type) { + case HSA_SVM_ATTR_ACCESS: + attrib = HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE; + break; + case HSA_SVM_ATTR_ACCESS_IN_PLACE: + attrib = HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE; + break; + case HSA_SVM_ATTR_NO_ACCESS: + attrib = HSA_AMD_SVM_ATTRIB_AGENT_NO_ACCESS; + break; + default: + assert(false && "Bad agent accessibility from KFD."); + } + } + break; + } + default: + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_ARGUMENT, + "Illegal or invalid attribute in Runtime::GetSvmAttrib"); + } + } + + return HSA_STATUS_SUCCESS; +} + +hsa_status_t Runtime::SvmPrefetch(void* ptr, size_t size, hsa_agent_t agent, + uint32_t num_dep_signals, const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal) { + uintptr_t base = reinterpret_cast(AlignDown(ptr, 4096)); + uintptr_t end = AlignUp(reinterpret_cast(ptr) + size, 4096); + size_t len = end - base; + + PrefetchOp* op = new PrefetchOp(); + MAKE_NAMED_SCOPE_GUARD(OpGuard, [&]() { delete op; }); + + Agent* dest = Agent::Convert(agent); + if (dest->device_type() == Agent::kAmdCpuDevice) + op->node_id = 0; + else + op->node_id = dest->node_id(); + + op->base = reinterpret_cast(base); + op->size = len; + op->completion = completion_signal; + if (num_dep_signals > 1) { + op->remaining_deps = num_dep_signals - 1; + for (int i = 0; i < num_dep_signals - 1; i++) op->dep_signals.push_back(dep_signals[i]); + } else { + op->remaining_deps = 0; + } + + { + ScopedAcquire lock(&prefetch_lock_); + // Remove all fully overlapped and trim partially overlapped ranges. + // Get iteration bounds + auto start = prefetch_map_.upper_bound(base); + if (start != prefetch_map_.begin()) start--; + auto stop = prefetch_map_.lower_bound(end); + + auto isEndNode = [&](decltype(start) node) { return node->second.next == prefetch_map_.end(); }; + auto isFirstNode = [&](decltype(start) node) { + return node->second.prev == prefetch_map_.end(); + }; + + // Trim and remove old ranges. + while (start != stop) { + uintptr_t startBase = start->first; + uintptr_t startEnd = startBase + start->second.bytes; + + auto ibase = Max(startBase, base); + auto iend = Min(startEnd, end); + // Check for overlap + if (ibase < iend) { + // Second range check + if (iend < startEnd) { + auto ret = prefetch_map_.insert( + std::make_pair(iend, PrefetchRange(startEnd - iend, start->second.op))); + assert(ret.second && "Prefetch map insert failed during range split."); + + auto it = ret.first; + it->second.prev = start; + it->second.next = start->second.next; + start->second.next = it; + if (!isEndNode(it)) it->second.next->second.prev = it; + } + + // Is the first interval of the old range valid + if (startBase < ibase) { + start->second.bytes = ibase - startBase; + } else { + if (isFirstNode(start)) { + start->second.op->prefetch_map_entry = start->second.next; + if (!isEndNode(start)) start->second.next->second.prev = prefetch_map_.end(); + } else { + start->second.prev->second.next = start->second.next; + if (!isEndNode(start)) start->second.next->second.prev = start->second.prev; + } + prefetch_map_.erase(start); + } + } + start++; + } + + // Insert new range. + auto ret = prefetch_map_.insert(std::make_pair(base, PrefetchRange(len, op))); + assert(ret.second && "Prefetch map insert failed."); + + auto it = ret.first; + op->prefetch_map_entry = it; + it->second.next = it->second.prev = prefetch_map_.end(); + } + + // Remove the prefetch's ranges from the map. + static auto removePrefetchRanges = [](PrefetchOp* op) { + ScopedAcquire lock(&Runtime::runtime_singleton_->prefetch_lock_); + auto it = op->prefetch_map_entry; + while (it != Runtime::runtime_singleton_->prefetch_map_.end()) { + auto next = it->second.next; + Runtime::runtime_singleton_->prefetch_map_.erase(it); + it = next; + } + }; + + // Prefetch Signal handler for synchronization. + static hsa_amd_signal_handler signal_handler = [](hsa_signal_value_t value, void* arg) { + PrefetchOp* op = reinterpret_cast(arg); + + if (op->remaining_deps > 0) { + op->remaining_deps--; + Runtime::runtime_singleton_->SetAsyncSignalHandler( + op->dep_signals[op->remaining_deps], HSA_SIGNAL_CONDITION_EQ, 0, signal_handler, arg); + return false; + } + + HSA_SVM_ATTRIBUTE attrib; + attrib.type = HSA_SVM_ATTR_PREFETCH_LOC; + attrib.value = op->node_id; + HSAKMT_STATUS error = hsaKmtSVMSetAttr(op->base, op->size, 1, &attrib); + assert(error == HSAKMT_STATUS_SUCCESS && "KFD Prefetch failed."); + + removePrefetchRanges(op); + + if (op->completion.handle != 0) Signal::Convert(op->completion)->SubRelaxed(1); + delete op; + + return false; + }; + + auto no_dependencies = [](void* arg) { signal_handler(0, arg); }; + + MAKE_NAMED_SCOPE_GUARD(RangeGuard, [&]() { removePrefetchRanges(op); }); + + hsa_status_t err; + if (num_dep_signals == 0) + err = AMD::hsa_amd_async_function(no_dependencies, op); + else + err = SetAsyncSignalHandler(dep_signals[num_dep_signals - 1], HSA_SIGNAL_CONDITION_EQ, 0, + signal_handler, op); + if (err != HSA_STATUS_SUCCESS) throw AMD::hsa_exception(err, "Signal handler unable to be set."); + + RangeGuard.Dismiss(); + OpGuard.Dismiss(); + return HSA_STATUS_SUCCESS; +} + +Agent* Runtime::GetSVMPrefetchAgent(void* ptr, size_t size) { + uintptr_t base = reinterpret_cast(AlignDown(ptr, 4096)); + uintptr_t end = AlignUp(reinterpret_cast(ptr) + size, 4096); + size_t len = end - base; + + std::vector> holes; + + ScopedAcquire lock(&Runtime::runtime_singleton_->prefetch_lock_); + auto start = prefetch_map_.upper_bound(base); + if (start != prefetch_map_.begin()) start--; + auto stop = prefetch_map_.lower_bound(end); + + // KFD returns -1 for no or mixed destinations. + uint32_t prefetch_node = -2; + if (start != stop) { + prefetch_node = start->second.op->node_id; + } + + while (start != stop) { + uintptr_t startBase = start->first; + uintptr_t startEnd = startBase + start->second.bytes; + + auto ibase = Max(base, startBase); + auto iend = Min(end, startEnd); + // Check for intersection with the query + if (ibase < iend) { + // If prefetch locations are different then we report null agent. + if (prefetch_node != start->second.op->node_id) return nullptr; + + // Push leading gap to an array for checking KFD. + if (base < ibase) holes.push_back(std::make_pair(base, ibase - base)); + + // Trim query range. + base = iend; + } + start++; + } + if (base < end) holes.push_back(std::make_pair(base, end - base)); + + HSA_SVM_ATTRIBUTE attrib; + attrib.type = HSA_SVM_ATTR_PREFETCH_LOC; + for (auto& range : holes) { + HSAKMT_STATUS error = + hsaKmtSVMGetAttr(reinterpret_cast(range.first), range.second, 1, &attrib); + assert(error == HSAKMT_STATUS_SUCCESS && "KFD prefetch query failed."); + + if (attrib.value == -1) return nullptr; + if (prefetch_node == -2) prefetch_node = attrib.value; + if (prefetch_node != attrib.value) return nullptr; + } + + assert(prefetch_node != -2 && "prefetch_node was not updated."); + assert(prefetch_node != -1 && "Should have already returned."); + return agents_by_node_[prefetch_node][0]; +} + } // namespace core } // namespace rocr diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h index 6ea7651ac7..88fbe93be2 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h @@ -56,6 +56,11 @@ class Flag { public: enum SDMA_OVERRIDE { SDMA_DISABLE, SDMA_ENABLE, SDMA_DEFAULT }; + // The values are meaningful and chosen to satisfy the thunk API. + enum XNACK_REQUEST { XNACK_DISABLE = 0, XNACK_ENABLE = 1, XNACK_UNCHANGED = 2 }; + static_assert(XNACK_DISABLE == 0, "XNACK_REQUEST enum values improperly changed."); + static_assert(XNACK_ENABLE == 1, "XNACK_REQUEST enum values improperly changed."); + explicit Flag() { Refresh(); } virtual ~Flag() {} @@ -136,6 +141,11 @@ class Flag { var = os::GetEnvVar("HSA_IGNORE_SRAMECC_MISREPORT"); check_sramecc_validity_ = (var == "1") ? false : true; + + // Legal values are zero "0" or one "1". Any other value will + // be interpreted as not defining the env variable. + var = os::GetEnvVar("HSA_XNACK"); + xnack_ = (var == "0") ? XNACK_DISABLE : ((var == "1") ? XNACK_ENABLE : XNACK_UNCHANGED); } bool check_flat_scratch() const { return check_flat_scratch_; } @@ -189,6 +199,8 @@ class Flag { bool check_sramecc_validity() const { return check_sramecc_validity_; } + XNACK_REQUEST xnack() const { return xnack_; } + private: bool check_flat_scratch_; bool enable_vm_fault_message_; @@ -222,6 +234,9 @@ class Flag { size_t force_sdma_size_; + // Indicates user preference for Xnack state. + XNACK_REQUEST xnack_; + DISALLOW_COPY_AND_ASSIGN(Flag); }; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def b/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def index 4666121cfb..eb853410e5 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def +++ b/projects/rocr-runtime/runtime/hsa-runtime/hsacore.so.def @@ -222,6 +222,9 @@ global: hsa_amd_deregister_deallocation_callback; hsa_amd_signal_value_pointer; _amdgpu_r_debug; + hsa_amd_svm_attributes_set; + hsa_amd_svm_attributes_get; + hsa_amd_svm_prefetch_async; local: *; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h index d8fdd472b9..fe1facca44 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa.h @@ -480,7 +480,21 @@ typedef enum { /** * String containing the ROCr build identifier. */ - HSA_AMD_SYSTEM_INFO_BUILD_VERSION = 0x200 + HSA_AMD_SYSTEM_INFO_BUILD_VERSION = 0x200, + /** + * Returns true if hsa_amd_svm_* APIs are supported by the driver. The type of + * this attribute is bool. + */ + HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED = 0x201, + // TODO: Should this be per Agent? + /** + * Returns true if all Agents have access to system allocated memory (such as + * that allocated by mmap, malloc, or new) by default. + * If false then system allocated memory may only be made SVM accessible to + * an Agent by declaration of accessibility with hsa_amd_svm_set_attributes. + * The type of this attribute is bool. + */ + HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT = 0x202 } hsa_system_info_t; /** diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h index bf3e9197ed..35dd21bfa5 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_api_trace.h @@ -183,6 +183,9 @@ struct AmdExtTable { decltype(hsa_amd_register_deallocation_callback)* hsa_amd_register_deallocation_callback_fn; decltype(hsa_amd_deregister_deallocation_callback)* hsa_amd_deregister_deallocation_callback_fn; decltype(hsa_amd_signal_value_pointer)* hsa_amd_signal_value_pointer_fn; + decltype(hsa_amd_svm_attributes_set)* hsa_amd_svm_attributes_set_fn; + decltype(hsa_amd_svm_attributes_get)* hsa_amd_svm_attributes_get_fn; + decltype(hsa_amd_svm_prefetch_async)* hsa_amd_svm_prefetch_async_fn; }; // Table to export HSA Core Runtime Apis diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h index 9df7c49aee..a0bc5d1d99 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -2116,6 +2116,162 @@ hsa_status_t HSA_API hsa_amd_register_deallocation_callback(void* ptr, hsa_status_t HSA_API hsa_amd_deregister_deallocation_callback(void* ptr, hsa_amd_deallocation_callback_t callback); +typedef enum hsa_amd_svm_model_s { + /** + * Updates to memory with this attribute conform to HSA memory consistency model. + */ + HSA_AMD_SVM_GLOBAL_FLAG_FINE_GRAINED = 0, + /** + * Writes to memory with this attribute can be performed by a single agent at a time. + */ + HSA_AMD_SVM_GLOBAL_FLAG_COARSE_GRAINED = 1 +} hsa_amd_svm_model_t; + +typedef enum hsa_amd_svm_attribute_s { + // Memory model attribute. + // Type of this attribute is hsa_amd_svm_model_t. + HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG = 0, + // Marks the range read only. This allows multiple physical copies to be + // placed local to each accessing device. + // Type of this attribute is bool. + HSA_AMD_SVM_ATTRIB_READ_ONLY = 1, + // Automatic migrations should attempt to keep the memory within the xgmi hive + // containing accessible agents. + // Type of this attribute is bool. + HSA_AMD_SVM_ATTRIB_HIVE_LOCAL = 2, + // Page granularity to migrate at once. Page granularity is specified as + // log2(page_count). + // Type of this attribute is uint64_t. + HSA_AMD_SVM_ATTRIB_MIGRATION_GRANULARITY = 3, + // Physical location to prefer when automatic migration occurs. + // Set to the null agent handle (handle == 0) to indicate there + // is no preferred location. + // Type of this attribute is hsa_agent_t. + HSA_AMD_SVM_ATTRIB_PREFERRED_LOCATION = 4, + // This attribute can not be used in ::hsa_amd_svm_attributes_set (see + // ::hsa_amd_svm_prefetch_async). + // Physical location of most recent prefetch command. + // If the prefetch location has not been set or is not uniform across the + // address range then returned hsa_agent_t::handle will be 0. + // Querying this attribute will return the destination agent of the most + // recent ::hsa_amd_svm_prefetch_async targeting the address range. If + // multiple async prefetches have been issued targeting the region and the + // most recently issued prefetch has completed then the query will return + // the location of the most recently completed prefetch. + // Type of this attribute is hsa_agent_t. + HSA_AMD_SVM_ATTRIB_PREFETCH_LOCATION = 5, + // This attribute can not be used in ::hsa_amd_svm_attributes_get. + // Enables an agent for access to the range. Access may incur a page fault + // and associated memory migration. Either this or + // HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE is required prior to SVM + // access if HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT is false. + // Type of this attribute is hsa_agent_t. + HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE = 0x200, + // This attribute can not be used in ::hsa_amd_svm_attributes_get. + // Enables an agent for access to the range without page faults. Access + // will not incur a page fault and will not cause access based migration. + // and associated memory migration. Either this or + // HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE is required prior to SVM access if + // HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT is false. + // Type of this attribute is hsa_agent_t. + HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE = 0x201, + // This attribute can not be used in ::hsa_amd_svm_attributes_get. + // Denies an agent access to the memory range. Access will cause a terminal + // segfault. + // Type of this attribute is hsa_agent_t. + HSA_AMD_SVM_ATTRIB_AGENT_NO_ACCESS = 0x202, + // This attribute can not be used in ::hsa_amd_svm_attributes_set. + // Returns the access attribute associated with the agent. + // The agent to query must be set in the attribute value field. + // The attribute enum will be replaced with the agent's current access + // attribute for the address range. + // TODO: Clarify KFD return value for non-uniform access attribute. + // Type of this attribute is hsa_agent_t. + HSA_AMD_SVM_ATTRIB_ACCESS_QUERY = 0x203, +} hsa_amd_svm_attribute_t; + +// List type for hsa_amd_svm_attributes_set/get. +typedef struct hsa_amd_svm_attribute_pair_s { + // hsa_amd_svm_attribute_t value. + uint64_t attribute; + // Attribute value. Bit values should be interpreted according to the type + // given in the associated attribute description. + uint64_t value; +} hsa_amd_svm_attribute_pair_t; + +/** + * @brief Sets SVM memory attributes. + * + * If HSA_AMD_SYSTEM_INFO_SVM_ACCESSIBLE_BY_DEFAULT returns false then enabling + * access to an Agent via this API (setting HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE + * or HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE) is required prior to SVM + * memory access by that Agent. + * + * Attributes HSA_AMD_SVM_ATTRIB_ACCESS_QUERY and HSA_AMD_SVM_ATTRIB_PREFETCH_LOCATION + * may not be used with this API. + * + * @param[in] ptr Will be aligned down to nearest page boundary. + * + * @param[in] size Will be aligned up to nearest page boundary. + * + * @param[in] attribute_list List of attributes to set for the address range. + * + * @param[in] attribute_count Length of @p attribute_list. + */ +hsa_status_t hsa_amd_svm_attributes_set(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count); + +/** + * @brief Gets SVM memory attributes. + * + * Attributes HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE, + * HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE_IN_PLACE and + * HSA_AMD_SVM_ATTRIB_PREFETCH_LOCATION may not be used with this API. + * + * Note that attribute HSA_AMD_SVM_ATTRIB_ACCESS_QUERY takes as input an + * hsa_agent_t and returns the current access type through its attribute field. + * + * @param[in] ptr Will be aligned down to nearest page boundary. + * + * @param[in] size Will be aligned up to nearest page boundary. + * + * @param[in] attribute_list List of attributes to set for the address range. + * + * @param[in] attribute_count Length of @p attribute_list. + */ +hsa_status_t hsa_amd_svm_attributes_get(void* ptr, size_t size, + hsa_amd_svm_attribute_pair_t* attribute_list, + size_t attribute_count); + +/** + * @brief Asynchronously migrates memory to an agent. + * + * Schedules memory migration to @p agent when @p dep_signals have been observed equal to zero. + * @p completion_signal will decrement when the migration is complete. + * + * @param[in] ptr Will be aligned down to nearest page boundary. + * + * @param[in] size Will be aligned up to nearest page boundary. + * + * @param[in] agent Agent to migrate to. + * + * @param[in] num_dep_signals Number of dependent signals. Can be 0. + * + * @param[in] dep_signals List of signals that must be waited on before the migration + * operation starts. The migration will start after every signal has been observed with + * the value 0. If @p num_dep_signals is 0, this argument is ignored. + * + * @param[in] completion_signal Signal used to indicate completion of the migration + * operation. When the migration operation is finished, the value of the signal is + * decremented. The runtime indicates that an error has occurred during the copy + * operation by setting the value of the completion signal to a negative + * number. If no completion signal is required this handle may be null. + */ +hsa_status_t hsa_amd_svm_prefetch_async(void* ptr, size_t size, hsa_agent_t agent, + uint32_t num_dep_signals, const hsa_signal_t* dep_signals, + hsa_signal_t completion_signal); + #ifdef __cplusplus } // end extern "C" block #endif