diff --git a/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp b/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp index 4a6dcd6001..09989b91d3 100644 --- a/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp +++ b/runtime/hsa-runtime/core/driver/kfd/amd_kfd_driver.cpp @@ -88,18 +88,18 @@ KfdDriver::KfdDriver(std::string devnode_name) hsa_status_t KfdDriver::Init() { HSAKMT_STATUS ret = - hsaKmtRuntimeEnable(&_amdgpu_r_debug, core::Runtime::runtime_singleton_->flag().debug()); + HSAKMT_CALL(hsaKmtRuntimeEnable(&_amdgpu_r_debug, core::Runtime::runtime_singleton_->flag().debug())); if (ret != HSAKMT_STATUS_SUCCESS && ret != HSAKMT_STATUS_NOT_SUPPORTED) return HSA_STATUS_ERROR; uint32_t caps_mask = 0; - if (hsaKmtGetRuntimeCapabilities(&caps_mask) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + if (HSAKMT_CALL(hsaKmtGetRuntimeCapabilities(&caps_mask)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; core::Runtime::runtime_singleton_->KfdVersion( ret != HSAKMT_STATUS_NOT_SUPPORTED, !!(caps_mask & HSA_RUNTIME_ENABLE_CAPS_SUPPORTS_CORE_DUMP_MASK)); - if (hsaKmtGetVersion(&version_) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + if (HSAKMT_CALL(hsaKmtGetVersion(&version_)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; if (version_.KernelInterfaceMajorVersion == kfd_version_major_min && version_.KernelInterfaceMinorVersion < kfd_version_major_min) @@ -117,10 +117,10 @@ hsa_status_t KfdDriver::Init() { } hsa_status_t KfdDriver::ShutDown() { - HSAKMT_STATUS ret = hsaKmtRuntimeDisable(); + HSAKMT_STATUS ret = HSAKMT_CALL(hsaKmtRuntimeDisable()); if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; - ret = hsaKmtReleaseSystemProperties(); + ret = HSAKMT_CALL(hsaKmtReleaseSystemProperties()); if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; @@ -143,32 +143,32 @@ hsa_status_t KfdDriver::QueryKernelModeDriver(core::DriverQuery query) { } hsa_status_t KfdDriver::Open() { - return hsaKmtOpenKFD() == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS + return HSAKMT_CALL(hsaKmtOpenKFD()) == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR; } hsa_status_t KfdDriver::Close() { - return hsaKmtCloseKFD() == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS + return HSAKMT_CALL(hsaKmtCloseKFD()) == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR; } hsa_status_t KfdDriver::GetSystemProperties(HsaSystemProperties& sys_props) const { - if (hsaKmtReleaseSystemProperties() != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + if (HSAKMT_CALL(hsaKmtReleaseSystemProperties()) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; - if (hsaKmtAcquireSystemProperties(&sys_props) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + if (HSAKMT_CALL(hsaKmtAcquireSystemProperties(&sys_props)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; return HSA_STATUS_SUCCESS; } hsa_status_t KfdDriver::GetNodeProperties(HsaNodeProperties& node_props, uint32_t node_id) const { - if (hsaKmtGetNodeProperties(node_id, &node_props) != HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtGetNodeProperties(node_id, &node_props)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; return HSA_STATUS_SUCCESS; } hsa_status_t KfdDriver::GetEdgeProperties(std::vector& io_link_props, uint32_t node_id) const { - if (hsaKmtGetNodeIoLinkProperties(node_id, io_link_props.size(), io_link_props.data()) != + if (HSAKMT_CALL(hsaKmtGetNodeIoLinkProperties(node_id, io_link_props.size(), io_link_props.data())) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; return HSA_STATUS_SUCCESS; @@ -182,7 +182,7 @@ hsa_status_t KfdDriver::GetMemoryProperties(uint32_t node_id, std::vector& mem_props) const { if (!mem_props.data()) return HSA_STATUS_ERROR_INVALID_ARGUMENT; - if (hsaKmtGetNodeMemoryProperties(node_id, mem_props.size(), mem_props.data()) != + if (HSAKMT_CALL(hsaKmtGetNodeMemoryProperties(node_id, mem_props.size(), mem_props.data())) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; @@ -193,7 +193,7 @@ hsa_status_t KfdDriver::GetCacheProperties(uint32_t node_id, uint32_t processor_ std::vector& cache_props) const { if (!cache_props.data()) return HSA_STATUS_ERROR_INVALID_ARGUMENT; - if (hsaKmtGetNodeCacheProperties(node_id, processor_id, cache_props.size(), cache_props.data()) != + if (HSAKMT_CALL(hsaKmtGetNodeCacheProperties(node_id, processor_id, cache_props.size(), cache_props.data())) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; @@ -269,7 +269,7 @@ KfdDriver::AllocateMemory(const core::MemoryRegion &mem_region, *mem = m_region.fragment_alloc(size); if ((alloc_flags & core::MemoryRegion::AllocateAsan) && - hsaKmtReplaceAsanHeaderPage(*mem) != HSAKMT_STATUS_SUCCESS) { + HSAKMT_CALL(hsaKmtReplaceAsanHeaderPage(*mem)) != HSAKMT_STATUS_SUCCESS) { m_region.fragment_free(*mem); *mem = nullptr; return HSA_STATUS_ERROR_OUT_OF_RESOURCES; @@ -339,7 +339,7 @@ KfdDriver::AllocateMemory(const core::MemoryRegion &mem_region, } if ((alloc_flags & core::MemoryRegion::AllocateAsan) && - hsaKmtReplaceAsanHeaderPage(*mem) != HSAKMT_STATUS_SUCCESS) { + HSAKMT_CALL(hsaKmtReplaceAsanHeaderPage(*mem)) != HSAKMT_STATUS_SUCCESS) { FreeKfdMemory(*mem, size); *mem = nullptr; return HSA_STATUS_ERROR_OUT_OF_RESOURCES; @@ -367,7 +367,7 @@ hsa_status_t KfdDriver::ExportDMABuf(void *mem, size_t size, int *dmabuf_fd, size_t *offset) { int dmabuf_fd_res = -1; size_t offset_res = 0; - if (hsaKmtExportDMABufHandle(mem, size, &dmabuf_fd_res, &offset_res) != + if (HSAKMT_CALL(hsaKmtExportDMABufHandle(mem, size, &dmabuf_fd_res, &offset_res)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; @@ -431,13 +431,13 @@ hsa_status_t KfdDriver::ReleaseShareableHandle(core::ShareableHandle &handle) { } hsa_status_t KfdDriver::SPMAcquire(uint32_t preferred_node_id) const { - if (hsaKmtSPMAcquire(preferred_node_id) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + if (HSAKMT_CALL(hsaKmtSPMAcquire(preferred_node_id)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; return HSA_STATUS_SUCCESS; } hsa_status_t KfdDriver::SPMRelease(uint32_t preferred_node_id) const { - if (hsaKmtSPMRelease(preferred_node_id) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + if (HSAKMT_CALL(hsaKmtSPMRelease(preferred_node_id)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; return HSA_STATUS_SUCCESS; } @@ -445,8 +445,8 @@ hsa_status_t KfdDriver::SPMRelease(uint32_t preferred_node_id) const { hsa_status_t KfdDriver::SPMSetDestBuffer(uint32_t preferred_node_id, uint32_t size_bytes, uint32_t* timeout, uint32_t* size_copied, void* dest_mem_addr, bool* is_spm_data_loss) const { - if (hsaKmtSPMSetDestBuffer(preferred_node_id, size_bytes, timeout, size_copied, dest_mem_addr, - is_spm_data_loss) != HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtSPMSetDestBuffer(preferred_node_id, size_bytes, timeout, size_copied, dest_mem_addr, + is_spm_data_loss)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; return HSA_STATUS_SUCCESS; @@ -455,7 +455,7 @@ hsa_status_t KfdDriver::SPMSetDestBuffer(uint32_t preferred_node_id, uint32_t si void *KfdDriver::AllocateKfdMemory(const HsaMemFlags &flags, uint32_t node_id, size_t size) { void *mem = nullptr; - const HSAKMT_STATUS status = hsaKmtAllocMemory(node_id, size, flags, &mem); + const HSAKMT_STATUS status = HSAKMT_CALL(hsaKmtAllocMemory(node_id, size, flags, &mem)); return (status == HSAKMT_STATUS_SUCCESS) ? mem : nullptr; } @@ -465,7 +465,7 @@ bool KfdDriver::FreeKfdMemory(void *mem, size_t size) { return false; } - if (hsaKmtFreeMemory(mem, size) != HSAKMT_STATUS_SUCCESS) { + if (HSAKMT_CALL(hsaKmtFreeMemory(mem, size)) != HSAKMT_STATUS_SUCCESS) { debug_print("Failed to free ptr:%p size:%lu\n", mem, size); return false; } @@ -481,15 +481,15 @@ bool KfdDriver::MakeKfdMemoryResident(size_t num_node, const uint32_t *nodes, *alternate_va = 0; - HSAKMT_STATUS kmt_status(hsaKmtMapMemoryToGPUNodes( + HSAKMT_STATUS kmt_status(HSAKMT_CALL(hsaKmtMapMemoryToGPUNodes( const_cast(mem), size, alternate_va, map_flag, num_node, - const_cast(nodes))); + const_cast(nodes)))); return (kmt_status == HSAKMT_STATUS_SUCCESS); } void KfdDriver::MakeKfdMemoryUnresident(const void *mem) { - hsaKmtUnmapMemoryToGPU(const_cast(mem)); + HSAKMT_CALL(hsaKmtUnmapMemoryToGPU(const_cast(mem))); } bool KfdDriver::BindXnackMode() { @@ -501,7 +501,7 @@ bool KfdDriver::BindXnackMode() { // Call to driver can fail and is a supported feature HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; if (config_xnack) { - status = hsaKmtSetXNACKMode(mode); + status = HSAKMT_CALL(hsaKmtSetXNACKMode(mode)); if (status == HSAKMT_STATUS_SUCCESS) { return (mode != Flag::XNACK_DISABLE); } @@ -510,7 +510,7 @@ bool KfdDriver::BindXnackMode() { // Get Xnack mode of devices bound by driver. This could happen // when a call to SET Xnack mode fails or user has no particular // preference - status = hsaKmtGetXNACKMode(&mode); + status = HSAKMT_CALL(hsaKmtGetXNACKMode(&mode)); if (status != HSAKMT_STATUS_SUCCESS) { debug_print( "KFD does not support xnack mode query.\nROCr must assume " @@ -523,7 +523,7 @@ bool KfdDriver::BindXnackMode() { hsa_status_t KfdDriver::IsModelEnabled(bool* enable) const { // AIE does not support streaming performance monitor. HSAKMT_STATUS status = HSAKMT_STATUS_ERROR; - status = hsaKmtModelEnabled(enable); + status = HSAKMT_CALL(hsaKmtModelEnabled(enable)); if (status != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; diff --git a/runtime/hsa-runtime/core/inc/interrupt_signal.h b/runtime/hsa-runtime/core/inc/interrupt_signal.h index a774b095d9..a928d2c5ac 100644 --- a/runtime/hsa-runtime/core/inc/interrupt_signal.h +++ b/runtime/hsa-runtime/core/inc/interrupt_signal.h @@ -207,11 +207,7 @@ class InterruptSignal : private LocalSignal, public Signal { return rtti_id_; } - /// @brief Notify driver of signal value change if necessary. - __forceinline void SetEvent() { - std::atomic_signal_fence(std::memory_order_seq_cst); - if (InWaiting()) hsaKmtSetEvent(event_); - } + void SetEvent(); DISALLOW_COPY_AND_ASSIGN(InterruptSignal); }; diff --git a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index a4ff1577ac..4b25eb63a4 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -285,11 +285,11 @@ AqlQueue::AqlQueue(core::SharedQueue* shared_queue, GpuAgent* agent, size_t req_ HSAKMT_STATUS kmt_status; if (core::Runtime::runtime_singleton_->KfdVersion().supports_exception_debugging) { queue_rsrc.ErrorReason = &exception_signal_->signal_.value; - kmt_status = hsaKmtCreateQueueExt(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, 0, ring_buf_, - ring_buf_alloc_bytes_, queue_event(), &queue_rsrc); + kmt_status = HSAKMT_CALL(hsaKmtCreateQueueExt(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, 0, ring_buf_, + ring_buf_alloc_bytes_, queue_event(), &queue_rsrc)); } else { - kmt_status = hsaKmtCreateQueueExt(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, 0, ring_buf_, - ring_buf_alloc_bytes_, NULL, &queue_rsrc); + kmt_status = HSAKMT_CALL(hsaKmtCreateQueueExt(node_id, HSA_QUEUE_COMPUTE_AQL, 100, priority_, 0, ring_buf_, + ring_buf_alloc_bytes_, NULL, &queue_rsrc)); } if (kmt_status != HSAKMT_STATUS_SUCCESS) throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES, @@ -302,7 +302,7 @@ AqlQueue::AqlQueue(core::SharedQueue* shared_queue, GpuAgent* agent, size_t req_ amd_queue_.hsa_queue.id = this->GetQueueId(); queue_id_ = queue_rsrc.QueueId; - MAKE_NAMED_SCOPE_GUARD(QueueGuard, [&]() { hsaKmtDestroyQueue(queue_id_); }); + MAKE_NAMED_SCOPE_GUARD(QueueGuard, [&]() { HSAKMT_CALL(hsaKmtDestroyQueue(queue_id_)); }); amd_queue_.scratch_max_use_index = UINT64_MAX; amd_queue_.alt_scratch_max_use_index = UINT64_MAX; @@ -837,14 +837,14 @@ int AqlQueue::CreateRingBufferFD(const char* ring_buf_shm_path, void AqlQueue::Suspend() { suspended_ = true; - auto err = hsaKmtUpdateQueue(queue_id_, 0, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL); + auto err = HSAKMT_CALL(hsaKmtUpdateQueue(queue_id_, 0, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL)); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtUpdateQueue failed."); } void AqlQueue::Resume() { if (suspended_) { suspended_ = false; - auto err = hsaKmtUpdateQueue(queue_id_, 100, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL); + auto err = HSAKMT_CALL(hsaKmtUpdateQueue(queue_id_, 100, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL)); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtUpdateQueue failed."); } } @@ -852,7 +852,7 @@ void AqlQueue::Resume() { hsa_status_t AqlQueue::Inactivate() { bool active = active_.exchange(false, std::memory_order_relaxed); if (active) { - auto err = hsaKmtDestroyQueue(queue_id_); + auto err = HSAKMT_CALL(hsaKmtDestroyQueue(queue_id_)); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtDestroyQueue failed."); atomic::Fence(std::memory_order_acquire); } @@ -865,7 +865,7 @@ hsa_status_t AqlQueue::SetPriority(HSA_QUEUE_PRIORITY priority) { } priority_ = priority; - auto err = hsaKmtUpdateQueue(queue_id_, 100, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL); + auto err = HSAKMT_CALL(hsaKmtUpdateQueue(queue_id_, 100, priority_, ring_buf_, ring_buf_alloc_bytes_, NULL)); return (err == HSAKMT_STATUS_SUCCESS ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR_OUT_OF_RESOURCES); } @@ -1525,7 +1525,7 @@ hsa_status_t AqlQueue::SetCUMasking(uint32_t num_cu_mask_count, const uint32_t* } HSAKMT_STATUS ret = - hsaKmtSetQueueCUMask(queue_id_, mask.size() * 32, reinterpret_cast(&mask[0])); + HSAKMT_CALL(hsaKmtSetQueueCUMask(queue_id_, mask.size() * 32, reinterpret_cast(&mask[0]))); if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; } @@ -2035,7 +2035,7 @@ void AqlQueue::InitScratchSRD() { hsa_status_t AqlQueue::EnableGWS(int gws_slot_count) { uint32_t discard; - auto status = hsaKmtAllocQueueGWS(queue_id_, gws_slot_count, &discard); + auto status = HSAKMT_CALL(hsaKmtAllocQueueGWS(queue_id_, gws_slot_count, &discard)); if (status != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; amd_queue_.hsa_queue.type = HSA_QUEUE_TYPE_COOPERATIVE; return HSA_STATUS_SUCCESS; diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index 0e212b20b6..d5f9187121 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -195,10 +195,10 @@ hsa_status_t BlitSdma: // boolean flag const HSA_QUEUE_TYPE kQueueType_ = rec_eng >= 0 ? HSA_QUEUE_SDMA_BY_ENG_ID : (use_xgmi ? HSA_QUEUE_SDMA_XGMI : HSA_QUEUE_SDMA); - if (HSAKMT_STATUS_SUCCESS != hsaKmtCreateQueueExt(agent_->node_id(), kQueueType_, 100, + if (HSAKMT_STATUS_SUCCESS != HSAKMT_CALL(hsaKmtCreateQueueExt(agent_->node_id(), kQueueType_, 100, HSA_QUEUE_PRIORITY_MAXIMUM, rec_eng, queue_start_addr_, kQueueSize, NULL, - &queue_resource_)) { + &queue_resource_))) { return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } @@ -226,7 +226,7 @@ hsa_status_t BlitSdma: if (queue_resource_.QueueId != 0) { // Release queue resources from the kernel - auto err = hsaKmtDestroyQueue(queue_resource_.QueueId); + auto err = HSAKMT_CALL(hsaKmtDestroyQueue(queue_resource_.QueueId)); assert(err == HSAKMT_STATUS_SUCCESS); memset(&queue_resource_, 0, sizeof(queue_resource_)); } diff --git a/runtime/hsa-runtime/core/runtime/amd_filter_device.cpp b/runtime/hsa-runtime/core/runtime/amd_filter_device.cpp index 4c6d1ceea2..7b87317bfb 100644 --- a/runtime/hsa-runtime/core/runtime/amd_filter_device.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_filter_device.cpp @@ -95,7 +95,7 @@ void RvdFilter::BuildDeviceUuidList(uint32_t numNodes) { HsaNodeProperties props = {0}; for (HSAuint32 idx = 0; idx < numNodes; idx++) { // Query for node properties and ignore Cpu devices - status = hsaKmtGetNodeProperties(idx, &props); + status = HSAKMT_CALL(hsaKmtGetNodeProperties(idx, &props)); if (status != HSAKMT_STATUS_SUCCESS) { continue; } diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index d8b804d7e8..6dd4838e00 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -122,7 +122,7 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props, bool xna const bool is_apu_node = (properties_.NumCPUCores > 0); profile_ = (is_apu_node) ? HSA_PROFILE_FULL : HSA_PROFILE_BASE; - HSAKMT_STATUS err = hsaKmtGetClockCounters(node_id(), &t0_); + HSAKMT_STATUS err = HSAKMT_CALL(hsaKmtGetClockCounters(node_id(), &t0_)); t1_ = t0_; historical_clock_ratio_ = 0.0; assert(err == HSAKMT_STATUS_SUCCESS && "hsaGetClockCounters error"); @@ -440,8 +440,8 @@ void GpuAgent::InitRegionList() { std::vector mem_props(properties_.NumMemoryBanks); if (HSAKMT_STATUS_SUCCESS == - hsaKmtGetNodeMemoryProperties(node_id(), properties_.NumMemoryBanks, - &mem_props[0])) { + HSAKMT_CALL(hsaKmtGetNodeMemoryProperties(node_id(), properties_.NumMemoryBanks, + &mem_props[0]))) { for (uint32_t mem_idx = 0; mem_idx < properties_.NumMemoryBanks; ++mem_idx) { // Ignore the one(s) with unknown size. @@ -525,7 +525,7 @@ void GpuAgent::InitScratchPool() { void* scratch_base = nullptr; HSAKMT_STATUS err = - hsaKmtAllocMemory(node_id(), max_scratch_len, flags, &scratch_base); + HSAKMT_CALL(hsaKmtAllocMemory(node_id(), max_scratch_len, flags, &scratch_base)); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtAllocMemory(Scratch) failed"); assert(IsMultipleOf(scratch_base, 0x1000) && "Scratch base is not page aligned!"); @@ -562,7 +562,7 @@ void GpuAgent::ReserveScratch() } size_t available; - HSAKMT_STATUS err = hsaKmtAvailableMemory(node_id(), &available); + HSAKMT_STATUS err = HSAKMT_CALL(hsaKmtAvailableMemory(node_id(), &available)); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtAvailableMemory failed"); ScopedAcquire lock(&scratch_lock_); if (!scratch_cache_.reserved_bytes() && reserved_sz && available > 8 * reserved_sz) { @@ -570,7 +570,7 @@ void GpuAgent::ReserveScratch() void* reserved_base = scratch_pool_.alloc(reserved_sz); assert(reserved_base && "Could not allocate reserved memory"); - if (hsaKmtMapMemoryToGPU(reserved_base, reserved_sz, &alt_va) == HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtMapMemoryToGPU(reserved_base, reserved_sz, &alt_va)) == HSAKMT_STATUS_SUCCESS) scratch_cache_.reserve(reserved_sz, reserved_base); else throw AMD::hsa_exception(HSA_STATUS_ERROR_OUT_OF_RESOURCES, "Reserve scratch memory failed."); @@ -582,8 +582,8 @@ void GpuAgent::InitCacheList() { // Similar to getting CPU cache but here we use FComputeIdLo. cache_props_.resize(properties_.NumCaches); if (HSAKMT_STATUS_SUCCESS != - hsaKmtGetNodeCacheProperties(node_id(), properties_.FComputeIdLo, - properties_.NumCaches, &cache_props_[0])) { + HSAKMT_CALL(hsaKmtGetNodeCacheProperties(node_id(), properties_.FComputeIdLo, + properties_.NumCaches, &cache_props_[0]))) { cache_props_.clear(); } else { // Only store GPU D-cache. @@ -611,7 +611,7 @@ void GpuAgent::InitLibDrm() { HSAKMT_STATUS status; HsaAMDGPUDeviceHandle device_handle; - status = hsaKmtGetAMDGPUDeviceHandle(node_id(), &device_handle); + status = HSAKMT_CALL(hsaKmtGetAMDGPUDeviceHandle(node_id(), &device_handle)); if (status != HSAKMT_STATUS_SUCCESS) throw AMD::hsa_exception(HSA_STATUS_ERROR, "Agent creation failed.\nlibdrm get device handle failed.\n"); @@ -941,7 +941,7 @@ void GpuAgent::ReleaseResources() { scratch_cache_.free_reserve(); if (scratch_pool_.base() != NULL) { - hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size()); + HSAKMT_CALL(hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size())); } for (int i = 0; i < QueueCount; i++) @@ -1613,7 +1613,7 @@ hsa_status_t GpuAgent::GetInfo(hsa_agent_info_t attribute, void* value) const { HSAuint64 availableBytes; HSAKMT_STATUS status; - status = hsaKmtAvailableMemory(node_id(), &availableBytes); + status = HSAKMT_CALL(hsaKmtAvailableMemory(node_id(), &availableBytes)); if (status != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT; @@ -1878,7 +1878,7 @@ void GpuAgent::AcquireQueueMainScratch(ScratchInfo& scratch) { if (scratch.main_queue_base != nullptr) { HSAuint64 alternate_va; if ((profile_ == HSA_PROFILE_FULL) || - (hsaKmtMapMemoryToGPU(scratch.main_queue_base, scratch.main_size, &alternate_va) == + (HSAKMT_CALL(hsaKmtMapMemoryToGPU(scratch.main_queue_base, scratch.main_size, &alternate_va)) == HSAKMT_STATUS_SUCCESS)) { if (scratch.large) scratch_used_large_ += scratch.main_size; scratch_cache_.insertMain(scratch); @@ -1931,7 +1931,7 @@ void GpuAgent::AcquireQueueMainScratch(ScratchInfo& scratch) { HSAuint64 alternate_va; if ((base != nullptr) && ((profile_ == HSA_PROFILE_FULL) || - (hsaKmtMapMemoryToGPU(base, size, &alternate_va) == HSAKMT_STATUS_SUCCESS))) { + (HSAKMT_CALL(hsaKmtMapMemoryToGPU(base, size, &alternate_va)) == HSAKMT_STATUS_SUCCESS))) { // Scratch allocated and either full profile or map succeeded. scratch.main_queue_base = base; scratch.main_size = size; @@ -2011,7 +2011,7 @@ void GpuAgent::AcquireQueueAltScratch(ScratchInfo& scratch) { if (scratch.alt_queue_base != nullptr) { HSAuint64 alternate_va; if ((profile_ == HSA_PROFILE_FULL) || - (hsaKmtMapMemoryToGPU(scratch.alt_queue_base, scratch.alt_size, &alternate_va) == + (HSAKMT_CALL(hsaKmtMapMemoryToGPU(scratch.alt_queue_base, scratch.alt_size, &alternate_va)) == HSAKMT_STATUS_SUCCESS)) { scratch_cache_.insertAlt(scratch); return; @@ -2052,7 +2052,7 @@ void GpuAgent::ReleaseQueueAltScratch(ScratchInfo& scratch) { void GpuAgent::ReleaseScratch(void* base, size_t size, bool large) { if (profile_ == HSA_PROFILE_BASE) { - if (HSAKMT_STATUS_SUCCESS != hsaKmtUnmapMemoryToGPU(base)) { + if (HSAKMT_STATUS_SUCCESS != HSAKMT_CALL(hsaKmtUnmapMemoryToGPU(base))) { assert(false && "Unmap scratch subrange failed!"); } } @@ -2195,9 +2195,9 @@ bool GpuAgent::current_coherency_type(hsa_amd_coherency_type_t type) { type1 = HSA_CACHING_CACHED; } - if (hsaKmtSetMemoryPolicy(node_id(), type0, type1, + if (HSAKMT_CALL(hsaKmtSetMemoryPolicy(node_id(), type0, type1, reinterpret_cast(ape1_base_), - ape1_size_) != HSAKMT_STATUS_SUCCESS) { + ape1_size_)) != HSAKMT_STATUS_SUCCESS) { return false; } current_coherency_type_ = type; @@ -2213,7 +2213,7 @@ uint16_t GpuAgent::GetSdmaMicrocodeVersion() const { } void GpuAgent::SyncClocks() { - HSAKMT_STATUS err = hsaKmtGetClockCounters(node_id(), &t1_); + HSAKMT_STATUS err = HSAKMT_CALL(hsaKmtGetClockCounters(node_id(), &t1_)); assert(err == HSAKMT_STATUS_SUCCESS && "hsaGetClockCounters error"); } @@ -2264,7 +2264,7 @@ hsa_status_t GpuAgent::UpdateTrapHandlerWithPCS(pcs_sampling_data_t* pcs_hosttra // Bind the trap handler to this node. HSAKMT_STATUS retKmt = - hsaKmtSetTrapHandler(node_id(), trap_code_buf_, trap_code_buf_size_, tma_addr, tma_size); + HSAKMT_CALL(hsaKmtSetTrapHandler(node_id(), trap_code_buf_, trap_code_buf_size_, tma_addr, tma_size)); return (retKmt != HSAKMT_STATUS_SUCCESS) ? HSA_STATUS_ERROR : HSA_STATUS_SUCCESS; } @@ -2306,8 +2306,8 @@ void GpuAgent::BindTrapHandler() { } // Bind the trap handler to this node. - HSAKMT_STATUS err = hsaKmtSetTrapHandler(node_id(), trap_code_buf_, trap_code_buf_size_, - tma_addr, tma_size); + HSAKMT_STATUS err = HSAKMT_CALL(hsaKmtSetTrapHandler(node_id(), trap_code_buf_, trap_code_buf_size_, + tma_addr, tma_size)); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtSetTrapHandler() failed"); } @@ -2571,12 +2571,12 @@ hsa_status_t GpuAgent::PcSamplingIterateConfig(hsa_ven_amd_pcs_iterate_configura return HSA_STATUS_ERROR; // First query to get size of list needed - HSAKMT_STATUS ret = hsaKmtPcSamplingQueryCapabilities(node_id(), NULL, 0, &size); + HSAKMT_STATUS ret = HSAKMT_CALL(hsaKmtPcSamplingQueryCapabilities(node_id(), NULL, 0, &size)); if (ret != HSAKMT_STATUS_SUCCESS || size == 0) return HSA_STATUS_ERROR; std::vector sampleInfoList(size); - ret = hsaKmtPcSamplingQueryCapabilities(node_id(), sampleInfoList.data(), sampleInfoList.size(), - &size); + ret = HSAKMT_CALL(hsaKmtPcSamplingQueryCapabilities(node_id(), sampleInfoList.data(), sampleInfoList.size(), + &size)); if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; @@ -2604,7 +2604,7 @@ hsa_status_t GpuAgent::PcSamplingCreate(pcs::PcsRuntime::PcSamplingSession& sess // Pass the sampling information to the kernel driver to create PC // sampling session. - HSAKMT_STATUS retkmt = hsaKmtPcSamplingCreate(node_id(), &sampleInfo, &thunkId); + HSAKMT_STATUS retkmt = HSAKMT_CALL(hsaKmtPcSamplingCreate(node_id(), &sampleInfo, &thunkId)); if (retkmt != HSAKMT_STATUS_SUCCESS) { return (retkmt == HSAKMT_STATUS_KERNEL_ALREADY_OPENED) ? (hsa_status_t)HSA_STATUS_ERROR_RESOURCE_BUSY : HSA_STATUS_ERROR; @@ -2813,7 +2813,7 @@ hsa_status_t GpuAgent::PcSamplingCreateFromId(HsaPcSamplingTraceId ioctlId, hsa_status_t GpuAgent::PcSamplingDestroy(pcs::PcsRuntime::PcSamplingSession& session) { if (PcSamplingStop(session) != HSA_STATUS_SUCCESS) return HSA_STATUS_ERROR; - HSAKMT_STATUS retKmt = hsaKmtPcSamplingDestroy(node_id(), session.ThunkId()); + HSAKMT_STATUS retKmt = HSAKMT_CALL(hsaKmtPcSamplingDestroy(node_id(), session.ThunkId())); hsa_ven_amd_pcs_method_kind_t sampling_method = session.method(); pcs_data_t* pcs_data = nullptr; @@ -2912,7 +2912,7 @@ hsa_status_t GpuAgent::PcSamplingStart(pcs::PcsRuntime::PcSamplingSession& sessi } // Start the sampling session in the kernel driver - if (hsaKmtPcSamplingStart(node_id(), session.ThunkId()) == HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtPcSamplingStart(node_id(), session.ThunkId())) == HSAKMT_STATUS_SUCCESS) return HSA_STATUS_SUCCESS; debug_print("Failed to start PC sampling session with thunkId:%d\n", session.ThunkId()); @@ -2933,7 +2933,7 @@ hsa_status_t GpuAgent::PcSamplingStop(pcs::PcsRuntime::PcSamplingSession& sessio session.stop(); // Stop PC sampling in the kernel driver - HSAKMT_STATUS retKmt = hsaKmtPcSamplingStop(node_id(), session.ThunkId()); + HSAKMT_STATUS retKmt = HSAKMT_CALL(hsaKmtPcSamplingStop(node_id(), session.ThunkId())); if (retKmt != HSAKMT_STATUS_SUCCESS) throw AMD::hsa_exception(HSA_STATUS_ERROR, "Failed to stop PC Sampling session."); diff --git a/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp b/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp index a19670f153..d91c80702c 100644 --- a/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_memory_region.cpp @@ -62,11 +62,11 @@ bool MemoryRegion::RegisterMemory(void* ptr, size_t size, const HsaMemFlags& Mem assert(ptr != NULL); assert(size != 0); - const HSAKMT_STATUS status = hsaKmtRegisterMemoryWithFlags(ptr, size, MemFlags); + const HSAKMT_STATUS status = HSAKMT_CALL(hsaKmtRegisterMemoryWithFlags(ptr, size, MemFlags)); return (status == HSAKMT_STATUS_SUCCESS); } -void MemoryRegion::DeregisterMemory(void* ptr) { hsaKmtDeregisterMemory(ptr); } +void MemoryRegion::DeregisterMemory(void* ptr) { HSAKMT_CALL(hsaKmtDeregisterMemory(ptr)); } bool MemoryRegion::MakeKfdMemoryResident(size_t num_node, const uint32_t* nodes, const void* ptr, size_t size, uint64_t* alternate_va, @@ -75,14 +75,14 @@ bool MemoryRegion::MakeKfdMemoryResident(size_t num_node, const uint32_t* nodes, assert(nodes != NULL); *alternate_va = 0; - const HSAKMT_STATUS status = hsaKmtMapMemoryToGPUNodes( - const_cast(ptr), size, alternate_va, map_flag, num_node, const_cast(nodes)); + const HSAKMT_STATUS status = HSAKMT_CALL(hsaKmtMapMemoryToGPUNodes( + const_cast(ptr), size, alternate_va, map_flag, num_node, const_cast(nodes))); return (status == HSAKMT_STATUS_SUCCESS); } bool MemoryRegion::MakeKfdMemoryUnresident(const void* ptr) { - const HSAKMT_STATUS status = hsaKmtUnmapMemoryToGPU(const_cast(ptr)); + const HSAKMT_STATUS status = HSAKMT_CALL(hsaKmtUnmapMemoryToGPU(const_cast(ptr))); return (status == HSAKMT_STATUS_SUCCESS); } diff --git a/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp b/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp index 82c09e6a32..8708ceaf96 100644 --- a/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp +++ b/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp @@ -77,7 +77,7 @@ HsaEvent* InterruptSignal::CreateEvent(HSA_EVENTTYPE type, bool manual_reset) { HsaEvent* ret = NULL; if (HSAKMT_STATUS_SUCCESS == - hsaKmtCreateEvent(&event_descriptor, manual_reset, false, &ret)) { + HSAKMT_CALL(hsaKmtCreateEvent(&event_descriptor, manual_reset, false, &ret))) { if (type == HSA_EVENTTYPE_MEMORY) { memset(&ret->EventData.EventData.MemoryAccessFault.Failure, 0, sizeof(HsaAccessAttributeFailure)); @@ -89,7 +89,7 @@ HsaEvent* InterruptSignal::CreateEvent(HSA_EVENTTYPE type, bool manual_reset) { return ret; } -void InterruptSignal::DestroyEvent(HsaEvent* evt) { hsaKmtDestroyEvent(evt); } +void InterruptSignal::DestroyEvent(HsaEvent* evt) { HSAKMT_CALL(hsaKmtDestroyEvent(evt)); } InterruptSignal::InterruptSignal(hsa_signal_value_t initial_value, HsaEvent* use_event) : LocalSignal(initial_value, false), Signal(signal()) { @@ -194,7 +194,7 @@ hsa_signal_value_t InterruptSignal::WaitRelaxed(hsa_signal_condition_t condition static_cast(signal_abort_timeout ? signal_abort_timeout * 1000 : 0xFFFFFFFFUL) ); - hsaKmtWaitOnEvent_Ext(event_, wait_ms, &event_age); + HSAKMT_CALL(hsaKmtWaitOnEvent_Ext(event_, wait_ms, &event_age)); } } @@ -370,6 +370,11 @@ hsa_signal_value_t InterruptSignal::CasAcqRel(hsa_signal_value_t expected, SetEvent(); return ret; } + /// @brief Notify driver of signal value change if necessary. + void InterruptSignal::SetEvent() { + std::atomic_signal_fence(std::memory_order_seq_cst); + if (InWaiting()) HSAKMT_CALL(hsaKmtSetEvent(event_)); + } } // namespace core } // namespace rocr diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 057ba82545..6490e09211 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -370,7 +370,7 @@ hsa_status_t Runtime::FreeMemory(void* ptr) { } if (alloc_flags & core::MemoryRegion::AllocateAsan) - assert(hsaKmtReturnAsanHeaderPage(ptr) == HSAKMT_STATUS_SUCCESS); + assert(HSAKMT_CALL(hsaKmtReturnAsanHeaderPage(ptr)) == HSAKMT_STATUS_SUCCESS); const hsa_status_t err = region->Free(ptr, size); if (err != HSA_STATUS_SUCCESS) { @@ -886,21 +886,21 @@ hsa_status_t Runtime::InteropMap(uint32_t num_agents, Agent** agents, agents[i]->GetInfo((hsa_agent_info_t)HSA_AMD_AGENT_INFO_DRIVER_NODE_ID, &nodes[i]); - if (hsaKmtRegisterGraphicsHandleToNodes(interop_handle, &info, num_agents, - nodes) != HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtRegisterGraphicsHandleToNodes(interop_handle, &info, num_agents, + nodes)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; HSAuint64 altAddress; HsaMemMapFlags map_flags; map_flags.Value = 0; map_flags.ui32.PageSize = HSA_PAGE_SIZE_64KB; - if (hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes, + if (HSAKMT_CALL(hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes, &altAddress, map_flags, num_agents, - nodes) != HSAKMT_STATUS_SUCCESS) { + nodes)) != HSAKMT_STATUS_SUCCESS) { map_flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - if (hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes, &altAddress, map_flags, - num_agents, nodes) != HSAKMT_STATUS_SUCCESS) { - hsaKmtDeregisterMemory(info.MemoryAddress); + if (HSAKMT_CALL(hsaKmtMapMemoryToGPUNodes(info.MemoryAddress, info.SizeInBytes, &altAddress, map_flags, + num_agents, nodes)) != HSAKMT_STATUS_SUCCESS) { + HSAKMT_CALL(hsaKmtDeregisterMemory(info.MemoryAddress)); return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } } @@ -919,9 +919,9 @@ hsa_status_t Runtime::InteropMap(uint32_t num_agents, Agent** agents, } hsa_status_t Runtime::InteropUnmap(void* ptr) { - if(hsaKmtUnmapMemoryToGPU(ptr)!=HSAKMT_STATUS_SUCCESS) + if(HSAKMT_CALL(hsaKmtUnmapMemoryToGPU(ptr))!=HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT; - if(hsaKmtDeregisterMemory(ptr)!=HSAKMT_STATUS_SUCCESS) + if(HSAKMT_CALL(hsaKmtDeregisterMemory(ptr))!=HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT; return HSA_STATUS_SUCCESS; } @@ -959,7 +959,7 @@ hsa_status_t Runtime::PtrInfo(const void* ptr, hsa_amd_pointer_info_t* info, voi // We don't care if this returns an error code. // The type will be HSA_EXT_POINTER_TYPE_UNKNOWN if so. - auto err = hsaKmtQueryPointerInfo(ptr, &thunkInfo); + auto err = HSAKMT_CALL(hsaKmtQueryPointerInfo(ptr, &thunkInfo)); if (err != HSAKMT_STATUS_SUCCESS || thunkInfo.Type == HSA_POINTER_UNKNOWN) { retInfo.type = HSA_EXT_POINTER_TYPE_UNKNOWN; memcpy(info, &retInfo, retInfo.size); @@ -1080,7 +1080,7 @@ hsa_status_t Runtime::SetPtrInfoData(const void* ptr, void* userptr) { } } // Cover entries not in the allocation map (graphics, lock,...) - if (hsaKmtSetMemoryUserData(ptr, userptr) == HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtSetMemoryUserData(ptr, userptr)) == HSAKMT_STATUS_SUCCESS) return HSA_STATUS_SUCCESS; return HSA_STATUS_ERROR_INVALID_ARGUMENT; } @@ -1185,7 +1185,7 @@ void Runtime::AsyncIPCSockServerConnLoop(void*) { if (!ptr) continue; // Export DMA Buf FD and wait for client import - int err = hsaKmtExportDMABufHandle(ptr, len, &dmabuf_fd, &fragOffset); + int err = HSAKMT_CALL(hsaKmtExportDMABufHandle(ptr, len, &dmabuf_fd, &fragOffset)); if (err != HSAKMT_STATUS_SUCCESS) continue; SendDmaBufFd(connection_fd, dmabuf_fd); err = read(connection_fd, buf, sizeof(buf)); @@ -1236,7 +1236,7 @@ hsa_status_t Runtime::IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* han if (!ipc_dmabuf_supported_) { HsaSharedMemoryHandle *sHandle = reinterpret_cast(handle); - if (hsaKmtShareMemory(block.base, block.length, sHandle) != HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtShareMemory(block.base, block.length, sHandle)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT; hsa_status_t err = HSA_STATUS_SUCCESS; @@ -1271,7 +1271,7 @@ hsa_status_t Runtime::IPCCreate(void* ptr, size_t len, hsa_amd_ipc_memory_t* han // deferred export will not run into this problem. int dmabuf_fd; uint64_t dmabufOffset; - HSAKMT_STATUS err = hsaKmtExportDMABufHandle(ptr, len, &dmabuf_fd, &dmabufOffset); + HSAKMT_STATUS err = HSAKMT_CALL(hsaKmtExportDMABufHandle(ptr, len, &dmabuf_fd, &dmabufOffset)); assert(dmabufOffset/pageSize == fragOffset && "DMA Buf inconsistent with pointer offset."); if (err != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; close(dmabuf_fd); @@ -1366,12 +1366,12 @@ int Runtime::IPCClientImport(uint32_t conn_handle, uint64_t dmabuf_fd_handle, HsaGraphicsResourceInfo info; HSA_REGISTER_MEM_FLAGS regFlags; regFlags.ui32.requiresVAddr = !!res ? 0 : 1; - int err = hsaKmtRegisterGraphicsHandleToNodesExt(dmabuf_fd, &info, numNodes, nodes, regFlags); + int err = HSAKMT_CALL(hsaKmtRegisterGraphicsHandleToNodesExt(dmabuf_fd, &info, numNodes, nodes, regFlags)); if (err == HSAKMT_STATUS_SUCCESS) { *importAddress = info.MemoryAddress; *importSize = info.SizeInBytes; if (res) { - hsaKmtDeregisterMemory(*importAddress); + HSAKMT_CALL(hsaKmtDeregisterMemory(*importAddress)); // Manually libDRM import and GPU map system memory AMD::GpuAgent* agent = reinterpret_cast(agents_by_node_[info.NodeId][0]); @@ -1414,8 +1414,8 @@ hsa_status_t Runtime::IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len, int ret = ipc_dmabuf_supported_ ? IPCClientImport(importHandle.handle[2], dmaBufFDHandle, res, numNodes, nodes, &importAddress, &importSize) : - hsaKmtRegisterSharedHandle(reinterpret_cast(&importHandle), - &importAddress, &importSize); + HSAKMT_CALL(hsaKmtRegisterSharedHandle(reinterpret_cast(&importHandle), + &importAddress, &importSize)); if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT; return HSA_STATUS_SUCCESS; @@ -1424,20 +1424,20 @@ hsa_status_t Runtime::IPCAttach(const hsa_amd_ipc_memory_t* handle, size_t len, auto mapMemoryToNodes = [&](unsigned int numNodes, HSAuint32 *nodes) { HSAuint64 altAddress; if (!numNodes) { - if (hsaKmtMapMemoryToGPU(importAddress, importSize, &altAddress) != HSAKMT_STATUS_SUCCESS) { - hsaKmtDeregisterMemory(importAddress); + if (HSAKMT_CALL(hsaKmtMapMemoryToGPU(importAddress, importSize, &altAddress)) != HSAKMT_STATUS_SUCCESS) { + HSAKMT_CALL(hsaKmtDeregisterMemory(importAddress)); return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } } else { HsaMemMapFlags map_flags; map_flags.Value = 0; map_flags.ui32.PageSize = HSA_PAGE_SIZE_64KB; - if (hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, numNodes, - nodes) != HSAKMT_STATUS_SUCCESS) { + if (HSAKMT_CALL(hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, numNodes, + nodes)) != HSAKMT_STATUS_SUCCESS) { map_flags.ui32.PageSize = HSA_PAGE_SIZE_4KB; - if (hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, numNodes, - nodes) != HSAKMT_STATUS_SUCCESS) { - hsaKmtDeregisterMemory(importAddress); + if (HSAKMT_CALL(hsaKmtMapMemoryToGPUNodes(importAddress, importSize, &altAddress, map_flags, numNodes, + nodes)) != HSAKMT_STATUS_SUCCESS) { + HSAKMT_CALL(hsaKmtDeregisterMemory(importAddress)); return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } } @@ -1537,9 +1537,9 @@ hsa_status_t Runtime::IPCDetach(void* ptr) { } if (!ldrmImportCleaned) { - if (hsaKmtUnmapMemoryToGPU(ptr) != HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtUnmapMemoryToGPU(ptr)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT; - if (hsaKmtDeregisterMemory(ptr) != HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtDeregisterMemory(ptr)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_INVALID_ARGUMENT; } return HSA_STATUS_SUCCESS; @@ -1599,7 +1599,7 @@ void Runtime::AsyncEventsLoop(void* _eventsInfo) { constexpr uint32_t wait_ms = 0xFFFFFFFEu; HsaEvent** end = std::unique(&hsa_events[0], &hsa_events[0] + unique_evts); unique_evts = uint32_t(end - &hsa_events[0]); - hsaKmtWaitOnMultipleEvents_Ext(&hsa_events[0], unique_evts, false, wait_ms, &event_age[0]); + HSAKMT_CALL(hsaKmtWaitOnMultipleEvents_Ext(&hsa_events[0], unique_evts, false, wait_ms, &event_age[0])); }; while (!async_events_control_.exit) { @@ -2718,7 +2718,7 @@ hsa_status_t Runtime::SetSvmAttrib(void* ptr, size_t size, uint8_t* base = AlignDown((uint8_t*)ptr, 4096); uint8_t* end = AlignUp((uint8_t*)ptr + size, 4096); size_t len = end - base; - HSAKMT_STATUS error = hsaKmtSVMSetAttr(base, len, attribs.size(), &attribs[0]); + HSAKMT_STATUS error = HSAKMT_CALL(hsaKmtSVMSetAttr(base, len, attribs.size(), &attribs[0])); if (error != HSAKMT_STATUS_SUCCESS) throw AMD::hsa_exception(HSA_STATUS_ERROR, "hsaKmtSVMSetAttr failed."); @@ -2804,7 +2804,7 @@ hsa_status_t Runtime::GetSvmAttrib(void* ptr, size_t size, uint8_t* end = AlignUp((uint8_t*)ptr + size, 4096); size_t len = end - base; if (attribs.size() != 0) { - HSAKMT_STATUS error = hsaKmtSVMGetAttr(base, len, attribs.size(), &attribs[0]); + HSAKMT_STATUS error = HSAKMT_CALL(hsaKmtSVMGetAttr(base, len, attribs.size(), &attribs[0])); if (error != HSAKMT_STATUS_SUCCESS) throw AMD::hsa_exception(HSA_STATUS_ERROR, "hsaKmtSVMGetAttr failed."); } @@ -2994,7 +2994,7 @@ hsa_status_t Runtime::SvmPrefetch(void* ptr, size_t size, hsa_agent_t agent, HSA_SVM_ATTRIBUTE attrib; attrib.type = HSA_SVM_ATTR_PREFETCH_LOC; attrib.value = op->node_id; - HSAKMT_STATUS error = hsaKmtSVMSetAttr(op->base, op->size, 1, &attrib); + HSAKMT_STATUS error = HSAKMT_CALL(hsaKmtSVMSetAttr(op->base, op->size, 1, &attrib)); assert(error == HSAKMT_STATUS_SUCCESS && "KFD Prefetch failed."); removePrefetchRanges(op); @@ -3064,7 +3064,7 @@ Agent* Runtime::GetSVMPrefetchAgent(void* ptr, size_t size) { attrib.type = HSA_SVM_ATTR_PREFETCH_LOC; for (auto& range : holes) { HSAKMT_STATUS error = - hsaKmtSVMGetAttr(reinterpret_cast(range.first), range.second, 1, &attrib); + HSAKMT_CALL(hsaKmtSVMGetAttr(reinterpret_cast(range.first), range.second, 1, &attrib)); assert(error == HSAKMT_STATUS_SUCCESS && "KFD prefetch query failed."); if (attrib.value == -1) return nullptr; @@ -3094,7 +3094,7 @@ hsa_status_t Runtime::DmaBufExport(const void* ptr, size_t size, int* dmabuf, ui int fd; uint64_t off; - HSAKMT_STATUS err = hsaKmtExportDMABufHandle(const_cast(ptr), size, &fd, &off); + HSAKMT_STATUS err = HSAKMT_CALL(hsaKmtExportDMABufHandle(const_cast(ptr), size, &fd, &off)); if (err == HSAKMT_STATUS_SUCCESS) { *dmabuf = fd; *offset = off; @@ -3138,10 +3138,10 @@ hsa_status_t Runtime::VMemoryAddressReserve(void** va, size_t size, uint64_t add memFlags.ui32.FixedAddress = 1; /* Try to reserving the VA requested by user */ - if (hsaKmtAllocMemoryAlign(0, size, alignment, memFlags, &addr) != HSAKMT_STATUS_SUCCESS) { + if (HSAKMT_CALL(hsaKmtAllocMemoryAlign(0, size, alignment, memFlags, &addr)) != HSAKMT_STATUS_SUCCESS) { memFlags.ui32.FixedAddress = 0; /* Could not reserved VA requested, allocate alternate VA */ - if (hsaKmtAllocMemoryAlign(0, size, alignment, memFlags, &addr) != HSAKMT_STATUS_SUCCESS) + if (HSAKMT_CALL(hsaKmtAllocMemoryAlign(0, size, alignment, memFlags, &addr)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; } @@ -3163,7 +3163,7 @@ hsa_status_t Runtime::VMemoryAddressFree(void* va, size_t size) { if (it->second.use_count > 0) return HSA_STATUS_ERROR_RESOURCE_FREE; - if (hsaKmtFreeMemory(va, size) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; + if (HSAKMT_CALL(hsaKmtFreeMemory(va, size)) != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR; reserved_address_map_.erase(it); return HSA_STATUS_SUCCESS; @@ -3603,8 +3603,8 @@ hsa_status_t Runtime::VMemoryExportShareableHandle(int* dmabuf_fd, uint64_t offset, ret; - ret = hsaKmtExportDMABufHandle(memoryHandle->second.thunk_handle, memoryHandle->second.size, - dmabuf_fd, &offset); + ret = HSAKMT_CALL(hsaKmtExportDMABufHandle(memoryHandle->second.thunk_handle, memoryHandle->second.size, + dmabuf_fd, &offset)); if (ret != HSAKMT_STATUS_SUCCESS) return HSA_STATUS_ERROR_OUT_OF_RESOURCES; return HSA_STATUS_SUCCESS; @@ -3640,7 +3640,7 @@ hsa_status_t Runtime::VMemoryImportShareableHandle(int dmabuf_fd, }; HsaGraphicsResourceInfo info; - int ret = hsaKmtRegisterGraphicsHandleToNodes(dmabuf_fd, &info, 0, NULL); + int ret = HSAKMT_CALL(hsaKmtRegisterGraphicsHandleToNodes(dmabuf_fd, &info, 0, NULL)); if (ret) return HSA_STATUS_ERROR_INCOMPATIBLE_ARGUMENTS; ThunkHandle thunk_handle = info.MemoryAddress; @@ -3661,7 +3661,7 @@ hsa_status_t Runtime::VMemoryImportShareableHandle(int dmabuf_fd, if (!region) return HSA_STATUS_ERROR_INVALID_ALLOCATION; HsaPointerInfo ptrInfo; - ret = hsaKmtQueryPointerInfo(info.MemoryAddress, &ptrInfo); + ret = HSAKMT_CALL(hsaKmtQueryPointerInfo(info.MemoryAddress, &ptrInfo)); if (ret != HSA_STATUS_SUCCESS || ptrInfo.Type == HSA_POINTER_UNKNOWN) return HSA_STATUS_ERROR_INVALID_ALLOCATION; diff --git a/runtime/hsa-runtime/core/runtime/signal.cpp b/runtime/hsa-runtime/core/runtime/signal.cpp index 69b916bd39..6e6124ce9e 100644 --- a/runtime/hsa-runtime/core/runtime/signal.cpp +++ b/runtime/hsa-runtime/core/runtime/signal.cpp @@ -320,7 +320,7 @@ uint32_t Signal::WaitMultiple(uint32_t signal_count, const hsa_signal_t* hsa_sig uint64_t ct=timer::duration_cast( time_remaining).count(); wait_ms = (ct>0xFFFFFFFEu) ? 0xFFFFFFFEu : ct; - hsaKmtWaitOnMultipleEvents_Ext(evts, unique_evts, wait_on_all, wait_ms, event_age); + HSAKMT_CALL(hsaKmtWaitOnMultipleEvents_Ext(evts, unique_evts, wait_on_all, wait_ms, event_age)); } } @@ -425,7 +425,7 @@ uint32_t Signal::WaitAnyExceptions(uint32_t signal_count, const hsa_signal_t* hs } } - hsaKmtWaitOnMultipleEvents_Ext(evts, unique_evts, false, wait_ms, event_age); + HSAKMT_CALL(hsaKmtWaitOnMultipleEvents_Ext(evts, unique_evts, false, wait_ms, event_age)); } //while } diff --git a/runtime/hsa-runtime/core/runtime/svm_profiler.cpp b/runtime/hsa-runtime/core/runtime/svm_profiler.cpp index 1db1162fef..029b209d29 100644 --- a/runtime/hsa-runtime/core/runtime/svm_profiler.cpp +++ b/runtime/hsa-runtime/core/runtime/svm_profiler.cpp @@ -143,8 +143,8 @@ void SvmProfileControl::PollSmi() { HSA_SMI_EVENT_MASK_FROM_INDEX(HSA_SMI_EVENT_UNMAP_FROM_GPU); for (int i = 0; i < core::Runtime::runtime_singleton_->gpu_agents().size(); i++) { - auto err = hsaKmtOpenSMI(core::Runtime::runtime_singleton_->gpu_agents()[i]->node_id(), - &files[i + 1].fd); + auto err = HSAKMT_CALL(hsaKmtOpenSMI(core::Runtime::runtime_singleton_->gpu_agents()[i]->node_id(), + &files[i + 1].fd)); assert(err == HSAKMT_STATUS_SUCCESS); files[i + 1].events = POLLIN; files[i + 1].revents = 0; diff --git a/runtime/hsa-runtime/image/image_manager_kv.cpp b/runtime/hsa-runtime/image/image_manager_kv.cpp index 2521bc5e76..1cd243d848 100644 --- a/runtime/hsa-runtime/image/image_manager_kv.cpp +++ b/runtime/hsa-runtime/image/image_manager_kv.cpp @@ -53,6 +53,7 @@ #include "inc/hsa_ext_amd.h" #include "core/inc/hsa_internal.h" #include "core/inc/hsa_ext_amd_impl.h" +#include "core/inc/runtime.h" #include "addrlib/inc/addrinterface.h" #include "addrlib/src/core/addrlib.h" #include "image_runtime.h" @@ -109,7 +110,7 @@ hsa_status_t ImageManagerKv::Initialize(hsa_agent_t agent_handle) { status = HSA::hsa_agent_get_info( agent_, static_cast(HSA_AMD_AGENT_INFO_DRIVER_NODE_ID), &node_id); assert(status == HSA_STATUS_SUCCESS); - HSAKMT_STATUS stat = hsaKmtGetTileConfig(node_id, &tileConfig); + HSAKMT_STATUS stat = HSAKMT_CALL(hsaKmtGetTileConfig(node_id, &tileConfig)); assert(stat == HSAKMT_STATUS_SUCCESS); // Initialize address library. diff --git a/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp b/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp index 2321614603..079da9438f 100644 --- a/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp +++ b/runtime/hsa-runtime/libamdhsacode/amd_core_dump.cpp @@ -50,6 +50,7 @@ #include #include #include "core/util/utils.h" +#include "core/inc/runtime.h" #include "./amd_hsa_code_util.hpp" #include "core/inc/amd_core_dump.hpp" #include "hsakmt/hsakmt.h" @@ -118,14 +119,14 @@ struct NoteSegmentBuilder : public SegmentBuilder { uint32_t runtime_size, agents_size, queue_size, n_entries, entry_size; HsaVersionInfo versionInfo = {0}; - if (hsaKmtDbgEnable(&runtime_ptr, &runtime_size)) { + if (HSAKMT_CALL(hsaKmtDbgEnable(&runtime_ptr, &runtime_size))) { fprintf(stderr, "Failed to enable debug interface, " "debugger might be already attached.\n"); return HSA_STATUS_ERROR; } std::unique_ptr runtime_info(runtime_ptr, std::free); - if (hsaKmtGetVersion(&versionInfo)) { + if (HSAKMT_CALL(hsaKmtGetVersion(&versionInfo))) { fprintf(stderr, "Failed to fetch driver ABI version.\n"); return HSA_STATUS_ERROR; } @@ -138,7 +139,7 @@ struct NoteSegmentBuilder : public SegmentBuilder { /* Store runtime_info_size in PT_NOTE package */ note_package_builder_.Write(runtime_size); - if (hsaKmtDbgGetDeviceData(&agents_ptr, &n_entries, &entry_size)) { + if (HSAKMT_CALL(hsaKmtDbgGetDeviceData(&agents_ptr, &n_entries, &entry_size))) { fprintf(stderr, "Failed to fetch agents snapshot.\n"); return HSA_STATUS_ERROR; } @@ -149,7 +150,7 @@ struct NoteSegmentBuilder : public SegmentBuilder { /* Store agent_info_entry_size in PT_NOTE package */ note_package_builder_.Write(entry_size); - if (hsaKmtDbgGetQueueData(&queues_ptr, &n_entries, &entry_size, true)) { + if (HSAKMT_CALL(hsaKmtDbgGetQueueData(&queues_ptr, &n_entries, &entry_size, true))) { fprintf(stderr, "Failed to fetch queues snapshot.\n"); return HSA_STATUS_ERROR; } @@ -163,7 +164,7 @@ struct NoteSegmentBuilder : public SegmentBuilder { PushInfo(runtime_info.get(), runtime_size); PushInfo(agents_info.get(), agents_size); PushInfo(queues_info.get(), queue_size); - if (hsaKmtDbgDisable()) { + if (HSAKMT_CALL(hsaKmtDbgDisable())) { fprintf(stderr, "Failed to disable debug interface.\n"); return HSA_STATUS_ERROR; }