From 6cb7b6ec6bb048746fb40f472b5b95d8572e44b1 Mon Sep 17 00:00:00 2001 From: Ioannis Assiouras Date: Thu, 2 May 2024 01:18:47 +0100 Subject: [PATCH] SWDEV-451594 - Change device kernel args to use HDP flush by default The Readback and Avoid HDP Flush memory ordering workaround is used as a fallback solution only when HDP flush register is invalid Change-Id: Ic284eba1f95ed22b0270d3abeb904fb902015b1a --- hipamd/src/hip_graph_internal.cpp | 29 ++++++++++--------------- rocclr/device/rocm/rocsettings.cpp | 18 +++++++-------- rocclr/device/rocm/rocvirtual.cpp | 35 ++++++++++++------------------ 3 files changed, 33 insertions(+), 49 deletions(-) diff --git a/hipamd/src/hip_graph_internal.cpp b/hipamd/src/hip_graph_internal.cpp index dccf7044a7..a571a1d62f 100644 --- a/hipamd/src/hip_graph_internal.cpp +++ b/hipamd/src/hip_graph_internal.cpp @@ -395,28 +395,21 @@ hipError_t GraphExec::CaptureAQLPackets() { } } - auto kernArgImpl = device->settings().kernel_arg_impl_; + if (device_kernarg_pool_) { + auto kernArgImpl = device->settings().kernel_arg_impl_; - const auto applyMemOrderingWA = - ((kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback) || - (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP)) && - kernarg_pool_size_graph_ > 0; - - if (device_kernarg_pool_ && applyMemOrderingWA) { - address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_; - volatile char kSentinel = *(dev_ptr - 1); - // Memory ordering workaround for pcie: execute sfence followed by - // write the last byte of kernarg. - _mm_sfence(); - *(dev_ptr - 1) = kSentinel; - // HDP flush is required to guarantee ordering in Navi and MI100 if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) { *device->info().hdpMemFlushCntl = 1u; + volatile auto kSentinel = *device->info().hdpMemFlushCntl; + } else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback && + kernarg_pool_size_graph_ != 0) { + address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_; + volatile auto kSentinel = *(dev_ptr - 1); + _mm_sfence(); + *(dev_ptr - 1) = kSentinel; + _mm_mfence(); + kSentinel = *(dev_ptr - 1); } - // Memory ordering workaround for pcie: execute mfence followed by - // read of the last byte of kernarg. - _mm_mfence(); - kSentinel = *(dev_ptr - 1); } ResetQueueIndex(); diff --git a/rocclr/device/rocm/rocsettings.cpp b/rocclr/device/rocm/rocsettings.cpp index 19bac547fd..c3c5566337 100644 --- a/rocclr/device/rocm/rocsettings.cpp +++ b/rocclr/device/rocm/rocsettings.cpp @@ -250,8 +250,6 @@ void Settings::setKernelArgImpl(const amd::Isa& isa, bool isXgmi, bool hasValidH const bool isMI300 = gfxipMajor == 9 && gfxipMinor == 4 && (gfxStepping == 0 || gfxStepping == 1 || gfxStepping == 2); const bool isMI200 = (gfxipMajor == 9 && gfxipMinor == 0 && gfxStepping == 10); - const bool isMI100 = (gfxipMajor == 9 && gfxipMinor == 0 && gfxStepping == 8); - const bool isNavi = (gfxipMajor >= 10); auto kernelArgImpl = KernelArgImpl::HostKernelArgs; @@ -259,15 +257,15 @@ void Settings::setKernelArgImpl(const amd::Isa& isa, bool isXgmi, bool hasValidH // The XGMI-connected path does not require the manual memory ordering // workarounds that the PCIe connected path requires kernelArgImpl = KernelArgImpl::DeviceKernelArgs; - } else if (isMI300 || isMI200) { - // Implement the kernel argument readback workaround. It works only on - // MI200, MI300 because of the strict guarantee on ordering of - // stores in those ASICS - kernelArgImpl = KernelArgImpl::DeviceKernelArgsReadback; - } else if (hasValidHDPFlush && (isNavi || isMI100)) { - // For dev >= gfx10 and MI100 ASICS implement the HDP flush to MMIO if the - // HDP flush register is valid + } else if (hasValidHDPFlush) { + // If the HDP flush register is valid implement the HDP flush to MMIO kernelArgImpl = KernelArgImpl::DeviceKernelArgsHDP; + } else if (isMI300 || isMI200) { + // Implement the kernel argument readback workaround + // (write all args -> sfence -> write last byte -> mfence -> read last byte) + // It works only on MI200 and MI300 because of the strict guarantee on + // ordering of stores in those ASICS + kernelArgImpl = KernelArgImpl::DeviceKernelArgsReadback; } // Enable device kernel args for MI300* for now diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index 75f8f4a797..e074b7e9f7 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -3215,12 +3215,6 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, bool isGraphCapture = vcmd != nullptr && vcmd->getCapturingState(); size_t argSize = std::min(gpuKernel.KernargSegmentByteSize(), signature.paramsSize()); - const auto kernArgImpl = dev().settings().kernel_arg_impl_; - const auto applyMemOrderingWA = - ((kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback) || - (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP)) && - roc_device_.info().largeBar_ && argSize > 0 && !isGraphCapture; - // Find all parameters for the current kernel if (!kernel.parameters().deviceKernelArgs() || gpuKernel.isInternalKernel()) { // Allocate buffer to hold kernel arguments @@ -3235,15 +3229,19 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, nontemporalMemcpy(argBuffer, parameters, argSize); - if (applyMemOrderingWA) { - // Memory ordering workaround for pcie: execute sfence followed by - // write the last byte of kernarg - _mm_sfence(); - *(argBuffer + argSize - 1) = *(parameters + argSize - 1); - // HDP flush is required to guarantee ordering in Navi and MI100 - if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) { - *dev().info().hdpMemFlushCntl = 1u; - } + if (roc_device_.info().largeBar_ && !isGraphCapture) { + const auto kernArgImpl = dev().settings().kernel_arg_impl_; + + if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) { + *dev().info().hdpMemFlushCntl = 1u; + volatile auto kSentinel = *dev().info().hdpMemFlushCntl; + } else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback && + argSize != 0) { + _mm_sfence(); + *(argBuffer + argSize - 1) = *(parameters + argSize - 1); + _mm_mfence(); + volatile auto kSentinel = *(argBuffer + argSize - 1); + } } } @@ -3305,12 +3303,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); aql_packet->setup = sizes.dimensions() << HSA_KERNEL_DISPATCH_PACKET_SETUP_DIMENSIONS; } - if (applyMemOrderingWA) { - // Memory ordering workaround for pcie: execute mfence followed by - // read of the last byte of kernarg - _mm_mfence(); - volatile char kSentinel = *(argBuffer + argSize - 1); - } + if (vcmd == nullptr) { // Dispatch the packet if (!dispatchAqlPacket(&dispatchPacket, aqlHeaderWithOrder,