AI/rocm-systems
2
0
Forka 0
Codice Problemi Pull Request Actions Pacchetti Progetti Rilasci Wiki Attività

SWDEV-451594 - Change device kernel args to use HDP flush by default

Sfoglia il codice sorgente 
The Readback and Avoid HDP Flush memory ordering workaround is
used as a fallback solution only when HDP flush register is invalid

Change-Id: Ic284eba1f95ed22b0270d3abeb904fb902015b1a
This commit is contained in:
Ioannis Assiouras
2024-05-02 01:18:47 +01:00
committato da Saleel Kudchadker
parent e53df57ffe
commit 6cb7b6ec6b
3 ha cambiato i file con 33 aggiunte e 49 eliminazioni
Scarica il file Patch Scarica il file Diff Expand all files Collapse all files
hipamd/src/hip_graph_internal.cpp
+11 -18
Vedi File
@@ -395,28 +395,21 @@ hipError_t GraphExec::CaptureAQLPackets() {
}
}
auto kernArgImpl = device->settings().kernel_arg_impl_;
if (device_kernarg_pool_) {
auto kernArgImpl = device->settings().kernel_arg_impl_;
const auto applyMemOrderingWA =
((kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback) ||
(kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP)) &&
kernarg_pool_size_graph_ > 0;
if (device_kernarg_pool_ && applyMemOrderingWA) {
address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_;
volatile char kSentinel = *(dev_ptr - 1);
// Memory ordering workaround for pcie: execute sfence followed by
// write the last byte of kernarg.
_mm_sfence();
*(dev_ptr - 1) = kSentinel;
// HDP flush is required to guarantee ordering in Navi and MI100
if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) {
*device->info().hdpMemFlushCntl = 1u;
volatile auto kSentinel = *device->info().hdpMemFlushCntl;
} else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback &&
kernarg_pool_size_graph_ != 0) {
address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_;
volatile auto kSentinel = *(dev_ptr - 1);
_mm_sfence();
*(dev_ptr - 1) = kSentinel;
_mm_mfence();
kSentinel = *(dev_ptr - 1);
}
// Memory ordering workaround for pcie: execute mfence followed by
// read of the last byte of kernarg.
_mm_mfence();
kSentinel = *(dev_ptr - 1);
}
ResetQueueIndex();
rocclr/device/rocm/rocsettings.cpp
+8 -10
Vedi File
@@ -250,8 +250,6 @@ void Settings::setKernelArgImpl(const amd::Isa& isa, bool isXgmi, bool hasValidH
const bool isMI300 = gfxipMajor == 9 && gfxipMinor == 4 &&
(gfxStepping == 0 || gfxStepping == 1 || gfxStepping == 2);
const bool isMI200 = (gfxipMajor == 9 && gfxipMinor == 0 && gfxStepping == 10);
const bool isMI100 = (gfxipMajor == 9 && gfxipMinor == 0 && gfxStepping == 8);
const bool isNavi = (gfxipMajor >= 10);
auto kernelArgImpl = KernelArgImpl::HostKernelArgs;
@@ -259,15 +257,15 @@ void Settings::setKernelArgImpl(const amd::Isa& isa, bool isXgmi, bool hasValidH
// The XGMI-connected path does not require the manual memory ordering
// workarounds that the PCIe connected path requires
kernelArgImpl = KernelArgImpl::DeviceKernelArgs;
} else if (isMI300 || isMI200) {
// Implement the kernel argument readback workaround. It works only on
// MI200, MI300 because of the strict guarantee on ordering of
// stores in those ASICS
kernelArgImpl = KernelArgImpl::DeviceKernelArgsReadback;
} else if (hasValidHDPFlush && (isNavi || isMI100)) {
// For dev >= gfx10 and MI100 ASICS implement the HDP flush to MMIO if the
// HDP flush register is valid
} else if (hasValidHDPFlush) {
// If the HDP flush register is valid implement the HDP flush to MMIO
kernelArgImpl = KernelArgImpl::DeviceKernelArgsHDP;
} else if (isMI300 || isMI200) {
// Implement the kernel argument readback workaround
// (write all args -> sfence -> write last byte -> mfence -> read last byte)
// It works only on MI200 and MI300 because of the strict guarantee on
// ordering of stores in those ASICS
kernelArgImpl = KernelArgImpl::DeviceKernelArgsReadback;
}
// Enable device kernel args for MI300* for now
rocclr/device/rocm/rocvirtual.cpp
+14 -21
Vedi File
@@ -3215,12 +3215,6 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
bool isGraphCapture = vcmd != nullptr && vcmd->getCapturingState();
size_t argSize = std::min(gpuKernel.KernargSegmentByteSize(), signature.paramsSize());
const auto kernArgImpl = dev().settings().kernel_arg_impl_;
const auto applyMemOrderingWA =
((kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback) ||
(kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP)) &&
roc_device_.info().largeBar_ && argSize > 0 && !isGraphCapture;
// Find all parameters for the current kernel
if (!kernel.parameters().deviceKernelArgs() || gpuKernel.isInternalKernel()) {
// Allocate buffer to hold kernel arguments
@@ -3235,15 +3229,19 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
nontemporalMemcpy(argBuffer, parameters, argSize);
if (applyMemOrderingWA) {
// Memory ordering workaround for pcie: execute sfence followed by
// write the last byte of kernarg
_mm_sfence();
*(argBuffer + argSize - 1) = *(parameters + argSize - 1);
// HDP flush is required to guarantee ordering in Navi and MI100
if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) {
*dev().info().hdpMemFlushCntl = 1u;
}
if (roc_device_.info().largeBar_ && !isGraphCapture) {
const auto kernArgImpl = dev().settings().kernel_arg_impl_;
if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) {
*dev().info().hdpMemFlushCntl = 1u;
volatile auto kSentinel = *dev().info().hdpMemFlushCntl;
} else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback &&
argSize != 0) {
_mm_sfence();
*(argBuffer + argSize - 1) = *(parameters + argSize - 1);
_mm_mfence();
volatile auto kSentinel = *(argBuffer + argSize - 1);
}
}
}
@@ -3305,12 +3303,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
(HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE);
aql_packet->setup = sizes.dimensions() << HSA_KERNEL_DISPATCH_PACKET_SETUP_DIMENSIONS;
}
if (applyMemOrderingWA) {
// Memory ordering workaround for pcie: execute mfence followed by
// read of the last byte of kernarg
_mm_mfence();
volatile char kSentinel = *(argBuffer + argSize - 1);
}
if (vcmd == nullptr) {
// Dispatch the packet
if (!dispatchAqlPacket(&dispatchPacket, aqlHeaderWithOrder,