diff --git a/projects/clr/rocclr/runtime/device/device.cpp b/projects/clr/rocclr/runtime/device/device.cpp index 3b5021ccb6..66c1687074 100644 --- a/projects/clr/rocclr/runtime/device/device.cpp +++ b/projects/clr/rocclr/runtime/device/device.cpp @@ -624,6 +624,8 @@ Settings::Settings() : value_(0) { if (amd::IS_HIP) { GPU_SINGLE_ALLOC_PERCENT = 100; } + + fenceScopeAgent_ = AMD_OPT_FLUSH; } void Memory::saveMapInfo(const void* mapAddress, const amd::Coord3D origin, diff --git a/projects/clr/rocclr/runtime/device/device.hpp b/projects/clr/rocclr/runtime/device/device.hpp index 9b0d04b008..932d543074 100644 --- a/projects/clr/rocclr/runtime/device/device.hpp +++ b/projects/clr/rocclr/runtime/device/device.hpp @@ -539,7 +539,8 @@ class Settings : public amd::HeapObject { uint enableXNACK_ : 1; //!< Enable XNACK feature uint enableCoopGroups_ : 1; //!< Enable cooperative groups feature uint enableCoopMultiDeviceGroups_ : 1; //!< Enable cooperative groups multi device - uint reserved_ : 12; + uint fenceScopeAgent_ : 1; //!< Enable fence scope agent in AQL dispatch packet + uint reserved_ : 11; }; uint value_; }; diff --git a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp index 91e8f43daa..408a973fc3 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.cpp @@ -51,16 +51,6 @@ namespace roc { static const uint16_t kInvalidAql = (HSA_PACKET_TYPE_INVALID << HSA_PACKET_HEADER_TYPE); -static const uint16_t kDispatchPacketHeaderNoSync = - (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | - (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | - (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); - -static const uint16_t kDispatchPacketHeader = - (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | (1 << HSA_PACKET_HEADER_BARRIER) | - (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | - (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); - static const uint16_t kBarrierPacketHeader = (HSA_PACKET_TYPE_BARRIER_AND << HSA_PACKET_HEADER_TYPE) | (1 << HSA_PACKET_HEADER_BARRIER) | (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | @@ -103,7 +93,7 @@ void VirtualGPU::MemoryDependency::validate(VirtualGPU& gpu, const Memory* memor if (maxMemObjectsInQueue_ == 0) { // Sync AQL packets - gpu.setAqlHeader(kDispatchPacketHeader); + gpu.setAqlHeader(gpu.dispatchPacketHeader_); return; } @@ -138,7 +128,7 @@ void VirtualGPU::MemoryDependency::validate(VirtualGPU& gpu, const Memory* memor if (flushL1Cache) { // Sync AQL packets - gpu.setAqlHeader(kDispatchPacketHeader); + gpu.setAqlHeader(gpu.dispatchPacketHeader_); // Clear memory dependency state const static bool All = true; @@ -197,7 +187,7 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para if (!cooperativeGroups && memoryDependency().maxMemObjectsInQueue() != 0) { // AQL packets - setAqlHeader(kDispatchPacketHeaderNoSync); + setAqlHeader(dispatchPacketHeaderNoSync_); } // Mark the tracker with a new kernel, @@ -236,7 +226,7 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para return false; } else if (sync) { // Sync AQL packets - setAqlHeader(kDispatchPacketHeader); + setAqlHeader(dispatchPacketHeader_); // Clear memory dependency state const static bool All = true; memoryDependency().clear(!All); @@ -295,7 +285,7 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para //! This condition is for SVM fine-grain if (dev().isFineGrainedSystem(true)) { // Sync AQL packets - setAqlHeader(kDispatchPacketHeader); + setAqlHeader(dispatchPacketHeader_); // Clear memory dependency state const static bool All = true; memoryDependency().clear(!All); @@ -381,7 +371,7 @@ bool VirtualGPU::processMemObjects(const amd::Kernel& kernel, const_address para if (hsaKernel.program()->hasGlobalStores()) { // Sync AQL packets - setAqlHeader(kDispatchPacketHeader); + setAqlHeader(dispatchPacketHeader_); // Clear memory dependency state const static bool All = true; memoryDependency().clear(!All); @@ -576,7 +566,28 @@ VirtualGPU::VirtualGPU(Device& device) kernarg_pool_base_ = nullptr; kernarg_pool_size_ = 0; kernarg_pool_cur_offset_ = 0; - aqlHeader_ = kDispatchPacketHeader; + + if (device.settings().fenceScopeAgent_) { + dispatchPacketHeaderNoSync_ = + (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | + (HSA_FENCE_SCOPE_AGENT << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | + (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); + dispatchPacketHeader_= + (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | (1 << HSA_PACKET_HEADER_BARRIER) | + (HSA_FENCE_SCOPE_AGENT << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | + (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); + } else { + dispatchPacketHeaderNoSync_ = + (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | + (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | + (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); + dispatchPacketHeader_= + (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_PACKET_HEADER_TYPE) | (1 << HSA_PACKET_HEADER_BARRIER) | + (HSA_FENCE_SCOPE_SYSTEM << HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE) | + (HSA_FENCE_SCOPE_NONE << HSA_PACKET_HEADER_RELEASE_FENCE_SCOPE); + } + + aqlHeader_ = dispatchPacketHeader_; barrier_signal_.handle = 0; // Note: Virtual GPU device creation must be a thread safe operation @@ -2187,7 +2198,7 @@ void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) { static_cast(queue->blitMgr()).RunGwsInit(workgroups); // Sync AQL packets - queue->setAqlHeader(kDispatchPacketHeader); + queue->setAqlHeader(dispatchPacketHeader_); // Submit kernel to HW if (!queue->submitKernelInternal(vcmd.sizes(), vcmd.kernel(), vcmd.parameters(), diff --git a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.hpp b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.hpp index d4f280ec9b..f45060322d 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocvirtual.hpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocvirtual.hpp @@ -333,6 +333,8 @@ class VirtualGPU : public device::VirtualDevice { SLOT_PM4_SIZE_AQLP = HSA_VEN_AMD_AQLPROFILE_LEGACY_PM4_PACKET_SIZE/ 64 }; + uint16_t dispatchPacketHeaderNoSync_; + uint16_t dispatchPacketHeader_; }; template diff --git a/projects/clr/rocclr/runtime/utils/flags.hpp b/projects/clr/rocclr/runtime/utils/flags.hpp index fbcedd9a97..75ea3a2efa 100644 --- a/projects/clr/rocclr/runtime/utils/flags.hpp +++ b/projects/clr/rocclr/runtime/utils/flags.hpp @@ -207,8 +207,10 @@ release(bool, PAL_ALWAYS_RESIDENT, false, \ "Force memory resources to become resident at allocation time") \ release(uint, HIP_HOST_COHERENT, 0, \ "Coherent memory in hipHostMalloc, 0x1 = memory is coherent with host"\ - "0x0 = memory is not coherent between host and GPU") - + "0x0 = memory is not coherent between host and GPU") \ +release(uint, AMD_OPT_FLUSH, 0, \ + "Kernel flush option , 0x0 = Use system-scope fence operations." \ + "0x1 = Use device-scope fence operations when possible.") namespace amd { extern bool IS_HIP;