diff --git a/projects/clr/rocclr/device/pal/palresource.cpp b/projects/clr/rocclr/device/pal/palresource.cpp index 5f8a86375a..40c5123648 100644 --- a/projects/clr/rocclr/device/pal/palresource.cpp +++ b/projects/clr/rocclr/device/pal/palresource.cpp @@ -1512,21 +1512,21 @@ bool Resource::partialMemCopyTo(VirtualGPU& gpu, const amd::Coord3D& srcOrigin, (size[0] < dev().settings().cpDmaCopySizeMax_)); if (cp_dma) { // Make sure compute is done before CP DMA start - gpu.addBarrier(RgpSqqtBarrierReason::MemDependency); + gpu.addBarrier(RgpSqqtBarrierReason::MemDependency, BarrierType::KernelToCopy); } else { gpu.releaseGpuMemoryFence(); gpu.engineID_ = SdmaEngine; + + if (gpu.validateSdmaOverlap(*this, dstResource)) { + // Note: PAL should insert a NOP into the command buffer for synchronization + gpu.addBarrier(RgpSqqtBarrierReason::MemDependency, BarrierType::CopyToCopy); + } } // Wait for the resources, since runtime may use async transfers wait(gpu, waitOnBusyEngine); dstResource.wait(gpu, waitOnBusyEngine); - if (gpu.validateSdmaOverlap(*this, dstResource)) { - // Note: PAL should insert a NOP into the command buffer for synchronization - gpu.addBarrier(); - } - Pal::ImageLayout imgLayout = {}; gpu.eventBegin(gpu.engineID_); gpu.queue(gpu.engineID_).addCmdMemRef(memRef()); @@ -1626,7 +1626,7 @@ bool Resource::partialMemCopyTo(VirtualGPU& gpu, const amd::Coord3D& srcOrigin, if (cp_dma) { // Make sure CP dma is done - gpu.addBarrier(RgpSqqtBarrierReason::MemDependency); + gpu.addBarrier(RgpSqqtBarrierReason::MemDependency, BarrierType::CopyToKernel); } gpu.eventEnd(gpu.engineID_, event); diff --git a/projects/clr/rocclr/device/pal/palvirtual.cpp b/projects/clr/rocclr/device/pal/palvirtual.cpp index a19d3aa1c6..da61f82314 100644 --- a/projects/clr/rocclr/device/pal/palvirtual.cpp +++ b/projects/clr/rocclr/device/pal/palvirtual.cpp @@ -2404,8 +2404,7 @@ void VirtualGPU::PostDeviceEnqueue(const amd::Kernel& kernel, const HSAILKernel& static_cast(gpuDefQueue->blitMgr()) .runScheduler(*gpuDefQueue->virtualQueue_, *gpuDefQueue->schedParams_, 0, gpuDefQueue->vqHeader_->aql_slot_num / (DeviceQueueMaskSize * maskGroups_)); - const static bool FlushL2 = true; - gpuDefQueue->addBarrier(RgpSqqtBarrierReason::PostDeviceEnqueue, FlushL2); + gpuDefQueue->addBarrier(RgpSqqtBarrierReason::PostDeviceEnqueue, BarrierType::FlushL2); // Get the address of PM4 template and add write it to params //! @note DMA flush must not occur between patch and the scheduler @@ -3020,8 +3019,7 @@ void VirtualGPU::submitSignal(amd::SignalCommand& vcmd) { engineID_ = static_cast(pGpuMemory->getGpuEvent(*this)->engineId_); // Make sure GPU finished operation and data reached memory before the marker write - static constexpr bool FlushL2 = true; - addBarrier(RgpSqqtBarrierReason::SignalSubmit, FlushL2); + addBarrier(RgpSqqtBarrierReason::SignalSubmit, BarrierType::FlushL2); // Workarounds: We had systems where an extra delay was necessary. { // Flush CB associated with the DGMA buffer diff --git a/projects/clr/rocclr/device/pal/palvirtual.hpp b/projects/clr/rocclr/device/pal/palvirtual.hpp index 170753426a..bf46db9272 100644 --- a/projects/clr/rocclr/device/pal/palvirtual.hpp +++ b/projects/clr/rocclr/device/pal/palvirtual.hpp @@ -66,6 +66,14 @@ struct AqlPacketMgmt : public amd::EmbeddedObject { std::atomic packet_index_; //!< The active packet slot index }; + enum class BarrierType : uint8_t { + KernelToKernel = 0, + KernelToCopy, + CopyToKernel, + CopyToCopy, + FlushL2 +}; + //! Virtual GPU class VirtualGPU : public device::VirtualDevice { public: @@ -478,18 +486,29 @@ class VirtualGPU : public device::VirtualDevice { //! Returns queue, associated with VirtualGPU Queue& queue(EngineType id) const { return *queues_[id]; } - void addBarrier(RgpSqqtBarrierReason reason = RgpSqqtBarrierReason::Unknown, - bool flushL2 = false) const { + void addBarrier(RgpSqqtBarrierReason reason = RgpSqqtBarrierReason::MemDependency, + BarrierType type = BarrierType::KernelToKernel) const { Pal::BarrierInfo barrier = {}; barrier.pipePointWaitCount = 1; Pal::HwPipePoint point = Pal::HwPipePostCs; barrier.pPipePoints = &point; barrier.transitionCount = 1; - uint32_t cacheMask = (flushL2) ? Pal::CoherCopy : Pal::CoherShader; - Pal::BarrierTransition trans = { - cacheMask, - cacheMask, - {nullptr, {{0, 0, 0}, 0, 0, 0}, Pal::LayoutShaderRead, Pal::LayoutShaderRead}}; + Pal::BarrierTransition trans = {}; + trans.srcCacheMask = Pal::CoherShader; + trans.dstCacheMask = Pal::CoherShader; + trans.imageInfo.oldLayout.usages = Pal::LayoutShaderRead; + trans.imageInfo.oldLayout.engines = Pal::LayoutComputeEngine; + trans.imageInfo.newLayout.usages = Pal::LayoutShaderRead; + trans.imageInfo.newLayout.engines = Pal::LayoutComputeEngine; + if (type == BarrierType::KernelToCopy) { + trans.dstCacheMask = Pal::CoherCopy; + } else if (type == BarrierType::CopyToKernel) { + trans.srcCacheMask = Pal::CoherCopy; + } else if (type == BarrierType::CopyToCopy) { + trans.dstCacheMask = trans.srcCacheMask = Pal::CoherCopy; + } else if (type == BarrierType::FlushL2) { + trans.dstCacheMask = trans.srcCacheMask = Pal::CoherCopy | Pal::CoherCpu; + } barrier.pTransitions = &trans; barrier.waitPoint = Pal::HwPipePreCs; barrier.reason = static_cast(reason);