From 88ed58735d64c589b92ff841ac56edd41705a1c0 Mon Sep 17 00:00:00 2001 From: kjayapra-amd Date: Mon, 23 Aug 2021 22:01:56 -0400 Subject: [PATCH] SWDEV-232903 - Move hipmemset Dword optimization to ROCclr. Change-Id: I3eae61720cbc6364f1aaac4865bfd8b6ded08097 --- rocclr/device/blit.cpp | 104 ++++++++++++++++++++++++ rocclr/device/blit.hpp | 31 ++++++++ rocclr/device/rocm/rocblit.cpp | 140 +++++++++++++++++++-------------- 3 files changed, 215 insertions(+), 60 deletions(-) diff --git a/rocclr/device/blit.cpp b/rocclr/device/blit.cpp index f2cbd0763a..8ae03f3da0 100644 --- a/rocclr/device/blit.cpp +++ b/rocclr/device/blit.cpp @@ -677,4 +677,108 @@ uint32_t HostBlitManager::sRGBmap(float fc) const { return (uint32_t)(c * 255.0 + 0.5); } + +bool HostBlitManager::FillBufferInfo::ExpandPattern64(size_t pattern, size_t pattern_size, + size_t& pattern64) { + + bool retval = true; + + do { + + // If the pattern is 0 or if the pattern_size is same as max size. + if (pattern == 0 || pattern_size == sizeof(size_t)) { + pattern64 = pattern; + break; + } + + // Clean Curr_pattern, since it was casted off from const void* with a lesser size than size_t. + ClearBits64(pattern, (pattern_size * 8)); + pattern64 = 0; + + if (pattern_size == sizeof(uint8_t)) { + pattern64 = pattern & 0xff; + pattern64 = ((pattern << 56) | (pattern << 48) | (pattern << 40) | (pattern << 32) + | (pattern << 24) | (pattern << 16) | (pattern << 8) | (pattern)); + } else if (pattern_size == sizeof(uint16_t)) { + pattern = pattern & 0xffff; + pattern64 = ((pattern << 48) | (pattern << 32) | (pattern << 16) | (pattern)); + } else if (pattern_size == sizeof(uint32_t)) { + pattern = pattern & 0xffffffff; + pattern64 = ((pattern << 32) | (pattern)); + } else { + LogPrintfError("Unsupported Pattern size: %u \n", pattern_size); + retval = false; + break; + } + + } while (0); + + return retval; +} + +bool HostBlitManager::FillBufferInfo::PackInfo(const device::Memory& memory, size_t fill_size, + size_t fill_origin, const void* pattern_ptr, + size_t pattern_size, + std::vector& packed_info) { + + // 1. Validate input arguments + guarantee(fill_size >= pattern_size, "Pattern Size cannot be greater than fill size"); + guarantee(fill_size <= memory.size(), "Cannot fill more than the mem object size"); + + // 2. Calculate the next closest dword aligned address for faster processing + size_t dst_addr = memory.virtualAddress() + fill_origin; + size_t aligned_dst_addr = amd::alignUp(dst_addr, sizeof(size_t)); + guarantee(aligned_dst_addr >= dst_addr, "Aligned address cannot be greater than destination" + "address"); + + // 3. If given address is not aligned calculate head and tail size. + size_t head_size = (aligned_dst_addr - dst_addr); + size_t aligned_size = ((fill_size - head_size) / sizeof(size_t)) * sizeof(size_t); + size_t tail_size = (fill_size - head_size) % sizeof(size_t); + + // 4. Clear unwanted bytes from the pattern if the pattern size is < sizeof(size_t). + size_t pattern = *(reinterpret_cast(const_cast(pattern_ptr))); + if (pattern_size < sizeof(size_t)) { + ClearBits64(pattern, (pattern_size * 8)); + } + + // 5. Fill the head, aligned, tail info if they exist. + FillBufferInfo fill_info; + if (head_size > 0) { + // Offsetted ptrs should align with pattern size. Runtime not responsible for rotating pattern. + guarantee((head_size % pattern_size) == 0); + + fill_info.fill_size_ = head_size; + packed_info.push_back(fill_info); + } + + fill_info.clearInfo(); + if (aligned_size > 0) { + // Offsetted ptrs should align with pattern size. Runtime not responsible for rotating pattern. + guarantee((aligned_size % pattern_size) == 0); + + if (pattern_size < sizeof(size_t)) { + if (!ExpandPattern64(pattern, pattern_size, fill_info.expanded_pattern_)) { + DevLogPrintfError("Failed Expanding the pattern for pattern:%u, pattern_size: %u", + pattern, pattern_size); + return false; + } + fill_info.pattern_expanded_ = true; + } + fill_info.fill_size_ = aligned_size; + packed_info.push_back(fill_info); + } + fill_info.clearInfo(); + + if (tail_size > 0) { + // Offsetted ptrs should align with pattern size. Runtime not responsible for rotating pattern. + guarantee((tail_size % pattern_size) == 0); + + fill_info.fill_size_ = tail_size; + packed_info.push_back(fill_info); + } + fill_info.clearInfo(); + + return true; +} } // namespace gpu diff --git a/rocclr/device/blit.hpp b/rocclr/device/blit.hpp index 8ffd6d85ca..fce6f503ef 100644 --- a/rocclr/device/blit.hpp +++ b/rocclr/device/blit.hpp @@ -351,6 +351,37 @@ class HostBlitManager : public device::BlitManager { VirtualDevice& vDev_; //!< Virtual device object const amd::Device& dev_; //!< Physical device + // Packed Fill Buffer + class FillBufferInfo { + public: + FillBufferInfo(): fill_size_(0), expanded_pattern_(0), pattern_expanded_(false) {} + + static bool PackInfo(const device::Memory& memory, size_t fill_size, + size_t fill_origin, const void* pattern, size_t pattern_size, + std::vector& packed_info); + + private: + static bool ExpandPattern64(size_t pattern, size_t pattern_size, size_t& pattern64); + + static inline void ClearBits64(size_t& pattern, size_t num_bits) { + pattern &= ~(~(static_cast(0)) << num_bits); + + } + + void clearInfo () { + fill_size_ = 0; + expanded_pattern_ = 0; + pattern_expanded_ = false; + } + + public: + size_t fill_size_; // Fill size for this command + size_t expanded_pattern_; // Pattern for this command + bool pattern_expanded_; // Boolean to check if pattern is expanded + }; + + + private: //! Disable copy constructor HostBlitManager(const HostBlitManager&); diff --git a/rocclr/device/rocm/rocblit.cpp b/rocclr/device/rocm/rocblit.cpp index 9af2033448..a5398409ff 100644 --- a/rocclr/device/rocm/rocblit.cpp +++ b/rocclr/device/rocm/rocblit.cpp @@ -1960,68 +1960,88 @@ bool KernelBlitManager::fillBuffer(device::Memory& memory, const void* pattern, synchronize(); return result; } else { - uint fillType = FillBufferAligned; - size_t globalWorkOffset[3] = {0, 0, 0}; - uint64_t fillSize = size[0] / patternSize; - size_t globalWorkSize = amd::alignUp(fillSize, 256); - size_t localWorkSize = 256; - uint32_t alignment = (patternSize & 0x7) == 0 ? - sizeof(uint64_t) : - (patternSize & 0x3) == 0 ? - sizeof(uint32_t) : - (patternSize & 0x1) == 0 ? - sizeof(uint16_t) : sizeof(uint8_t); - // Program kernels arguments for the fill operation - cl_mem mem = as_cl(memory.owner()); - if (alignment == sizeof(uint64_t)) { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), &mem); - } else if (alignment == sizeof(uint32_t)) { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), &mem); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); - } else if (alignment == sizeof(uint16_t)) { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), &mem); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); - } else { - setArgument(kernels_[fillType], 0, sizeof(cl_mem), &mem); - setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); - setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); + // Pack the fill buffer info, that handles unaligned memories. + std::vector packed_vector{}; + FillBufferInfo::PackInfo(memory, size[0], origin[0], pattern, patternSize, packed_vector); + + size_t overall_offset = origin[0]; + for (auto& packed_obj: packed_vector) { + uint fillType = FillBufferAligned; + size_t globalWorkOffset[3] = {0, 0, 0}; + size_t globalWorkSize = amd::alignUp(packed_obj.fill_size_, 256); + size_t localWorkSize = 256; + + uint32_t kpattern_size32 = (packed_obj.pattern_expanded_) ? sizeof(size_t) : patternSize; + size_t kfill_size = packed_obj.fill_size_/kpattern_size32; + size_t koffset = overall_offset; + overall_offset += packed_obj.fill_size_; + + uint32_t alignment = (kpattern_size32 & 0x7) == 0 ? + sizeof(uint64_t) : + (kpattern_size32 & 0x3) == 0 ? + sizeof(uint32_t) : + (kpattern_size32 & 0x1) == 0 ? + sizeof(uint16_t) : sizeof(uint8_t); + + // Program kernels arguments for the fill operation + cl_mem mem = as_cl(memory.owner()); + if (alignment == sizeof(uint64_t)) { + setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 3, sizeof(cl_mem), &mem); + } else if (alignment == sizeof(uint32_t)) { + setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 2, sizeof(cl_mem), &mem); + setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); + } else if (alignment == sizeof(uint16_t)) { + setArgument(kernels_[fillType], 0, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 1, sizeof(cl_mem), &mem); + setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); + } else { + setArgument(kernels_[fillType], 0, sizeof(cl_mem), &mem); + setArgument(kernels_[fillType], 1, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 2, sizeof(cl_mem), nullptr); + setArgument(kernels_[fillType], 3, sizeof(cl_mem), nullptr); + } + + Memory* gpuCB = dev().getRocMemory(constantBuffer_); + if (gpuCB == nullptr) { + return false; + } + + // Find offset in the current constant buffer to allow multipel fills + uint32_t constBufOffset = ConstantBufferOffset(); + auto constBuf = reinterpret_cast
(constantBuffer_->getHostMem()) + constBufOffset; + + // If pattern has been expanded, use the expanded pattern, otherwise use the default pattern. + if (packed_obj.pattern_expanded_) { + memcpy(constBuf, &packed_obj.expanded_pattern_, kpattern_size32); + } else { + memcpy(constBuf, pattern, kpattern_size32); + } + + mem = as_cl(gpuCB->owner()); + setArgument(kernels_[fillType], 4, sizeof(cl_mem), &mem, constBufOffset); + + koffset /= alignment; + kpattern_size32 /= alignment; + + setArgument(kernels_[fillType], 5, sizeof(uint32_t), &kpattern_size32); + setArgument(kernels_[fillType], 6, sizeof(koffset), &koffset); + setArgument(kernels_[fillType], 7, sizeof(kfill_size), &kfill_size); + + // Create ND range object for the kernel's execution + amd::NDRangeContainer ndrange(1, globalWorkOffset, &globalWorkSize, &localWorkSize); + + // Execute the blit + address parameters = captureArguments(kernels_[fillType]); + result = gpu().submitKernelInternal(ndrange, *kernels_[fillType], parameters, nullptr); + releaseArguments(parameters); } - Memory* gpuCB = dev().getRocMemory(constantBuffer_); - if (gpuCB == nullptr) { - return false; - } - // Find offset in the current constant buffer to allow multipel fills - uint32_t constBufOffset = ConstantBufferOffset(); - auto constBuf = reinterpret_cast
(constantBuffer_->getHostMem()) + constBufOffset; - memcpy(constBuf, pattern, patternSize); - - mem = as_cl(gpuCB->owner()); - setArgument(kernels_[fillType], 4, sizeof(cl_mem), &mem, constBufOffset); - uint64_t offset = origin[0]; - - patternSize/= alignment; - offset /= alignment; - - setArgument(kernels_[fillType], 5, sizeof(uint32_t), &patternSize); - setArgument(kernels_[fillType], 6, sizeof(offset), &offset); - setArgument(kernels_[fillType], 7, sizeof(fillSize), &fillSize); - - // Create ND range object for the kernel's execution - amd::NDRangeContainer ndrange(1, globalWorkOffset, &globalWorkSize, &localWorkSize); - - // Execute the blit - address parameters = captureArguments(kernels_[fillType]); - result = gpu().submitKernelInternal(ndrange, *kernels_[fillType], parameters, nullptr); - releaseArguments(parameters); } synchronize();