From e3446ea82cade39df05f04c7fcd80ad5779c9c38 Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 4 May 2015 16:20:24 -0400 Subject: [PATCH] P4 to Git Change 1147021 by yaxunl@yaxunl_stg_win50 on 2015/05/04 16:06:06 ECR #304775 - Wave limiter: Fix bug in adaptation. Dumped waves/simd value is incorrect. Should exit adptation only after the changed waves/simd value is applied. Added wave limiter manager to handle situation that one kernel is enqueued to more than one queues. Create wave limiter for each virtual device. Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#245 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.cpp#283 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.hpp#109 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuvirtual.cpp#360 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuwavelimiter.cpp#4 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuwavelimiter.hpp#3 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#70 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/command.hpp#76 edit [ROCm/clr commit: 51efa976bfc18ac4742b992013038056c614b280] --- projects/clr/rocclr/runtime/device/device.hpp | 5 +- .../rocclr/runtime/device/gpu/gpukernel.cpp | 2 +- .../rocclr/runtime/device/gpu/gpukernel.hpp | 7 +- .../rocclr/runtime/device/gpu/gpuvirtual.cpp | 1 + .../runtime/device/gpu/gpuwavelimiter.cpp | 133 ++++++++++++------ .../runtime/device/gpu/gpuwavelimiter.hpp | 53 +++++-- .../clr/rocclr/runtime/platform/command.cpp | 4 +- 7 files changed, 144 insertions(+), 61 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/device.hpp b/projects/clr/rocclr/runtime/device/device.hpp index 17deaa9a18..b56bc64ffc 100644 --- a/projects/clr/rocclr/runtime/device/device.hpp +++ b/projects/clr/rocclr/runtime/device/device.hpp @@ -71,7 +71,7 @@ namespace option { class Options; } // option -struct ProfilingCallback: public amd::EmbeddedObject { +struct ProfilingCallback: public amd::HeapObject { virtual void callback (ulong duration) = 0; }; } @@ -938,7 +938,8 @@ public: } //! Get profiling callback object - virtual amd::ProfilingCallback* getProfilingCallback() const { + virtual amd::ProfilingCallback* getProfilingCallback( + const device::VirtualDevice *vdv) { return NULL; } diff --git a/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp b/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp index 7fb62de2f8..29c5b7f30a 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp @@ -1683,7 +1683,7 @@ Kernel::run(VirtualGPU& gpu, GpuEvent* calEvent, bool lastRun) const } } - gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH()); + gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH(&gpu)); if (!gpu.runProgramGrid(*calEvent, const_cast(&gpu.cal()->progGrid_), gpu.vmMems(), gpu.cal_.memCount_)) { LogError("Failed to execute the program!"); diff --git a/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp b/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp index f271754ea0..783e14cba5 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp @@ -654,8 +654,9 @@ public: ) const; //! Get profiling callback object - virtual amd::ProfilingCallback* getProfilingCallback() const { - return waveLimiter_.getProfilingCallback(); + virtual amd::ProfilingCallback* getProfilingCallback( + const device::VirtualDevice *vdev){ + return waveLimiter_.getProfilingCallback(vdev); } protected: @@ -775,7 +776,7 @@ private: //! @todo remove the blit kernel hack bool blitKernelHack_; //!< No VM hack for kernel blit - WaveLimiter waveLimiter_; //!< adaptively control number of waves + WaveLimiterManager waveLimiter_; //!< adaptively control number of waves }; enum HSAIL_ADDRESS_QUALIFIER{ diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp index 43c6e42e8d..902c6fa885 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp @@ -2806,6 +2806,7 @@ VirtualGPU::awaitCompletion(CommandBatch* cb, const amd::Event* waitingEvent) // Make sure that profiling is enabled if (profileEnabled_) { + profileEnabled_ = false; return profilingCollectResults(cb, waitingEvent); } // Mark the first command in the batch as running diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp index 798a6b763b..58b624a59b 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp @@ -27,28 +27,17 @@ void WaveLimiter::clearData() { clear(trial_); clear(ratio_); discontinuous_ = false; - waveSet_ = false; dataCount_ = 0; } -void WaveLimiter::enable() { - if (waves_ > 0) { - return; - } - auto gpuDev = reinterpret_cast(&owner_->dev()); - auto hwInfo = gpuDev->hwInfo(); - // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1 - setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE, - owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_); - if (!enable_) { - return; - } - waves_ = MaxWave; -} - -WaveLimiter::WaveLimiter(Kernel *owner) : - owner_(owner), dumper_(owner_->name()) { - auto gpuDev = reinterpret_cast(&owner_->dev()); +WaveLimiter::WaveLimiter( + Kernel* owner, + uint seqNum, + bool enable, + bool enableDump): + owner_(owner), + dumper_(owner_->name() + "_" + std::to_string(seqNum), enableDump) { + auto gpuDev = static_cast(&owner_->dev()); auto attrib = gpuDev->getAttribs(); auto hwInfo = gpuDev->hwInfo(); setIfNotDefault(SIMDPerSH_, GPU_WAVE_LIMIT_CU_PER_SH, @@ -73,10 +62,10 @@ WaveLimiter::WaveLimiter(Kernel *owner) : ".txt"); } - waves_ = GPU_WAVES_PER_SIMD; + waves_ = MaxWave; + currWaves_ = MaxWave; bestWave_ = MaxWave; - enable_ = false; - waveSet_ = false; + enable_ = enable; } WaveLimiter::~WaveLimiter() { @@ -85,8 +74,8 @@ WaveLimiter::~WaveLimiter() { } } -uint WaveLimiter::getWavesPerSH() const { - waveSet_ = true; +uint WaveLimiter::getWavesPerSH(){ + currWaves_ = waves_; return waves_ * SIMDPerSH_; } @@ -94,6 +83,7 @@ void WaveLimiter::updateData(ulong time) { auto count = dataCount_ - 1; assert(count < 2 * MaxWave + 1); assert(time > 0); + assert(currWaves_ == waves_); if (count % 2 == 0) { assert(waves_ == MaxWave); auto pos = count / 2; @@ -124,7 +114,8 @@ void WaveLimiter::outputTrace() { } traceStream_ << "[WaveLimiter] " << owner_->name() << " state=" << state_ - << " waves=" << waves_ << " bestWave=" << bestWave_ << '\n'; + << " currWaves=" << currWaves_ << " waves=" << waves_ + << " bestWave=" << bestWave_ << '\n'; output(traceStream_, "\n measure = ", measure_); output(traceStream_, "\n reference = ", reference_); output(traceStream_, "\n ratio = ", ratio_); @@ -132,7 +123,7 @@ void WaveLimiter::outputTrace() { } void WaveLimiter::callback(ulong duration) { - dumper_.addData(duration, waves_, static_cast(state_)); + dumper_.addData(duration, currWaves_, static_cast(state_)); if (!enable_) { return; @@ -151,7 +142,7 @@ void WaveLimiter::callback(ulong duration) { return; case ADAPT: assert(duration > 0); - if (waveSet_) { + if (waves_ == currWaves_) { dataCount_++; updateData(duration); waves_ = MaxWave + 1 - dataCount_ / 2; @@ -163,19 +154,17 @@ void WaveLimiter::callback(ulong duration) { } else { waves_ = MaxWave; } - waveSet_ = false; return; } + waves_ = bestWave_; + if (dataCount_ >= AdaptCount) { + dynRunCount_ = RunCount; + } else { + dynRunCount_ = AdaptCount; + } + countAll_ = rand() % MaxWave; + state_ = RUN; } - waves_ = bestWave_; - waveSet_ = false; - if (dataCount_ >= AdaptCount) { - dynRunCount_ = RunCount; - } else { - dynRunCount_ = AdaptCount; - } - countAll_ = rand() % MaxWave; - state_ = RUN; return; case RUN: if (countAll_ < dynRunCount_) { @@ -188,8 +177,8 @@ void WaveLimiter::callback(ulong duration) { } } -WaveLimiter::DataDumper::DataDumper(const std::string &kernelName) { - enable_ = !flagIsDefault(GPU_WAVE_LIMIT_DUMP); +WaveLimiter::DataDumper::DataDumper(const std::string &kernelName, bool enable) { + enable_ = enable; if (enable_) { fileName_ = std::string(GPU_WAVE_LIMIT_DUMP) + kernelName + ".csv"; } @@ -218,11 +207,65 @@ void WaveLimiter::DataDumper::addData(ulong time, uint wave, char state) { state_.push_back(state); } -amd::ProfilingCallback* WaveLimiter::getProfilingCallback() const { - if (enable_ || dumper_.enabled()) { - return const_cast(this); - } - return NULL; -} +WaveLimiterManager::WaveLimiterManager(Kernel* kernel): + owner_(kernel), + enable_(false), + enableDump_(!flagIsDefault(GPU_WAVE_LIMIT_DUMP)), + fixed_(GPU_WAVES_PER_SIMD) { +} + +WaveLimiterManager::~WaveLimiterManager() { + for (auto &I: limiters_) { + delete I.second; + } +} + +uint WaveLimiterManager::getWavesPerSH(const device::VirtualDevice *vdev) const { + if (fixed_ > 0) { + return fixed_; + } + if (!enable_) { + return 0; + } + auto loc = limiters_.find(vdev); + assert (loc != limiters_.end()); + assert(loc->second != NULL); + return loc->second->getWavesPerSH(); +} + +amd::ProfilingCallback* WaveLimiterManager::getProfilingCallback( + const device::VirtualDevice *vdev) { + assert(vdev != NULL); + if (!enable_ && !enableDump_) { + return NULL; + } + + amd::ScopedLock SL(monitor_); + auto loc = limiters_.find(vdev); + if (loc != limiters_.end()) { + return loc->second; + } + + auto limiter = new WaveLimiter(owner_, limiters_.size(), enable_, + enableDump_); + if (limiter == NULL) { + enable_ = false; + return NULL; + } + limiters_[vdev] = limiter; + return limiter; +} + +void WaveLimiterManager::enable() { + if (fixed_ > 0) { + return; + } + auto gpuDev = static_cast(&owner_->dev()); + auto hwInfo = gpuDev->hwInfo(); + // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1 + setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE, + owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_); +} + } diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp index bc7816d25f..49c25adddf 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp @@ -6,10 +6,12 @@ #define GPUWAVELIMITER_HPP_ #include "platform/command.hpp" +#include "thread/thread.hpp" #include #include #include #include +#include //! \namespace gpu GPU Device Implementation namespace gpu { @@ -19,11 +21,11 @@ class Kernel; // Adaptively limit the number of waves per SIMD based on kernel execution time class WaveLimiter: public amd::ProfilingCallback { public: - explicit WaveLimiter(Kernel*); - ~WaveLimiter(); - uint getWavesPerSH() const; - amd::ProfilingCallback* getProfilingCallback() const; - void enable(); + explicit WaveLimiter(Kernel*, uint seqNum, bool enable, bool enableDump); + virtual ~WaveLimiter(); + + //! Get waves per shader array to be used for kernel execution. + uint getWavesPerSH(); private: enum StateKind { @@ -32,9 +34,13 @@ private: class DataDumper { public: - explicit DataDumper(const std::string &kernelName); + explicit DataDumper(const std::string &kernelName, bool enable); ~DataDumper(); + + //! Record execution time, waves/simd and state of wave limiter. void addData(ulong time, uint wave, char state); + + //! Whether this data dumper is enabled. bool enabled() const { return enable_;} private: bool enable_; @@ -52,7 +58,7 @@ private: bool enable_; uint SIMDPerSH_; // Number of SIMDs per SH - uint waves_; // waves_ per SIMD + uint waves_; // Waves per SIMD to be set uint bestWave_; // Optimal waves per SIMD uint countAll_; // Number of kernel executions uint dynRunCount_; @@ -60,7 +66,7 @@ private: Kernel *owner_; DataDumper dumper_; std::ofstream traceStream_; - mutable bool waveSet_; + uint currWaves_; // Current waves per SIMD uint dataCount_; static uint MaxWave; // Maximum number of waves per SIMD @@ -70,9 +76,16 @@ private: static uint AbandonThresh; // Threshold to abandon adaptation static uint DscThresh; // Threshold for identifying discontinuities + //! Call back from Event::recordProfilingInfo to get execution time. virtual void callback(ulong duration); + + //! Update measurement data and optimal waves/simd with execution time. void updateData(ulong time); + + //! Output trace of measurement/adaptation. void outputTrace(); + + //! Clear measurement data for the next adaptation. void clearData(); template void clear(T& A) { @@ -88,5 +101,29 @@ private: } } }; + +// Create wave limiter for each virtual device for a kernel and manages the wave limiters. +class WaveLimiterManager { +public: + explicit WaveLimiterManager(Kernel* owner); + virtual ~WaveLimiterManager(); + + //! Get waves per shader array for a specific virtual device. + uint getWavesPerSH(const device::VirtualDevice *) const; + + //! Provide call back function for a specific virtual device. + amd::ProfilingCallback* getProfilingCallback(const device::VirtualDevice *); + + //! Enable wave limiter manager by kernel metadata and flags. + void enable(); +private: + Kernel *owner_; // The kernel which owns this object + std::unordered_map limiters_; // Maps virtual device to wave limiter + bool enable_; // Whether the adaptation is enabled + bool enableDump_; // Whether the data dumper is enabled + uint fixed_; // The fixed waves/simd value if not zero + amd::Monitor monitor_; // The mutex for updating the wave limiter map +}; } #endif diff --git a/projects/clr/rocclr/runtime/platform/command.cpp b/projects/clr/rocclr/runtime/platform/command.cpp index a2fe754606..cf2cbfe0f5 100644 --- a/projects/clr/rocclr/runtime/platform/command.cpp +++ b/projects/clr/rocclr/runtime/platform/command.cpp @@ -266,8 +266,8 @@ NDRangeKernelCommand::NDRangeKernelCommand( { parameters_ = kernel.parameters().capture(queue.device()); auto& device = queue.device(); - auto devKernel = kernel.getDeviceKernel(device); - profilingInfo_.setCallback(devKernel->getProfilingCallback()); + auto devKernel = const_cast(kernel.getDeviceKernel(device)); + profilingInfo_.setCallback(devKernel->getProfilingCallback(queue.vdev())); fixme_guarantee(parameters_ != NULL && "out of memory"); kernel_.retain(); }