diff --git a/projects/clr/rocclr/runtime/device/device.hpp b/projects/clr/rocclr/runtime/device/device.hpp index 17deaa9a18..b56bc64ffc 100644 --- a/projects/clr/rocclr/runtime/device/device.hpp +++ b/projects/clr/rocclr/runtime/device/device.hpp @@ -71,7 +71,7 @@ namespace option { class Options; } // option -struct ProfilingCallback: public amd::EmbeddedObject { +struct ProfilingCallback: public amd::HeapObject { virtual void callback (ulong duration) = 0; }; } @@ -938,7 +938,8 @@ public: } //! Get profiling callback object - virtual amd::ProfilingCallback* getProfilingCallback() const { + virtual amd::ProfilingCallback* getProfilingCallback( + const device::VirtualDevice *vdv) { return NULL; } diff --git a/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp b/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp index 7fb62de2f8..29c5b7f30a 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp @@ -1683,7 +1683,7 @@ Kernel::run(VirtualGPU& gpu, GpuEvent* calEvent, bool lastRun) const } } - gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH()); + gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH(&gpu)); if (!gpu.runProgramGrid(*calEvent, const_cast(&gpu.cal()->progGrid_), gpu.vmMems(), gpu.cal_.memCount_)) { LogError("Failed to execute the program!"); diff --git a/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp b/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp index f271754ea0..783e14cba5 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp @@ -654,8 +654,9 @@ public: ) const; //! Get profiling callback object - virtual amd::ProfilingCallback* getProfilingCallback() const { - return waveLimiter_.getProfilingCallback(); + virtual amd::ProfilingCallback* getProfilingCallback( + const device::VirtualDevice *vdev){ + return waveLimiter_.getProfilingCallback(vdev); } protected: @@ -775,7 +776,7 @@ private: //! @todo remove the blit kernel hack bool blitKernelHack_; //!< No VM hack for kernel blit - WaveLimiter waveLimiter_; //!< adaptively control number of waves + WaveLimiterManager waveLimiter_; //!< adaptively control number of waves }; enum HSAIL_ADDRESS_QUALIFIER{ diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp index 43c6e42e8d..902c6fa885 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp @@ -2806,6 +2806,7 @@ VirtualGPU::awaitCompletion(CommandBatch* cb, const amd::Event* waitingEvent) // Make sure that profiling is enabled if (profileEnabled_) { + profileEnabled_ = false; return profilingCollectResults(cb, waitingEvent); } // Mark the first command in the batch as running diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp index 798a6b763b..58b624a59b 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp @@ -27,28 +27,17 @@ void WaveLimiter::clearData() { clear(trial_); clear(ratio_); discontinuous_ = false; - waveSet_ = false; dataCount_ = 0; } -void WaveLimiter::enable() { - if (waves_ > 0) { - return; - } - auto gpuDev = reinterpret_cast(&owner_->dev()); - auto hwInfo = gpuDev->hwInfo(); - // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1 - setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE, - owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_); - if (!enable_) { - return; - } - waves_ = MaxWave; -} - -WaveLimiter::WaveLimiter(Kernel *owner) : - owner_(owner), dumper_(owner_->name()) { - auto gpuDev = reinterpret_cast(&owner_->dev()); +WaveLimiter::WaveLimiter( + Kernel* owner, + uint seqNum, + bool enable, + bool enableDump): + owner_(owner), + dumper_(owner_->name() + "_" + std::to_string(seqNum), enableDump) { + auto gpuDev = static_cast(&owner_->dev()); auto attrib = gpuDev->getAttribs(); auto hwInfo = gpuDev->hwInfo(); setIfNotDefault(SIMDPerSH_, GPU_WAVE_LIMIT_CU_PER_SH, @@ -73,10 +62,10 @@ WaveLimiter::WaveLimiter(Kernel *owner) : ".txt"); } - waves_ = GPU_WAVES_PER_SIMD; + waves_ = MaxWave; + currWaves_ = MaxWave; bestWave_ = MaxWave; - enable_ = false; - waveSet_ = false; + enable_ = enable; } WaveLimiter::~WaveLimiter() { @@ -85,8 +74,8 @@ WaveLimiter::~WaveLimiter() { } } -uint WaveLimiter::getWavesPerSH() const { - waveSet_ = true; +uint WaveLimiter::getWavesPerSH(){ + currWaves_ = waves_; return waves_ * SIMDPerSH_; } @@ -94,6 +83,7 @@ void WaveLimiter::updateData(ulong time) { auto count = dataCount_ - 1; assert(count < 2 * MaxWave + 1); assert(time > 0); + assert(currWaves_ == waves_); if (count % 2 == 0) { assert(waves_ == MaxWave); auto pos = count / 2; @@ -124,7 +114,8 @@ void WaveLimiter::outputTrace() { } traceStream_ << "[WaveLimiter] " << owner_->name() << " state=" << state_ - << " waves=" << waves_ << " bestWave=" << bestWave_ << '\n'; + << " currWaves=" << currWaves_ << " waves=" << waves_ + << " bestWave=" << bestWave_ << '\n'; output(traceStream_, "\n measure = ", measure_); output(traceStream_, "\n reference = ", reference_); output(traceStream_, "\n ratio = ", ratio_); @@ -132,7 +123,7 @@ void WaveLimiter::outputTrace() { } void WaveLimiter::callback(ulong duration) { - dumper_.addData(duration, waves_, static_cast(state_)); + dumper_.addData(duration, currWaves_, static_cast(state_)); if (!enable_) { return; @@ -151,7 +142,7 @@ void WaveLimiter::callback(ulong duration) { return; case ADAPT: assert(duration > 0); - if (waveSet_) { + if (waves_ == currWaves_) { dataCount_++; updateData(duration); waves_ = MaxWave + 1 - dataCount_ / 2; @@ -163,19 +154,17 @@ void WaveLimiter::callback(ulong duration) { } else { waves_ = MaxWave; } - waveSet_ = false; return; } + waves_ = bestWave_; + if (dataCount_ >= AdaptCount) { + dynRunCount_ = RunCount; + } else { + dynRunCount_ = AdaptCount; + } + countAll_ = rand() % MaxWave; + state_ = RUN; } - waves_ = bestWave_; - waveSet_ = false; - if (dataCount_ >= AdaptCount) { - dynRunCount_ = RunCount; - } else { - dynRunCount_ = AdaptCount; - } - countAll_ = rand() % MaxWave; - state_ = RUN; return; case RUN: if (countAll_ < dynRunCount_) { @@ -188,8 +177,8 @@ void WaveLimiter::callback(ulong duration) { } } -WaveLimiter::DataDumper::DataDumper(const std::string &kernelName) { - enable_ = !flagIsDefault(GPU_WAVE_LIMIT_DUMP); +WaveLimiter::DataDumper::DataDumper(const std::string &kernelName, bool enable) { + enable_ = enable; if (enable_) { fileName_ = std::string(GPU_WAVE_LIMIT_DUMP) + kernelName + ".csv"; } @@ -218,11 +207,65 @@ void WaveLimiter::DataDumper::addData(ulong time, uint wave, char state) { state_.push_back(state); } -amd::ProfilingCallback* WaveLimiter::getProfilingCallback() const { - if (enable_ || dumper_.enabled()) { - return const_cast(this); - } - return NULL; -} +WaveLimiterManager::WaveLimiterManager(Kernel* kernel): + owner_(kernel), + enable_(false), + enableDump_(!flagIsDefault(GPU_WAVE_LIMIT_DUMP)), + fixed_(GPU_WAVES_PER_SIMD) { +} + +WaveLimiterManager::~WaveLimiterManager() { + for (auto &I: limiters_) { + delete I.second; + } +} + +uint WaveLimiterManager::getWavesPerSH(const device::VirtualDevice *vdev) const { + if (fixed_ > 0) { + return fixed_; + } + if (!enable_) { + return 0; + } + auto loc = limiters_.find(vdev); + assert (loc != limiters_.end()); + assert(loc->second != NULL); + return loc->second->getWavesPerSH(); +} + +amd::ProfilingCallback* WaveLimiterManager::getProfilingCallback( + const device::VirtualDevice *vdev) { + assert(vdev != NULL); + if (!enable_ && !enableDump_) { + return NULL; + } + + amd::ScopedLock SL(monitor_); + auto loc = limiters_.find(vdev); + if (loc != limiters_.end()) { + return loc->second; + } + + auto limiter = new WaveLimiter(owner_, limiters_.size(), enable_, + enableDump_); + if (limiter == NULL) { + enable_ = false; + return NULL; + } + limiters_[vdev] = limiter; + return limiter; +} + +void WaveLimiterManager::enable() { + if (fixed_ > 0) { + return; + } + auto gpuDev = static_cast(&owner_->dev()); + auto hwInfo = gpuDev->hwInfo(); + // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1 + setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE, + owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_); +} + } diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp index bc7816d25f..49c25adddf 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp @@ -6,10 +6,12 @@ #define GPUWAVELIMITER_HPP_ #include "platform/command.hpp" +#include "thread/thread.hpp" #include #include #include #include +#include //! \namespace gpu GPU Device Implementation namespace gpu { @@ -19,11 +21,11 @@ class Kernel; // Adaptively limit the number of waves per SIMD based on kernel execution time class WaveLimiter: public amd::ProfilingCallback { public: - explicit WaveLimiter(Kernel*); - ~WaveLimiter(); - uint getWavesPerSH() const; - amd::ProfilingCallback* getProfilingCallback() const; - void enable(); + explicit WaveLimiter(Kernel*, uint seqNum, bool enable, bool enableDump); + virtual ~WaveLimiter(); + + //! Get waves per shader array to be used for kernel execution. + uint getWavesPerSH(); private: enum StateKind { @@ -32,9 +34,13 @@ private: class DataDumper { public: - explicit DataDumper(const std::string &kernelName); + explicit DataDumper(const std::string &kernelName, bool enable); ~DataDumper(); + + //! Record execution time, waves/simd and state of wave limiter. void addData(ulong time, uint wave, char state); + + //! Whether this data dumper is enabled. bool enabled() const { return enable_;} private: bool enable_; @@ -52,7 +58,7 @@ private: bool enable_; uint SIMDPerSH_; // Number of SIMDs per SH - uint waves_; // waves_ per SIMD + uint waves_; // Waves per SIMD to be set uint bestWave_; // Optimal waves per SIMD uint countAll_; // Number of kernel executions uint dynRunCount_; @@ -60,7 +66,7 @@ private: Kernel *owner_; DataDumper dumper_; std::ofstream traceStream_; - mutable bool waveSet_; + uint currWaves_; // Current waves per SIMD uint dataCount_; static uint MaxWave; // Maximum number of waves per SIMD @@ -70,9 +76,16 @@ private: static uint AbandonThresh; // Threshold to abandon adaptation static uint DscThresh; // Threshold for identifying discontinuities + //! Call back from Event::recordProfilingInfo to get execution time. virtual void callback(ulong duration); + + //! Update measurement data and optimal waves/simd with execution time. void updateData(ulong time); + + //! Output trace of measurement/adaptation. void outputTrace(); + + //! Clear measurement data for the next adaptation. void clearData(); template void clear(T& A) { @@ -88,5 +101,29 @@ private: } } }; + +// Create wave limiter for each virtual device for a kernel and manages the wave limiters. +class WaveLimiterManager { +public: + explicit WaveLimiterManager(Kernel* owner); + virtual ~WaveLimiterManager(); + + //! Get waves per shader array for a specific virtual device. + uint getWavesPerSH(const device::VirtualDevice *) const; + + //! Provide call back function for a specific virtual device. + amd::ProfilingCallback* getProfilingCallback(const device::VirtualDevice *); + + //! Enable wave limiter manager by kernel metadata and flags. + void enable(); +private: + Kernel *owner_; // The kernel which owns this object + std::unordered_map limiters_; // Maps virtual device to wave limiter + bool enable_; // Whether the adaptation is enabled + bool enableDump_; // Whether the data dumper is enabled + uint fixed_; // The fixed waves/simd value if not zero + amd::Monitor monitor_; // The mutex for updating the wave limiter map +}; } #endif diff --git a/projects/clr/rocclr/runtime/platform/command.cpp b/projects/clr/rocclr/runtime/platform/command.cpp index a2fe754606..cf2cbfe0f5 100644 --- a/projects/clr/rocclr/runtime/platform/command.cpp +++ b/projects/clr/rocclr/runtime/platform/command.cpp @@ -266,8 +266,8 @@ NDRangeKernelCommand::NDRangeKernelCommand( { parameters_ = kernel.parameters().capture(queue.device()); auto& device = queue.device(); - auto devKernel = kernel.getDeviceKernel(device); - profilingInfo_.setCallback(devKernel->getProfilingCallback()); + auto devKernel = const_cast(kernel.getDeviceKernel(device)); + profilingInfo_.setCallback(devKernel->getProfilingCallback(queue.vdev())); fixme_guarantee(parameters_ != NULL && "out of memory"); kernel_.retain(); }