diff --git a/projects/clr/rocclr/runtime/device/device.hpp b/projects/clr/rocclr/runtime/device/device.hpp index b45a7cb131..17deaa9a18 100644 --- a/projects/clr/rocclr/runtime/device/device.hpp +++ b/projects/clr/rocclr/runtime/device/device.hpp @@ -71,6 +71,9 @@ namespace option { class Options; } // option +struct ProfilingCallback: public amd::EmbeddedObject { + virtual void callback (ulong duration) = 0; +}; } enum OclExtensions { @@ -872,6 +875,7 @@ public: size_t compileSizeHint_[3]; //!< kernel compiled workgroup size hint std::string compileVecTypeHint_; //!< kernel compiled vector type hint bool uniformWorkGroupSize_; //!< uniform work group size option + bool limitWave_; //!< adaptively limit waves per SH }; //! Default constructor @@ -933,6 +937,11 @@ public: return workGroupInfo_.compileSizeHint_[dim]; } + //! Get profiling callback object + virtual amd::ProfilingCallback* getProfilingCallback() const { + return NULL; + } + void setVecTypeHint(const std::string& hint) { workGroupInfo_.compileVecTypeHint_ = hint; diff --git a/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp b/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp index b332749399..5c606fd84a 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp @@ -58,6 +58,7 @@ const MetaDataConst ArgState[ArgStateTotal] = { "printfid:", KernelArg::PrintfBufId, { 0, 0, 0, 0, 0, 0, 0 } }, { "wsh:", KernelArg::GroupingHint, { 0, 0, 0, 0, 0, 0, 0 } }, { "vth:", KernelArg::VecTypeHint, { 0, 0, 0, 0, 0, 0, 0 } }, + { "limitwave:", KernelArg::LimitWave, { 0, 0, 0, 0, 0, 0, 0 } }, }; const DataTypeConst DataType[] = @@ -829,6 +830,10 @@ Kernel::create( } } + // Wave limiter needs to be initialized after kernel metadata is parsed + // Since it depends on it. + waveLimiter_.enable(); + if (result) { buildError_ = CL_SUCCESS; } @@ -846,6 +851,7 @@ Kernel::Kernel( const InitData* initData) : NullKernel(name, gpuDev, prog) , blitKernelHack_(false) + , waveLimiter_(this) { hwPrivateSize_ = 0; if (NULL != initData) { @@ -1671,6 +1677,7 @@ Kernel::run(VirtualGPU& gpu, GpuEvent* calEvent, bool lastRun) const } } + gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH()); if (!gpu.runProgramGrid(*calEvent, const_cast(&gpu.cal()->progGrid_), gpu.vmMems(), gpu.cal_.memCount_)) { LogError("Failed to execute the program!"); @@ -2711,6 +2718,15 @@ NullKernel::parseArguments(const std::string& metaData, uint* uavRefCount) } // Process next ... continue; + case KernelArg::LimitWave: + { + uint tmp; + if (!getuint(metaData, &pos, &tmp)) { + return false; + } + workGroupInfo_.limitWave_ = tmp!=0; + } + continue; default: break; } diff --git a/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp b/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp index 88c089cf64..f271754ea0 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp @@ -15,6 +15,7 @@ #include "device/gpu/gpuvirtual.hpp" #include "sc/Interface/SCHSAInterface.h" #include "device/gpu/gpuprintf.hpp" +#include "device/gpu/gpuwavelimiter.hpp" #include "hsa.h" //! \namespace gpu GPU Device Implementation namespace gpu { @@ -187,6 +188,7 @@ public: PrintfBufId, GroupingHint, VecTypeHint, + LimitWave, TotalTypes }; @@ -312,7 +314,7 @@ struct MetaDataConst }; const uint DescTotal = 15; -const uint BasicTypeTotal = 14; +const uint BasicTypeTotal = 15; const uint ArgStateTotal = DescTotal + BasicTypeTotal; //! The constant array that describes different metadata properties @@ -651,6 +653,11 @@ public: VirtualGPU::GslKernelDesc* desc //!< Kernel descriptor ) const; + //! Get profiling callback object + virtual amd::ProfilingCallback* getProfilingCallback() const { + return waveLimiter_.getProfilingCallback(); + } + protected: //! Initializes the kernel parameters for the abstraction layer bool initParameters(); @@ -767,6 +774,8 @@ private: //! @todo remove the blit kernel hack bool blitKernelHack_; //!< No VM hack for kernel blit + + WaveLimiter waveLimiter_; //!< adaptively control number of waves }; enum HSAIL_ADDRESS_QUALIFIER{ diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp new file mode 100644 index 0000000000..47353e9e42 --- /dev/null +++ b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp @@ -0,0 +1,190 @@ +// +// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved. +// + +#include "device/gpu/gpukernel.hpp" +#include "device/gpu/gpuwavelimiter.hpp" +#include "os/os.hpp" +#include "utils/flags.hpp" + +namespace gpu { + +uint WaveLimiter::MaxWave; +uint WaveLimiter::WarmUpCount; +uint WaveLimiter::AdaptCount; +uint WaveLimiter::RunCount; +uint WaveLimiter::AbandonThresh; + +void WaveLimiter::clearData() { + waves_ = MaxWave; + countAll_ = 0; + clear(counts_); + clear(sum_); + clear(average_); + clear(ratio_); +} + +void WaveLimiter::enable() { + if (waves_ > 0) { + return; + } + auto gpuDev = reinterpret_cast(&owner_->dev()); + auto hwInfo = gpuDev->hwInfo(); + // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1 + setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE, + owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_); + if (!enable_) { + return; + } + waves_ = MaxWave; +} + +WaveLimiter::WaveLimiter(Kernel *owner) : + owner_(owner), dumper_(owner_->name()) { + auto gpuDev = reinterpret_cast(&owner_->dev()); + auto attrib = gpuDev->getAttribs(); + auto hwInfo = gpuDev->hwInfo(); + setIfNotDefault(SIMDPerSH_, GPU_WAVE_LIMIT_CU_PER_SH, + attrib.numberOfCUsperShaderArray * hwInfo->simdPerCU_); + + state_ = WARMUP; + dynRunCount_ = RunCount; + auto size = MaxWave + 1; + counts_.resize(size); + sum_.resize(size); + average_.resize(size); + ratio_.resize(size); + clearData(); + if (!flagIsDefault(GPU_WAVE_LIMIT_TRACE)) { + traceStream_.open(std::string(GPU_WAVE_LIMIT_TRACE) + owner_->name() + + ".txt"); + } + + MaxWave = GPU_WAVE_LIMIT_MAX_WAVE; + WarmUpCount = GPU_WAVE_LIMIT_WARMUP; + AdaptCount = GPU_WAVE_LIMIT_ADAPT * MaxWave; + RunCount = GPU_WAVE_LIMIT_RUN * MaxWave; + AbandonThresh = GPU_WAVE_LIMIT_ABANDON; + + waves_ = GPU_WAVES_PER_SIMD; + bestWave_ = MaxWave; + enable_ = false; +} + +WaveLimiter::~WaveLimiter() { + if (traceStream_.is_open()) { + traceStream_.close(); + } +} + +uint WaveLimiter::getWavesPerSH() const { + return waves_ * SIMDPerSH_; +} + +void WaveLimiter::updateData(ulong time) { + sum_[waves_] += time; + counts_[waves_]++; + average_[waves_] = sum_[waves_] / counts_[waves_]; + ratio_[waves_] = average_[waves_] * 100 / average_[MaxWave]; + if (average_[bestWave_] > average_[waves_]) { + bestWave_ = waves_; + } + outputTrace(); +} + +void WaveLimiter::outputTrace() { + if (!traceStream_.is_open()) { + return; + } + + traceStream_ << "[WaveLimiter] " << owner_->name() << " state=" << state_ + << " waves=" << waves_ << " bestWave=" << bestWave_ << '\n'; + output(traceStream_, "\n counts = ", counts_); + output(traceStream_, "\n sum = ", sum_); + output(traceStream_, "\n average = ", average_); + output(traceStream_, "\n ratio = ", ratio_); + traceStream_ << "\n\n"; +} + +void WaveLimiter::callback(ulong duration) { + dumper_.addData(duration, waves_, static_cast(state_)); + + if (!enable_) { + return; + } + + countAll_++; + + switch (state_) { + case WARMUP: + if (countAll_ < WarmUpCount) { + return; + } + state_ = ADAPT; + bestWave_ = MaxWave; + clearData(); + return; + case ADAPT: + updateData(duration); + if (countAll_ < AdaptCount && ratio_[waves_] < AbandonThresh) { + waves_ = MaxWave - (countAll_ % MaxWave); + return; + } + waves_ = bestWave_; + if (countAll_ >= AdaptCount) { + dynRunCount_ = RunCount; + } else { + dynRunCount_ = AdaptCount; + } + countAll_ = rand() % MaxWave; + state_ = RUN; + return; + case RUN: + if (countAll_ < dynRunCount_) { + return; + } + state_ = ADAPT; + bestWave_ = MaxWave; + clearData(); + return; + } +} + +WaveLimiter::DataDumper::DataDumper(const std::string &kernelName) { + enable_ = !flagIsDefault(GPU_WAVE_LIMIT_DUMP); + if (enable_) { + fileName_ = std::string(GPU_WAVE_LIMIT_DUMP) + kernelName + ".csv"; + } +} + +WaveLimiter::DataDumper::~DataDumper() { + if (!enable_) { + return; + } + + std::ofstream OFS(fileName_); + for (size_t i = 0, e = time_.size(); i != e; ++i) { + OFS << i << ',' << time_[i] << ',' << wavePerSIMD_[i] << ',' + << static_cast(state_[i]) << '\n'; + } + OFS.close(); +} + +void WaveLimiter::DataDumper::addData(ulong time, uint wave, char state) { + if (!enable_) { + return; + } + + time_.push_back(time); + wavePerSIMD_.push_back(wave); + state_.push_back(state); +} + +amd::ProfilingCallback* WaveLimiter::getProfilingCallback() const { + if (enable_ || dumper_.enabled()) { + return const_cast(this); + } + return NULL; +} +} + diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp new file mode 100644 index 0000000000..ca5f945e68 --- /dev/null +++ b/projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp @@ -0,0 +1,88 @@ +// +// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved. +// + +#ifndef GPUWAVELIMITER_HPP_ +#define GPUWAVELIMITER_HPP_ + +#include "platform/command.hpp" +#include +#include +#include +#include + +//! \namespace gpu GPU Device Implementation +namespace gpu { + +class Kernel; + +// Adaptively limit the number of waves per SIMD based on kernel execution time +class WaveLimiter: public amd::ProfilingCallback { +public: + explicit WaveLimiter(Kernel*); + ~WaveLimiter(); + uint getWavesPerSH() const; + amd::ProfilingCallback* getProfilingCallback() const; + void enable(); + +private: + enum StateKind { + WARMUP, ADAPT, RUN + }; + + class DataDumper { + public: + explicit DataDumper(const std::string &kernelName); + ~DataDumper(); + void addData(ulong time, uint wave, char state); + bool enabled() const { return enable_;} + private: + bool enable_; + std::string fileName_; + std::vector time_; + std::vector wavePerSIMD_; + std::vector state_; + }; + + std::vector counts_; + std::vector sum_; + std::vector average_; + std::vector ratio_; + + bool enable_; + uint SIMDPerSH_; // Number of SIMDs per SH + uint waves_; // waves_ per SIMD + uint bestWave_; // Optimal waves per SIMD + uint countAll_; // Number of kernel executions + uint dynRunCount_; + StateKind state_; + Kernel *owner_; + DataDumper dumper_; + std::ofstream traceStream_; + + static uint MaxWave; // Maximum number of waves per SIMD + static uint WarmUpCount; // Number of kernel executions for warm up + static uint AdaptCount; // Number of kernel executions for adapting + static uint RunCount; // Number of kernel executions for normal run + static uint AbandonThresh; // Threshold to abandon adaptation + + virtual void callback(ulong duration); + void updateData(ulong time); + void outputTrace(); + void clearData(); + + template void clear(T& A) { + for (auto &I : A) { + I = 0; + } + } + template void output(std::ofstream &ofs, const std::string &prompt, + T& A) { + ofs << prompt; + for (auto &I : A) { + ofs << ' ' << static_cast(I); + } + } +}; +} +#endif diff --git a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/include/cal/cal.h b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/include/cal/cal.h index 5b729f82c9..149b4fbf5f 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/include/cal/cal.h +++ b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/include/cal/cal.h @@ -207,6 +207,7 @@ typedef struct CALdeviceattribsRec { CALuint memoryClock; /**< GPU memory clock rate in megahertz */ CALuint wavefrontSize; /**< Wavefront size */ CALuint numberOfSIMD; /**< Number of SIMDs */ + CALuint numberOfCUsperShaderArray; /**< Number of CUs per shader array */ bool doublePrecision; /**< double precision supported */ bool localDataShare; /**< local data share supported */ bool globalDataShare; /**< global data share supported */ diff --git a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.cpp b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.cpp index 65537c48ad..0c391407e1 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.cpp @@ -5,6 +5,7 @@ #include "cm_if.h" #include "amuABI.h" #include "shader/ProgramObject.h" +#include "shader/ComputeProgramObject.h" #include "query/QueryObject.h" #include "query/PerformanceQueryObject.h" #include "constbuffer/ConstantBufferObject.h" @@ -400,6 +401,13 @@ CALGSLContext::setProgram(gslProgramObject func) m_rs->setCurrentProgramObject(GSL_COMPUTE_PROGRAM, func); } +void +CALGSLContext::setWavesPerSH(gslProgramObject func, uint32 wavesPerSH)const +{ + auto compProg = static_cast(func); + compProg->setWavesPerSH(wavesPerSH); +} + bool CALGSLContext::runProgramGrid(GpuEvent& event, const ProgramGrid* pProgramGrid, const gslMemObject* mems, uint32 numMems) { diff --git a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.h b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.h index b951d171aa..3a27116027 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.h +++ b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLContext.h @@ -41,6 +41,7 @@ public: void setUavMask(const CALUavMask& uavMask); void setUAVChannelOrder(uint32 physUnit, gslMemObject mem); void setProgram(gslProgramObject func); + void setWavesPerSH(gslProgramObject func, uint32 wavesPerSH)const; bool runProgramGrid(GpuEvent& event, const ProgramGrid* pProgramGrid, const gslMemObject* mems, uint32 numMems); bool runProgramVideoDecode(GpuEvent& event, gslMemObject mo, const CALprogramVideoDecode& decode); void runAqlDispatch(GpuEvent& event, const void* aqlPacket, const gslMemObject* mems, diff --git a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLDevice.cpp b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLDevice.cpp index 40f9120929..10a3d7e2ee 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLDevice.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gslbe/src/rt/GSLDevice.cpp @@ -116,6 +116,7 @@ CALGSLDevice::getAttribs_int(gsl::gsCtx* cs) m_attribs.engineClock = cs->getMaxEngineClock(); m_attribs.memoryClock = cs->getMaxMemoryClock(); m_attribs.numberOfSIMD = cs->getNumSIMD(); + m_attribs.numberOfCUsperShaderArray = cs->getNumCUsPerShaderArray(); m_attribs.wavefrontSize = cs->getWaveFrontSize(); m_attribs.doublePrecision = cs->getIsDoublePrecisionSupported(); m_attribs.localDataShare = cs->getIsLocalDataShareSupported(); diff --git a/projects/clr/rocclr/runtime/platform/command.cpp b/projects/clr/rocclr/runtime/platform/command.cpp index 98f13d014b..0b71140125 100644 --- a/projects/clr/rocclr/runtime/platform/command.cpp +++ b/projects/clr/rocclr/runtime/platform/command.cpp @@ -68,6 +68,9 @@ Event::recordProfilingInfo(cl_int status, uint64_t timeStamp) break; default: profilingInfo_.end_ = timeStamp; + if (profilingInfo_.callback_ != NULL) { + profilingInfo_.callback_->callback(timeStamp - profilingInfo_.start_); + } break; } return timeStamp; @@ -259,6 +262,9 @@ NDRangeKernelCommand::NDRangeKernelCommand( kernel_(kernel), sizes_(sizes) { parameters_ = kernel.parameters().capture(queue.device()); + auto& device = queue.device(); + auto devKernel = kernel.getDeviceKernel(device); + profilingInfo_.setCallback(devKernel->getProfilingCallback()); fixme_guarantee(parameters_ != NULL && "out of memory"); kernel_.retain(); } diff --git a/projects/clr/rocclr/runtime/platform/command.hpp b/projects/clr/rocclr/runtime/platform/command.hpp index d630dcc08b..e7311c72ba 100644 --- a/projects/clr/rocclr/runtime/platform/command.hpp +++ b/projects/clr/rocclr/runtime/platform/command.hpp @@ -89,10 +89,8 @@ protected: ProfilingInfo(bool enabled = false) : enabled_(enabled) { if (enabled) { - queued_ = 0ULL; - submitted_ = 0ULL; - start_ = 0ULL; - end_ = 0ULL; + clear(); + callback_ = NULL; } } @@ -100,7 +98,22 @@ protected: uint64_t submitted_; uint64_t start_; uint64_t end_; - const bool enabled_; + bool enabled_; + ProfilingCallback *callback_; + void clear() { + queued_ = 0ULL; + submitted_ = 0ULL; + start_ = 0ULL; + end_ = 0ULL; + } + void setCallback(ProfilingCallback *callback) { + if (callback == NULL) { + return; + } + enabled_ = true; + clear(); + callback_ = callback; + } } profilingInfo_; diff --git a/projects/clr/rocclr/runtime/utils/flags.hpp b/projects/clr/rocclr/runtime/utils/flags.hpp index db12e0f640..69a4bf894d 100644 --- a/projects/clr/rocclr/runtime/utils/flags.hpp +++ b/projects/clr/rocclr/runtime/utils/flags.hpp @@ -177,7 +177,27 @@ debug(bool, GPU_FORCE_SINGLE_FP_DENORM, false, \ debug(bool, OCL_FORCE_CPU_SVM, false, \ "force svm support for CPU") \ debug(bool, GPU_ENABLE_HW_DEBUG, false, \ - "Enable HW DEBUG for GPU") + "Enable HW DEBUG for GPU") \ +release(uint, GPU_WAVES_PER_SIMD, 0, \ + "Force the number of waves per SIMD (1-10)") \ +release(bool, GPU_WAVE_LIMIT_ENABLE, false, \ + "1 = Enable adaptive wave limiter") \ +release_on_stg(uint, GPU_WAVE_LIMIT_CU_PER_SH, 0, \ + "Assume the number of CU per SH for wave limiter") \ +release_on_stg(uint, GPU_WAVE_LIMIT_MAX_WAVE, 10, \ + "Set maximum waves per SIMD to try for wave limiter") \ +release_on_stg(uint, GPU_WAVE_LIMIT_WARMUP, 100, \ + "Set warming up kernel execution count for wave limiter") \ +release_on_stg(uint, GPU_WAVE_LIMIT_ADAPT, 1, \ + "Set adapting factor for wave limiter") \ +release_on_stg(uint, GPU_WAVE_LIMIT_RUN, 40, \ + "Set running factor for wave limiter") \ +release_on_stg(uint, GPU_WAVE_LIMIT_ABANDON, 105, \ + "Set abandon threshold for wave limiter") \ +release_on_stg(cstring, GPU_WAVE_LIMIT_DUMP, "", \ + "File path prefix for dumping wave limiter output") \ +release_on_stg(cstring, GPU_WAVE_LIMIT_TRACE, "", \ + "File path prefix for tracing wave limiter") @@ -243,6 +263,12 @@ public: #define flagIsDefault(name) \ (amd::Flag::cannotSet##name || amd::Flag::isDefault(amd::Flag::k##name)) +#define setIfNotDefault(var, opt, other) \ + if (!flagIsDefault(opt)) \ + var = (opt);\ + else \ + var = (other); + // @} } // namespace amd