P4 to Git Change 1147021 by yaxunl@yaxunl_stg_win50 on 2015/05/04 16:06:06

ECR #304775 - Wave limiter: Fix bug in adaptation.

	Dumped waves/simd value is incorrect.
	Should exit adptation only after the changed waves/simd value is applied.
	Added wave limiter manager to handle situation that one kernel is enqueued to more than one queues. Create wave limiter for each virtual device.

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#245 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.cpp#283 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.hpp#109 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuvirtual.cpp#360 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuwavelimiter.cpp#4 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuwavelimiter.hpp#3 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#70 edit
... //depot/stg/opencl/drivers/opencl/runtime/platform/command.hpp#76 edit


[ROCm/clr commit: 51efa976bf]

projects/clr/rocclr/runtime/device/device.hpp

@@ -71,7 +71,7 @@ namespace option {
 class Options;
 } // option
 
-struct ProfilingCallback: public amd::EmbeddedObject {
+struct ProfilingCallback: public amd::HeapObject {
     virtual void callback (ulong duration) = 0;
 };
 }
@@ -938,7 +938,8 @@ public:
     }
 
     //! Get profiling callback object
-    virtual amd::ProfilingCallback* getProfilingCallback() const {
+    virtual amd::ProfilingCallback* getProfilingCallback(
+        const device::VirtualDevice *vdv) {
         return NULL;
     }
 

projects/clr/rocclr/runtime/device/gpu/gpukernel.cpp

@@ -1683,7 +1683,7 @@ Kernel::run(VirtualGPU& gpu, GpuEvent* calEvent, bool lastRun) const
         }
     }
 
-    gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH());
+    gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH(&gpu));
     if (!gpu.runProgramGrid(*calEvent,
         const_cast<ProgramGrid*>(&gpu.cal()->progGrid_), gpu.vmMems(), gpu.cal_.memCount_)) {
         LogError("Failed to execute the program!");

projects/clr/rocclr/runtime/device/gpu/gpukernel.hpp

@@ -654,8 +654,9 @@ public:
         ) const;
 
     //! Get profiling callback object
-    virtual amd::ProfilingCallback* getProfilingCallback() const {
-        return waveLimiter_.getProfilingCallback();
+    virtual amd::ProfilingCallback* getProfilingCallback(
+        const device::VirtualDevice *vdev){
+        return waveLimiter_.getProfilingCallback(vdev);
     }
 
 protected:
@@ -775,7 +776,7 @@ private:
     //! @todo remove the blit kernel hack
     bool    blitKernelHack_;    //!< No VM hack for kernel blit
 
-    WaveLimiter waveLimiter_; //!< adaptively control number of waves
+    WaveLimiterManager waveLimiter_; //!< adaptively control number of waves
 };
 
 enum HSAIL_ADDRESS_QUALIFIER{

projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp

@@ -2806,6 +2806,7 @@ VirtualGPU::awaitCompletion(CommandBatch* cb, const amd::Event* waitingEvent)
 
     // Make sure that profiling is enabled
     if (profileEnabled_) {
+        profileEnabled_ = false;
         return profilingCollectResults(cb, waitingEvent);
     }
     // Mark the first command in the batch as running

projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.cpp

@@ -27,28 +27,17 @@ void WaveLimiter::clearData() {
     clear(trial_);
     clear(ratio_);
     discontinuous_ = false;
-    waveSet_ = false;
     dataCount_ = 0;
 }
 
-void WaveLimiter::enable() {
-    if (waves_ > 0) {
-        return;
-    }
-    auto gpuDev = reinterpret_cast<const Device*>(&owner_->dev());
-    auto hwInfo = gpuDev->hwInfo();
-    // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1
-    setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE,
-         owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_);
-    if (!enable_) {
-        return;
-    }
-    waves_ = MaxWave;
-}
-
-WaveLimiter::WaveLimiter(Kernel *owner) :
-        owner_(owner), dumper_(owner_->name()) {
-    auto gpuDev = reinterpret_cast<const Device*>(&owner_->dev());
+WaveLimiter::WaveLimiter(
+        Kernel* owner,
+        uint    seqNum,
+        bool    enable,
+        bool    enableDump):
+        owner_(owner),
+        dumper_(owner_->name() + "_" + std::to_string(seqNum), enableDump) {
+    auto gpuDev = static_cast<const Device*>(&owner_->dev());
     auto attrib = gpuDev->getAttribs();
     auto hwInfo = gpuDev->hwInfo();
     setIfNotDefault(SIMDPerSH_, GPU_WAVE_LIMIT_CU_PER_SH,
@@ -73,10 +62,10 @@ WaveLimiter::WaveLimiter(Kernel *owner) :
             ".txt");
     }
 
-    waves_ = GPU_WAVES_PER_SIMD;
+    waves_ = MaxWave;
+    currWaves_ = MaxWave;
     bestWave_ = MaxWave;
-    enable_ = false;
-    waveSet_ = false;
+    enable_ = enable;
 }
 
 WaveLimiter::~WaveLimiter() {
@@ -85,8 +74,8 @@ WaveLimiter::~WaveLimiter() {
     }
 }
 
-uint WaveLimiter::getWavesPerSH() const {
-    waveSet_ = true;
+uint WaveLimiter::getWavesPerSH(){
+    currWaves_ = waves_;
     return waves_ * SIMDPerSH_;
 }
 
@@ -94,6 +83,7 @@ void WaveLimiter::updateData(ulong time) {
     auto count = dataCount_ - 1;
     assert(count < 2 * MaxWave + 1);
     assert(time > 0);
+    assert(currWaves_ == waves_);
     if (count % 2 == 0) {
         assert(waves_ == MaxWave);
         auto pos = count / 2;
@@ -124,7 +114,8 @@ void WaveLimiter::outputTrace() {
     }
 
     traceStream_ << "[WaveLimiter] " << owner_->name() << " state=" << state_
-            << " waves=" << waves_ << " bestWave=" << bestWave_ << '\n';
+            << " currWaves=" << currWaves_ << " waves=" << waves_
+            << " bestWave=" << bestWave_ << '\n';
     output(traceStream_, "\n measure = ", measure_);
     output(traceStream_, "\n reference = ", reference_);
     output(traceStream_, "\n ratio = ", ratio_);
@@ -132,7 +123,7 @@ void WaveLimiter::outputTrace() {
 }
 
 void WaveLimiter::callback(ulong duration) {
-    dumper_.addData(duration, waves_, static_cast<char>(state_));
+    dumper_.addData(duration, currWaves_, static_cast<char>(state_));
 
     if (!enable_) {
         return;
@@ -151,7 +142,7 @@ void WaveLimiter::callback(ulong duration) {
         return;
     case ADAPT:
         assert(duration > 0);
-        if (waveSet_) {
+        if (waves_ == currWaves_) {
             dataCount_++;
             updateData(duration);
             waves_ = MaxWave + 1 - dataCount_ / 2;
@@ -163,19 +154,17 @@ void WaveLimiter::callback(ulong duration) {
                 } else {
                     waves_ = MaxWave;
                 }
-                waveSet_ = false;
                 return;
             }
+            waves_ = bestWave_;
+            if (dataCount_ >= AdaptCount) {
+                dynRunCount_ = RunCount;
+            } else {
+                dynRunCount_ = AdaptCount;
+            }
+            countAll_ = rand() % MaxWave;
+            state_ = RUN;
         }
-        waves_ = bestWave_;
-        waveSet_ = false;
-        if (dataCount_ >= AdaptCount) {
-            dynRunCount_ = RunCount;
-        } else {
-            dynRunCount_ = AdaptCount;
-        }
-        countAll_ = rand() % MaxWave;
-        state_ = RUN;
         return;
     case RUN:
         if (countAll_ < dynRunCount_) {
@@ -188,8 +177,8 @@ void WaveLimiter::callback(ulong duration) {
     }
 }
 
-WaveLimiter::DataDumper::DataDumper(const std::string &kernelName) {
-    enable_ = !flagIsDefault(GPU_WAVE_LIMIT_DUMP);
+WaveLimiter::DataDumper::DataDumper(const std::string &kernelName, bool enable) {
+    enable_ = enable;
     if (enable_) {
         fileName_ = std::string(GPU_WAVE_LIMIT_DUMP) + kernelName + ".csv";
     }
@@ -218,11 +207,65 @@ void WaveLimiter::DataDumper::addData(ulong time, uint wave, char state) {
     state_.push_back(state);
 }
 
-amd::ProfilingCallback* WaveLimiter::getProfilingCallback() const {
-    if (enable_ || dumper_.enabled()) {
-        return const_cast<WaveLimiter*>(this);
-    }
-    return NULL;
-}
+WaveLimiterManager::WaveLimiterManager(Kernel* kernel):
+        owner_(kernel),
+        enable_(false),
+        enableDump_(!flagIsDefault(GPU_WAVE_LIMIT_DUMP)),
+        fixed_(GPU_WAVES_PER_SIMD) {
+}
+
+WaveLimiterManager::~WaveLimiterManager() {
+    for (auto &I: limiters_) {
+        delete I.second;
+    }
+}
+
+uint WaveLimiterManager::getWavesPerSH(const device::VirtualDevice *vdev) const {
+    if (fixed_ > 0) {
+        return fixed_;
+    }
+    if (!enable_) {
+        return 0;
+    }
+    auto loc = limiters_.find(vdev);
+    assert (loc != limiters_.end());
+    assert(loc->second != NULL);
+    return loc->second->getWavesPerSH();
+}
+
+amd::ProfilingCallback* WaveLimiterManager::getProfilingCallback(
+        const device::VirtualDevice *vdev) {
+    assert(vdev != NULL);
+    if (!enable_ && !enableDump_) {
+        return NULL;
+    }
+
+    amd::ScopedLock SL(monitor_);
+    auto loc = limiters_.find(vdev);
+    if (loc != limiters_.end()) {
+        return loc->second;
+    }
+
+    auto limiter = new WaveLimiter(owner_, limiters_.size(), enable_,
+            enableDump_);
+    if (limiter == NULL) {
+        enable_ = false;
+        return NULL;
+    }
+    limiters_[vdev] = limiter;
+    return limiter;
+}
+
+void WaveLimiterManager::enable() {
+    if (fixed_ > 0) {
+        return;
+    }
+    auto gpuDev = static_cast<const Device*>(&owner_->dev());
+    auto hwInfo = gpuDev->hwInfo();
+    // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1
+    setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE,
+         owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_);
+}
+
 }
 

projects/clr/rocclr/runtime/device/gpu/gpuwavelimiter.hpp

@@ -6,10 +6,12 @@
 #define GPUWAVELIMITER_HPP_
 
 #include "platform/command.hpp"
+#include "thread/thread.hpp"
 #include <cstdio>
 #include <cstdlib>
 #include <cstdint>
 #include <fstream>
+#include <unordered_map>
 
 //! \namespace gpu GPU Device Implementation
 namespace gpu {
@@ -19,11 +21,11 @@ class Kernel;
 // Adaptively limit the number of waves per SIMD based on kernel execution time
 class WaveLimiter: public amd::ProfilingCallback {
 public:
-    explicit WaveLimiter(Kernel*);
-    ~WaveLimiter();
-    uint getWavesPerSH() const;
-    amd::ProfilingCallback* getProfilingCallback() const;
-    void enable();
+    explicit WaveLimiter(Kernel*, uint seqNum, bool enable, bool enableDump);
+    virtual ~WaveLimiter();
+
+    //! Get waves per shader array to be used for kernel execution.
+    uint getWavesPerSH();
 
 private:
     enum StateKind {
@@ -32,9 +34,13 @@ private:
 
     class DataDumper {
     public:
-        explicit DataDumper(const std::string &kernelName);
+        explicit DataDumper(const std::string &kernelName, bool enable);
         ~DataDumper();
+
+        //! Record execution time, waves/simd and state of wave limiter.
         void addData(ulong time, uint wave, char state);
+
+        //! Whether this data dumper is enabled.
         bool enabled() const { return enable_;}
     private:
         bool enable_;
@@ -52,7 +58,7 @@ private:
 
     bool enable_;
     uint SIMDPerSH_;     // Number of SIMDs per SH
-    uint waves_;         // waves_ per SIMD
+    uint waves_;         // Waves per SIMD to be set
     uint bestWave_;      // Optimal waves per SIMD
     uint countAll_;      // Number of kernel executions
     uint dynRunCount_;
@@ -60,7 +66,7 @@ private:
     Kernel *owner_;
     DataDumper dumper_;
     std::ofstream traceStream_;
-    mutable bool waveSet_;
+    uint currWaves_;     // Current waves per SIMD
     uint dataCount_;
 
     static uint MaxWave;       // Maximum number of waves per SIMD
@@ -70,9 +76,16 @@ private:
     static uint AbandonThresh; // Threshold to abandon adaptation
     static uint DscThresh;     // Threshold for identifying discontinuities
 
+    //! Call back from Event::recordProfilingInfo to get execution time.
     virtual void callback(ulong duration);
+
+    //! Update measurement data and optimal waves/simd with execution time.
     void updateData(ulong time);
+
+    //! Output trace of measurement/adaptation.
     void outputTrace();
+
+    //! Clear measurement data for the next adaptation.
     void clearData();
 
     template<class T> void clear(T& A) {
@@ -88,5 +101,29 @@ private:
         }
     }
 };
+
+// Create wave limiter for each virtual device for a kernel and manages the wave limiters.
+class WaveLimiterManager {
+public:
+    explicit WaveLimiterManager(Kernel* owner);
+    virtual ~WaveLimiterManager();
+
+    //! Get waves per shader array for a specific virtual device.
+    uint getWavesPerSH(const device::VirtualDevice *) const;
+
+    //! Provide call back function for a specific virtual device.
+    amd::ProfilingCallback* getProfilingCallback(const device::VirtualDevice *);
+
+    //! Enable wave limiter manager by kernel metadata and flags.
+    void enable();
+private:
+    Kernel *owner_;                // The kernel which owns this object
+    std::unordered_map<const device::VirtualDevice *,
+        WaveLimiter*> limiters_;   // Maps virtual device to wave limiter
+    bool enable_;                  // Whether the adaptation is enabled
+    bool enableDump_;              // Whether the data dumper is enabled
+    uint fixed_;                   // The fixed waves/simd value if not zero
+    amd::Monitor monitor_;         // The mutex for updating the wave limiter map
+};
 }
 #endif

projects/clr/rocclr/runtime/platform/command.cpp

@@ -266,8 +266,8 @@ NDRangeKernelCommand::NDRangeKernelCommand(
 {
     parameters_ = kernel.parameters().capture(queue.device());
     auto& device = queue.device();
-    auto devKernel = kernel.getDeviceKernel(device);
-    profilingInfo_.setCallback(devKernel->getProfilingCallback());
+    auto devKernel = const_cast<device::Kernel*>(kernel.getDeviceKernel(device));
+    profilingInfo_.setCallback(devKernel->getProfilingCallback(queue.vdev()));
     fixme_guarantee(parameters_ != NULL && "out of memory");
     kernel_.retain();
 }