P4 to Git Change 1136853 by yaxunl@yaxunl_stg_win50 on 2015/04/01 19:36:09

ECR #304775 - Implement profile driven wave limiter Part 2/2: OCL changes. Profile driven wave limiter measures kernel execution time in real time and adaptively limits the number of waves per SH. This is to mitigate cache thrashing issues. Affected files ... ... //depot/stg/opencl/drivers/opencl/compiler/llvm32/include/llvm/Transforms/IPO/AMDKernelPerfHint.h#2 edit ... //depot/stg/opencl/drivers/opencl/compiler/llvm32/lib/Target/AMDIL/AMDILKernelManager.cpp#2 edit ... //depot/stg/opencl/drivers/opencl/compiler/llvm32/lib/Transforms/IPO/AMDKernelPerfHint.cpp#2 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#244 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.cpp#281 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpukernel.hpp#108 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuwavelimiter.cpp#1 add ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuwavelimiter.hpp#1 add ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gslbe/src/include/cal/cal.h#35 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gslbe/src/rt/GSLContext.cpp#71 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gslbe/src/rt/GSLContext.h#44 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gslbe/src/rt/GSLDevice.cpp#114 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/command.cpp#68 edit ... //depot/stg/opencl/drivers/opencl/runtime/platform/command.hpp#74 edit ... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#228 edit [ROCm/clr commit: 493d26db66]
2015-04-01 19:45:26 -04:00
@@ -71,6 +71,9 @@ namespace option {
 class Options;
 } // option

+struct ProfilingCallback: public amd::EmbeddedObject {
+    virtual void callback (ulong duration) = 0;
+};
 }

 enum OclExtensions {
@@ -872,6 +875,7 @@ public:
        size_t   compileSizeHint_[3];    //!< kernel compiled workgroup size hint
        std::string compileVecTypeHint_; //!< kernel compiled vector type hint
        bool     uniformWorkGroupSize_;  //!< uniform work group size option
+        bool     limitWave_;             //!< adaptively limit waves per SH
    };

    //! Default constructor
@@ -933,6 +937,11 @@ public:
      return workGroupInfo_.compileSizeHint_[dim];
    }

+    //! Get profiling callback object
+    virtual amd::ProfilingCallback* getProfilingCallback() const {
+        return NULL;
+    }
+
    void setVecTypeHint(const std::string& hint)
    {
        workGroupInfo_.compileVecTypeHint_ = hint;
@@ -58,6 +58,7 @@ const MetaDataConst ArgState[ArgStateTotal] =
    { "printfid:",          KernelArg::PrintfBufId,     { 0, 0, 0, 0, 0, 0, 0 } },
    { "wsh:",               KernelArg::GroupingHint,    { 0, 0, 0, 0, 0, 0, 0 } },
    { "vth:",               KernelArg::VecTypeHint,     { 0, 0, 0, 0, 0, 0, 0 } },
+    { "limitwave:",         KernelArg::LimitWave,        { 0, 0, 0, 0, 0, 0, 0 } },
 };

 const DataTypeConst DataType[] =
@@ -829,6 +830,10 @@ Kernel::create(
        }
    }

+    // Wave limiter needs to be initialized after kernel metadata is parsed
+    // Since it depends on it.
+    waveLimiter_.enable();
+
    if (result) {
        buildError_ = CL_SUCCESS;
    }
@@ -846,6 +851,7 @@ Kernel::Kernel(
    const InitData*     initData)
    : NullKernel(name, gpuDev, prog)
    , blitKernelHack_(false)
+    , waveLimiter_(this)
 {
    hwPrivateSize_ = 0;
    if (NULL != initData) {
@@ -1671,6 +1677,7 @@ Kernel::run(VirtualGPU& gpu, GpuEvent* calEvent, bool lastRun) const
        }
    }

+    gpu.setWavesPerSH(gpu.gslKernelDesc()->func_, waveLimiter_.getWavesPerSH());
    if (!gpu.runProgramGrid(*calEvent,
        const_cast<ProgramGrid*>(&gpu.cal()->progGrid_), gpu.vmMems(), gpu.cal_.memCount_)) {
        LogError("Failed to execute the program!");
@@ -2711,6 +2718,15 @@ NullKernel::parseArguments(const std::string& metaData, uint* uavRefCount)
                    }
                    // Process next ...
                    continue;
+                case KernelArg::LimitWave:
+                    {
+                        uint tmp;
+                        if (!getuint(metaData, &pos, &tmp)) {
+                            return false;
+                        }
+                        workGroupInfo_.limitWave_ = tmp!=0;
+                    }
+                    continue;
                default:
                    break;
                }
@@ -15,6 +15,7 @@
 #include "device/gpu/gpuvirtual.hpp"
 #include "sc/Interface/SCHSAInterface.h"
 #include "device/gpu/gpuprintf.hpp"
+#include "device/gpu/gpuwavelimiter.hpp"
 #include "hsa.h"
 //! \namespace gpu GPU Device Implementation
 namespace gpu {
@@ -187,6 +188,7 @@ public:
        PrintfBufId,
        GroupingHint,
        VecTypeHint,
+        LimitWave,
        TotalTypes
    };

@@ -312,7 +314,7 @@ struct MetaDataConst
 };

 const uint DescTotal        = 15;
-const uint BasicTypeTotal   = 14;
+const uint BasicTypeTotal   = 15;
 const uint ArgStateTotal    = DescTotal + BasicTypeTotal;

 //! The constant array that describes different metadata properties
@@ -651,6 +653,11 @@ public:
        VirtualGPU::GslKernelDesc*  desc    //!< Kernel descriptor
        ) const;

+    //! Get profiling callback object
+    virtual amd::ProfilingCallback* getProfilingCallback() const {
+        return waveLimiter_.getProfilingCallback();
+    }
+
 protected:
    //! Initializes the kernel parameters for the abstraction layer
    bool initParameters();
@@ -767,6 +774,8 @@ private:

    //! @todo remove the blit kernel hack
    bool    blitKernelHack_;    //!< No VM hack for kernel blit
+
+    WaveLimiter waveLimiter_; //!< adaptively control number of waves
 };

 enum HSAIL_ADDRESS_QUALIFIER{
@@ -0,0 +1,190 @@
+//
+// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved.
+//
+
+#include "device/gpu/gpukernel.hpp"
+#include "device/gpu/gpuwavelimiter.hpp"
+#include "os/os.hpp"
+#include "utils/flags.hpp"
+
+namespace gpu {
+
+uint WaveLimiter::MaxWave;
+uint WaveLimiter::WarmUpCount;
+uint WaveLimiter::AdaptCount;
+uint WaveLimiter::RunCount;
+uint WaveLimiter::AbandonThresh;
+
+void WaveLimiter::clearData() {
+    waves_ = MaxWave;
+    countAll_ = 0;
+    clear(counts_);
+    clear(sum_);
+    clear(average_);
+    clear(ratio_);
+}
+
+void WaveLimiter::enable() {
+    if (waves_ > 0) {
+        return;
+    }
+    auto gpuDev = reinterpret_cast<const Device*>(&owner_->dev());
+    auto hwInfo = gpuDev->hwInfo();
+    // Enable it only for SI+, unless GPU_WAVE_LIMIT_ENABLE is set to 1
+    setIfNotDefault(enable_, GPU_WAVE_LIMIT_ENABLE,
+         owner_->workGroupInfo()->limitWave_ && gpuDev->settings().siPlus_);
+    if (!enable_) {
+        return;
+    }
+    waves_ = MaxWave;
+}
+
+WaveLimiter::WaveLimiter(Kernel *owner) :
+        owner_(owner), dumper_(owner_->name()) {
+    auto gpuDev = reinterpret_cast<const Device*>(&owner_->dev());
+    auto attrib = gpuDev->getAttribs();
+    auto hwInfo = gpuDev->hwInfo();
+    setIfNotDefault(SIMDPerSH_, GPU_WAVE_LIMIT_CU_PER_SH,
+            attrib.numberOfCUsperShaderArray * hwInfo->simdPerCU_);
+
+    state_ = WARMUP;
+    dynRunCount_ = RunCount;
+    auto size = MaxWave + 1;
+    counts_.resize(size);
+    sum_.resize(size);
+    average_.resize(size);
+    ratio_.resize(size);
+    clearData();
+    if (!flagIsDefault(GPU_WAVE_LIMIT_TRACE)) {
+        traceStream_.open(std::string(GPU_WAVE_LIMIT_TRACE) + owner_->name() +
+            ".txt");
+    }
+
+    MaxWave = GPU_WAVE_LIMIT_MAX_WAVE;
+    WarmUpCount = GPU_WAVE_LIMIT_WARMUP;
+    AdaptCount = GPU_WAVE_LIMIT_ADAPT * MaxWave;
+    RunCount = GPU_WAVE_LIMIT_RUN * MaxWave;
+    AbandonThresh = GPU_WAVE_LIMIT_ABANDON;
+
+    waves_ = GPU_WAVES_PER_SIMD;
+    bestWave_ = MaxWave;
+    enable_ = false;
+}
+
+WaveLimiter::~WaveLimiter() {
+    if (traceStream_.is_open()) {
+        traceStream_.close();
+    }
+}
+
+uint WaveLimiter::getWavesPerSH() const {
+    return waves_ * SIMDPerSH_;
+}
+
+void WaveLimiter::updateData(ulong time) {
+    sum_[waves_] += time;
+    counts_[waves_]++;
+    average_[waves_] = sum_[waves_] / counts_[waves_];
+    ratio_[waves_] = average_[waves_] * 100 / average_[MaxWave];
+    if (average_[bestWave_] > average_[waves_]) {
+        bestWave_ = waves_;
+    }
+    outputTrace();
+}
+
+void WaveLimiter::outputTrace() {
+    if (!traceStream_.is_open()) {
+        return;
+    }
+
+    traceStream_ << "[WaveLimiter] " << owner_->name() << " state=" << state_
+            << " waves=" << waves_ << " bestWave=" << bestWave_ << '\n';
+    output(traceStream_, "\n counts = ", counts_);
+    output(traceStream_, "\n sum = ", sum_);
+    output(traceStream_, "\n average = ", average_);
+    output(traceStream_, "\n ratio = ", ratio_);
+    traceStream_ << "\n\n";
+}
+
+void WaveLimiter::callback(ulong duration) {
+    dumper_.addData(duration, waves_, static_cast<char>(state_));
+
+    if (!enable_) {
+        return;
+    }
+
+    countAll_++;
+
+    switch (state_) {
+    case WARMUP:
+        if (countAll_ < WarmUpCount) {
+            return;
+        }
+        state_ = ADAPT;
+        bestWave_ = MaxWave;
+        clearData();
+        return;
+    case ADAPT:
+        updateData(duration);
+        if (countAll_ < AdaptCount && ratio_[waves_] < AbandonThresh) {
+            waves_ = MaxWave - (countAll_ % MaxWave);
+            return;
+        }
+        waves_ = bestWave_;
+        if (countAll_ >= AdaptCount) {
+            dynRunCount_ = RunCount;
+        } else {
+            dynRunCount_ = AdaptCount;
+        }
+        countAll_ = rand() % MaxWave;
+        state_ = RUN;
+        return;
+    case RUN:
+        if (countAll_ < dynRunCount_) {
+            return;
+        }
+        state_ = ADAPT;
+        bestWave_ = MaxWave;
+        clearData();
+        return;
+    }
+}
+
+WaveLimiter::DataDumper::DataDumper(const std::string &kernelName) {
+    enable_ = !flagIsDefault(GPU_WAVE_LIMIT_DUMP);
+    if (enable_) {
+        fileName_ = std::string(GPU_WAVE_LIMIT_DUMP) + kernelName + ".csv";
+    }
+}
+
+WaveLimiter::DataDumper::~DataDumper() {
+    if (!enable_) {
+        return;
+    }
+
+    std::ofstream OFS(fileName_);
+    for (size_t i = 0, e = time_.size(); i != e; ++i) {
+        OFS << i << ',' << time_[i] << ',' << wavePerSIMD_[i] << ','
+            << static_cast<uint>(state_[i]) << '\n';
+    }
+    OFS.close();
+}
+
+void WaveLimiter::DataDumper::addData(ulong time, uint wave, char state) {
+    if (!enable_) {
+        return;
+    }
+
+    time_.push_back(time);
+    wavePerSIMD_.push_back(wave);
+    state_.push_back(state);
+}
+
+amd::ProfilingCallback* WaveLimiter::getProfilingCallback() const {
+    if (enable_ || dumper_.enabled()) {
+        return const_cast<WaveLimiter*>(this);
+    }
+    return NULL;
+}
+}
+
@@ -0,0 +1,88 @@
+//
+// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved.
+//
+
+#ifndef GPUWAVELIMITER_HPP_
+#define GPUWAVELIMITER_HPP_
+
+#include "platform/command.hpp"
+#include <cstdio>
+#include <cstdlib>
+#include <cstdint>
+#include <fstream>
+
+//! \namespace gpu GPU Device Implementation
+namespace gpu {
+
+class Kernel;
+
+// Adaptively limit the number of waves per SIMD based on kernel execution time
+class WaveLimiter: public amd::ProfilingCallback {
+public:
+    explicit WaveLimiter(Kernel*);
+    ~WaveLimiter();
+    uint getWavesPerSH() const;
+    amd::ProfilingCallback* getProfilingCallback() const;
+    void enable();
+
+private:
+    enum StateKind {
+        WARMUP, ADAPT, RUN
+    };
+
+    class DataDumper {
+    public:
+        explicit DataDumper(const std::string &kernelName);
+        ~DataDumper();
+        void addData(ulong time, uint wave, char state);
+        bool enabled() const { return enable_;}
+    private:
+        bool enable_;
+        std::string fileName_;
+        std::vector<ulong> time_;
+        std::vector<uint> wavePerSIMD_;
+        std::vector<char> state_;
+    };
+
+    std::vector<ulong> counts_;
+    std::vector<ulong> sum_;
+    std::vector<ulong> average_;
+    std::vector<ulong> ratio_;
+
+    bool enable_;
+    uint SIMDPerSH_;     // Number of SIMDs per SH
+    uint waves_;         // waves_ per SIMD
+    uint bestWave_;      // Optimal waves per SIMD
+    uint countAll_;      // Number of kernel executions
+    uint dynRunCount_;
+    StateKind state_;
+    Kernel *owner_;
+    DataDumper dumper_;
+    std::ofstream traceStream_;
+
+    static uint MaxWave;       // Maximum number of waves per SIMD
+    static uint WarmUpCount;   // Number of kernel executions for warm up
+    static uint AdaptCount;    // Number of kernel executions for adapting
+    static uint RunCount;      // Number of kernel executions for normal run
+    static uint AbandonThresh; // Threshold to abandon adaptation
+
+    virtual void callback(ulong duration);
+    void updateData(ulong time);
+    void outputTrace();
+    void clearData();
+
+    template<class T> void clear(T& A) {
+        for (auto &I : A) {
+            I = 0;
+        }
+    }
+    template<class T> void output(std::ofstream &ofs, const std::string &prompt,
+            T& A) {
+        ofs << prompt;
+        for (auto &I : A) {
+            ofs << ' ' << static_cast<ulong>(I);
+        }
+    }
+};
+}
+#endif
@@ -207,6 +207,7 @@ typedef struct CALdeviceattribsRec {
    CALuint    memoryClock;                         /**< GPU memory clock rate in megahertz */
    CALuint    wavefrontSize;                       /**< Wavefront size */
    CALuint    numberOfSIMD;                        /**< Number of SIMDs */
+    CALuint    numberOfCUsperShaderArray;           /**< Number of CUs per shader array */
    bool       doublePrecision;                     /**< double precision supported */
    bool       localDataShare;                      /**< local data share supported */
    bool       globalDataShare;                     /**< global data share supported */
@@ -5,6 +5,7 @@
 #include "cm_if.h"
 #include "amuABI.h"
 #include "shader/ProgramObject.h"
+#include "shader/ComputeProgramObject.h"
 #include "query/QueryObject.h"
 #include "query/PerformanceQueryObject.h"
 #include "constbuffer/ConstantBufferObject.h"
@@ -400,6 +401,13 @@ CALGSLContext::setProgram(gslProgramObject func)
    m_rs->setCurrentProgramObject(GSL_COMPUTE_PROGRAM, func);
 }

+void
+CALGSLContext::setWavesPerSH(gslProgramObject func, uint32 wavesPerSH)const
+{
+    auto compProg = static_cast<gsl::ComputeProgramObject*>(func);
+    compProg->setWavesPerSH(wavesPerSH);
+}
+
 bool
 CALGSLContext::runProgramGrid(GpuEvent& event, const ProgramGrid* pProgramGrid, const gslMemObject* mems, uint32 numMems)
 {
@@ -41,6 +41,7 @@ public:
    void             setUavMask(const CALUavMask& uavMask);
    void             setUAVChannelOrder(uint32 physUnit, gslMemObject mem);
    void             setProgram(gslProgramObject func);
+    void             setWavesPerSH(gslProgramObject func, uint32 wavesPerSH)const;
    bool             runProgramGrid(GpuEvent& event, const ProgramGrid* pProgramGrid, const gslMemObject* mems, uint32 numMems);
    bool             runProgramVideoDecode(GpuEvent& event, gslMemObject mo, const CALprogramVideoDecode& decode);
    void             runAqlDispatch(GpuEvent& event, const void* aqlPacket, const gslMemObject* mems,
@@ -116,6 +116,7 @@ CALGSLDevice::getAttribs_int(gsl::gsCtx* cs)
    m_attribs.engineClock = cs->getMaxEngineClock();
    m_attribs.memoryClock = cs->getMaxMemoryClock();
    m_attribs.numberOfSIMD = cs->getNumSIMD();
+    m_attribs.numberOfCUsperShaderArray = cs->getNumCUsPerShaderArray();
    m_attribs.wavefrontSize = cs->getWaveFrontSize();
    m_attribs.doublePrecision = cs->getIsDoublePrecisionSupported();
    m_attribs.localDataShare = cs->getIsLocalDataShareSupported();
@@ -68,6 +68,9 @@ Event::recordProfilingInfo(cl_int status, uint64_t timeStamp)
        break;
    default:
        profilingInfo_.end_ = timeStamp;
+        if (profilingInfo_.callback_ != NULL) {
+            profilingInfo_.callback_->callback(timeStamp - profilingInfo_.start_);
+        }
        break;
    }
    return timeStamp;
@@ -259,6 +262,9 @@ NDRangeKernelCommand::NDRangeKernelCommand(
        kernel_(kernel), sizes_(sizes)
 {
    parameters_ = kernel.parameters().capture(queue.device());
+    auto& device = queue.device();
+    auto devKernel = kernel.getDeviceKernel(device);
+    profilingInfo_.setCallback(devKernel->getProfilingCallback());
    fixme_guarantee(parameters_ != NULL && "out of memory");
    kernel_.retain();
 }
@@ -89,10 +89,8 @@ protected:
        ProfilingInfo(bool enabled = false) : enabled_(enabled)
        {
            if (enabled) {
-                queued_    = 0ULL;
-                submitted_ = 0ULL;
-                start_     = 0ULL;
-                end_       = 0ULL;
+                clear();
+                callback_  = NULL;
            }
        }

@@ -100,7 +98,22 @@ protected:
        uint64_t   submitted_;
        uint64_t   start_;
        uint64_t   end_;
-        const bool enabled_;
+        bool       enabled_;
+        ProfilingCallback *callback_;
+        void clear() {
+            queued_    = 0ULL;
+            submitted_ = 0ULL;
+            start_     = 0ULL;
+            end_       = 0ULL;
+        }
+        void setCallback(ProfilingCallback *callback) {
+            if (callback == NULL) {
+                return;
+            }
+            enabled_ = true;
+            clear();
+            callback_ = callback;
+        }

    } profilingInfo_;

@@ -177,7 +177,27 @@ debug(bool, GPU_FORCE_SINGLE_FP_DENORM, false,                                \
 debug(bool, OCL_FORCE_CPU_SVM, false, \
        "force svm support for CPU")                                          \
 debug(bool, GPU_ENABLE_HW_DEBUG, false,                                       \
-        "Enable HW DEBUG for GPU")
+        "Enable HW DEBUG for GPU")                                            \
+release(uint, GPU_WAVES_PER_SIMD, 0,                                          \
+        "Force the number of waves per SIMD (1-10)")                          \
+release(bool, GPU_WAVE_LIMIT_ENABLE, false,                                   \
+        "1 = Enable adaptive wave limiter")                                   \
+release_on_stg(uint, GPU_WAVE_LIMIT_CU_PER_SH, 0,                             \
+        "Assume the number of CU per SH for wave limiter")                    \
+release_on_stg(uint, GPU_WAVE_LIMIT_MAX_WAVE, 10,                             \
+        "Set maximum waves per SIMD to try for wave limiter")                 \
+release_on_stg(uint, GPU_WAVE_LIMIT_WARMUP, 100,                              \
+        "Set warming up kernel execution count for wave limiter")             \
+release_on_stg(uint, GPU_WAVE_LIMIT_ADAPT, 1,                                 \
+        "Set adapting factor for wave limiter")                               \
+release_on_stg(uint, GPU_WAVE_LIMIT_RUN, 40,                                  \
+        "Set running factor for wave limiter")                                \
+release_on_stg(uint, GPU_WAVE_LIMIT_ABANDON, 105,                             \
+        "Set abandon threshold for wave limiter")                             \
+release_on_stg(cstring, GPU_WAVE_LIMIT_DUMP, "",                              \
+        "File path prefix for dumping wave limiter output")                   \
+release_on_stg(cstring, GPU_WAVE_LIMIT_TRACE, "",                             \
+        "File path prefix for tracing wave limiter")



@@ -243,6 +263,12 @@ public:
 #define flagIsDefault(name) \
    (amd::Flag::cannotSet##name || amd::Flag::isDefault(amd::Flag::k##name))

+#define setIfNotDefault(var, opt, other) \
+  if (!flagIsDefault(opt)) \
+    var = (opt);\
+  else \
+    var = (other);
+
 //  @}

 } // namespace amd