rocm-systems/rocclr/runtime/device/cpu/cpucommand.cpp

//
// Copyright 2010 Advanced Micro Devices, Inc. All rights reserved.
//

#include "device/cpu/cpucommand.hpp"
#include "device/cpu/cpubuiltins.hpp"
#include "device/cpu/cpudevice.hpp"
#include "device/cpu/cputables.hpp"
#include "platform/command.hpp"
#include "platform/commandqueue.hpp"
#include "platform/program.hpp"
#include "platform/kernel.hpp"
#include "platform/sampler.hpp"
#include "thread/thread.hpp"
#include "os/os.hpp"
#include "utils/util.hpp"
#include "utils/options.hpp"

#include <amdocl/cl_kernel.h>
#include <algorithm>

namespace cpu {

#define CPU_WORKER_THREAD_TOTAL_STACK_SIZE (CPU_WORKER_THREAD_STACK_SIZE + \
    CLK_PRIVATE_MEMORY_SIZE * (CPU_MAX_WORKGROUP_SIZE + 1))

WorkerThread::WorkerThread(const cpu::Device& device) :
    Thread("CPU Worker Thread", CPU_WORKER_THREAD_TOTAL_STACK_SIZE),
    queueLock_("WorkerThread::queueLock"), waitingOp_(0), terminated_(false)
{
    localDataSize_ = (size_t) device.info().localMemSize_;
    localDataStorage_ = (address) amd::AlignedMemory::allocate(
        localDataSize_ + __CPU_SCRATCH_SIZE, sizeof(cl_long16));

#if defined(__linux__) && defined(NUMA_SUPPORT)
    const nodemask_t* numaMask = device.getNumaMask();
    if (numaMask != NULL) {
        numa_bind(numaMask);
    }
#endif
}

WorkerThread::~WorkerThread()
{
    guarantee(Thread::current() != this && "thread suicide!");
    amd::AlignedMemory::deallocate(localDataStorage_);
}

bool
WorkerThread::terminate()
{
    terminated_ = true;

    if (Thread::current() != this) {
        // FIXME_lmoriche: fix termination handshake
        while (state() < Thread::FINISHED) {
            flush();
            amd::Os::yield();
        }
    }

    return true;
}

void 
WorkerThread::enqueue(Operation& op)
{
    while (waitingOp_ != 0) {
        amd::Os::yield();
    }
    op.clone(operation());
    ++waitingOp_;
}

void
WorkerThread::loop()
{
    baseWorkItemsStack_ = amd::alignDown(stackBase() -
        CPU_WORKER_THREAD_STACK_SIZE, CLK_PRIVATE_MEMORY_SIZE);
#if defined(WIN32)
    amd::Os::touchStackPages(baseWorkItemsStack_, amd::Os::currentStackPtr());
#endif // WINDOWS
    Operation *op = operation();

    queueLock_.lock();
    while (true) {
        while (waitingOp_ == 0) {
            if (terminated_) {
                break;
            }
            queueLock_.wait();
        }
        if (terminated_) {
            break;
        }
        op->command().setStatus(CL_RUNNING);
        op->execute();
        op->cleanup();
        --waitingOp_;
    }
    queueLock_.unlock();
}

void
NativeFn::execute()
{
    cl_int status = static_cast<amd::NativeFnCommand&>(command()).invoke();
    command().setStatus(status);
}

static void
nop() { /*Do nothing*/ }


template <NDRangeKernelBatch::ExecutionNature NATURE,
          NDRangeKernelBatch::ExecutionOrder ORDER =
              NDRangeKernelBatch::ORDER_DEFAULT>
class NDRangeKernelBatchMode : public NDRangeKernelBatch
{
private:
    void executeWorkGroup(WorkGroup& wg)
    {
        if (NATURE == NATURE_WG_LEVEL_EXEC) {
            wg.executeWorkItem();
        }
        else if ((NATURE == NATURE_1_WORK_ITEM) ||
                 (wg.getNumWorkItems() == 1)) {
            wg.executeWorkItem();
        }
        else {
            wg.getBaseWorkItem()->setNext(&wg.getWorkerThread().mainFiber());
            if (NATURE == NATURE_WITHOUT_BARRIER) {
                wg.executeWithoutBarrier();
            }
            else { // NATURE == NATURE_WITH_BARRIER
                wg.executeWithBarrier();
            }
        }
        // Yield at the end of each workgroup to avoid starving GPU device
        amd::Os::yield();
    }

public:
    void executeMode(WorkGroup& wg)
    {
        const amd::NDRange& offset =
            static_cast<amd::NDRangeKernelCommand&>(command_).sizes().offset();
        WorkItem* workItem0 = wg.getBaseWorkItem();
        clk_builtins_t tableTask;
        size_t prevOpId = 0, opId = (size_t)-1;

        if (NATURE == NATURE_1_WORK_ITEM) {
            tableTask = Builtins::dispatchTable_;

            // If local size == 1 then barrier() becomes a nop.
            tableTask.barrier_ptr = (void (*)(cl_mem_fence_flags)) nop;
            workItem0->infoBlock().builtins = &tableTask;
            workItem0->setNext(&wg.getWorkerThread().mainFiber());
        }

        while (getNextOperationId(opId)) {
            workItem0->incrementGroupId(groupIds_, offset, opId - prevOpId);
            uint workDims = workItem0->infoBlock().work_dim;
            size_t numWorkItems = workItem0->infoBlock().local_size[0] *
              (workDims >= 2 ? workItem0->infoBlock().local_size[1] : 1) *
              (workDims >= 3 ? workItem0->infoBlock().local_size[2] : 1);
            wg.setNumWorkItems(numWorkItems);
            if(numWorkItems == 1) {
              tableTask = Builtins::dispatchTable_;
              tableTask.barrier_ptr = (void (*)(cl_mem_fence_flags)) nop;
              workItem0->infoBlock().builtins = &tableTask;
              workItem0->setNext(&wg.getWorkerThread().mainFiber());
              executeWorkGroup(wg);
              tableTask.barrier_ptr = &WorkItem::barrier;
            } else {
              executeWorkGroup(wg);
            }
            prevOpId = opId;
        }

//#define DISABLE_TASK_STEALING
#if !defined(DISABLE_TASK_STEALING) && 0
        size_t maxId = numCores_;
        size_t stolenId = coreId_ + 1;
        NDRangeKernelBatch* workingBatch = this;
        size_t numStolenIds = 1;
        const size_t maxStealingSize = 3;
        const size_t minAdaptiveStealingDiff = numCores_ * maxStealingSize;
    
        while (true) {
            for (; stolenId < maxId; ++stolenId) {
                WorkerThread* worker = virtualDevice_.getWorkerThread(stolenId);

                // In case were we have less operations than Worker Threads
                if (worker->isOperationValid()) {
                    workingBatch = static_cast<NDRangeKernelBatch*>(
                        worker->operation());

                    numStolenIds =
                        workingBatch->getNextOperationIds(opId, numStolenIds);
                    if (numStolenIds > 0) {
                        do {
                            for (size_t i = 0; i < numStolenIds; ++i) {
                                workItem0->setGroupId(groupIds_, offset, opId);
                                executeWorkGroup(wg);
                                opId += numCores_;
                            }

                            // adaptive stealing
                            if (numWorkGroups_ - opId > minAdaptiveStealingDiff) {
                                numStolenIds = maxStealingSize;
                            }
                            else {
                                while (workingBatch->getNextOperationId(opId)) {
                                    workItem0->setGroupId(groupIds_, offset, opId);
                                    executeWorkGroup(wg);
                                }
                                break;
                            }
                            numStolenIds = workingBatch->getNextOperationIds(
                                opId, numStolenIds);
                        } while (numStolenIds > 0);
                    }
                    numStolenIds = 1;
                }
            } // for (stolenId..maxId)

            if (stolenId == coreId_) {
                break;
            }

            stolenId = 0;
            maxId = coreId_;
        } // while (true)
#endif
    }
};


inline bool
NDRangeKernelBatch::getNextOperationId(size_t& opId)
{
    if (currentOpId_ >= numWorkGroups_) {
        return false;
    }
    opId = amd::AtomicOperation::add(numCores_, &currentOpId_);
    return opId < numWorkGroups_;
}


inline size_t
NDRangeKernelBatch::getNextOperationIds(size_t& opId, size_t count)
{
    size_t topId = numCores_ * count;
    if (currentOpId_ >= numWorkGroups_) {
        return 0;
    }

    opId = amd::AtomicOperation::add(topId, &currentOpId_);
    const size_t numWorkGroups = numWorkGroups_;
    if (opId >= numWorkGroups) {
        return 0;
    }

    topId += opId;
    if (topId >= (numWorkGroups + numCores_)) {
        count -= (topId - numWorkGroups) / numCores_;
    }

    return count;
}

// Process the parameters, allocate LDS.
bool
NDRangeKernelBatch::patchParameters(
    const cpu::Kernel& cpuKernel,
    address params,
    address& localMemPtr,
    const address localMemLimit,
    size_t localMemSize) const
{
    amd::NDRangeKernelCommand& command =
        static_cast<amd::NDRangeKernelCommand&>(command_);

    const amd::Device& device = command.queue()->device();

    const amd::Kernel& kernel = command.kernel();
    const amd::KernelSignature& signature = kernel.signature();
    const amd::KernelParameters& kernelParam = kernel.parameters();

    const_address cmdParams = command.parameters();

    unsigned effectiveOffset = 0;

    // DD -- on CPU device, real effective offset is NATIVELY aligned
    // Here all source arguments are in place, so we're safe just iterating
    for (size_t i = 0; i < signature.numParameters(); ++i) {
        const amd::KernelParameterDescriptor& desc = signature.at(i);
        const void* cmdParam = cmdParams + desc.offset_;
        void    *param;
        size_t   prmSize = cpuKernel.getArgSize(i);

        // Align i'th parameter on multiple of its size. Parameter size is power of 2.
        size_t alignment = cpuKernel.getArgAlignment(i);
        effectiveOffset = amd::alignUp(effectiveOffset, std::min(alignment, size_t(16)));
        param = params + effectiveOffset;
        if (desc.size_ == 0) {
            // __local memory parameter
            localMemPtr = amd::alignUp(localMemPtr, sizeof(cl_long16));

            size_t length = *static_cast<const size_t*>(cmdParam);
            *static_cast<void**>(param) = localMemPtr;
            localMemPtr += length;

            if (localMemPtr > localMemLimit) {
                command.setException(CL_MEM_OBJECT_ALLOCATION_FAILURE);
                return false;
            }
        }
        else if (desc.type_ == T_POINTER) {
            // __global memory parameter
            cl_mem_object_type pointer_type = CL_MEM_OBJECT_BUFFER;
            if (kernelParam.boundToSvmPointer(device, cmdParams, i)) {
                *reinterpret_cast<void**>(param) =
                        *reinterpret_cast<void* const *>(cmdParam);
            }
            else {
                void* hostMemPtr = NULL;
                amd::Memory* memArg =
                    *reinterpret_cast<amd::Memory* const *>(cmdParam);
                if (memArg != NULL) {
                    hostMemPtr = memArg->getHostMem();
                    if (hostMemPtr == NULL) {
                        command.setException(CL_MEM_OBJECT_ALLOCATION_FAILURE);
                        return false;
                    }
                    pointer_type = memArg->getType();
                }
                // For images on CPU devices, pass "struct {int4 p0; int4 p1}".
                // That allows an obvious implementation for
                // __amdil_get_image[23]d_params[01].
                // That makes the rest of the .bc implementation for
                // images relatively straight forward.
                if (pointer_type == CL_MEM_OBJECT_IMAGE1D ||
                    pointer_type == CL_MEM_OBJECT_IMAGE2D ||
                    pointer_type == CL_MEM_OBJECT_IMAGE3D ||
                    pointer_type == CL_MEM_OBJECT_IMAGE1D_ARRAY  ||
                    pointer_type == CL_MEM_OBJECT_IMAGE1D_BUFFER ||
                    pointer_type == CL_MEM_OBJECT_IMAGE2D_ARRAY) {
                    amd::Image::Impl& impl = memArg->asImage()->getImpl();
                    impl.reserved_ = hostMemPtr;
                    *reinterpret_cast<void**>(param) = (void*)&impl;
                } else {
                    *reinterpret_cast<void**>(param) = hostMemPtr;
                }
            }
        }
        else if (desc.type_ == T_SAMPLER) {
            // Switch from an Amd::Sampler to the 32bit integer
            // variable that is a clk_sampler.
            amd::Sampler* samplerArg =
                *reinterpret_cast<amd::Sampler* const *>(cmdParam);
            *reinterpret_cast<uint32_t*>(param) = (uint32_t)samplerArg->state();
        }
        else {
            //Using HCtoDCmap
            HCtoDCmap arg_map = cpuKernel.getHCtoDCmap(i);
            unsigned int arg_offset = effectiveOffset;
            int err_code = 0;
            int inStruct = 0;
            int sys_64bit = LP64_SWITCH(0, 1);  // Mapping only required for 32 bit targets
            if (CPU_USE_ALIGNMENT_MAP == 0 && !sys_64bit) {
               effectiveOffset += arg_map.copy_params(param, cmdParam, arg_offset, err_code, inStruct);
               if (err_code) {
                   return false;
               }
               prmSize = arg_map.dc_size;
            }
            else {
                ::memcpy(param, cmdParam, desc.size_);
            }
        }
        effectiveOffset += prmSize;
    }

    localMemPtr = amd::alignUp(localMemPtr, sizeof(cl_long16));
    if ((localMemPtr + localMemSize) > localMemLimit) {
        command.setException(CL_MEM_OBJECT_ALLOCATION_FAILURE);
        return false;
    }

    return true;
}

void
NDRangeKernelBatch::execute()
{
    amd::NDRangeKernelCommand& command =
        static_cast<amd::NDRangeKernelCommand&>(command_);

    const cpu::Kernel& kernel = static_cast<const cpu::Kernel&>(
                  *command.kernel().getDeviceKernel(command.queue()->device()));

    WorkerThread& thread = *WorkerThread::current();

    const size_t numWorkItems = command.sizes().local().product();

    address params = thread.baseWorkItemsStack();
    address baseLocalMemPtr = thread.localDataStorage();
    address patchedLocalMemPtr = thread.localDataStorage() + __CPU_SCRATCH_SIZE;
    if (!patchParameters(kernel, params,
        patchedLocalMemPtr, patchedLocalMemPtr + thread.localDataSize(),
        kernel.workGroupInfo()->localMemSize_)) {
        return;
    }

    WorkItem* workItem0 = ::new((WorkItem*)params - 1) WorkItem(
        command.sizes(), baseLocalMemPtr, patchedLocalMemPtr);

    WorkGroup wg(command, kernel, thread, params, workItem0, numWorkItems);

    if (numWorkItems == 1) {
        static_cast<NDRangeKernelBatchMode<NATURE_1_WORK_ITEM>*>(this)->
            executeMode(wg);
    }
    else if (kernel.hasBarrier()) {
        static_cast<NDRangeKernelBatchMode<NATURE_WITH_BARRIER>*>(this)->
            executeMode(wg);
    }
    else {
        static_cast<NDRangeKernelBatchMode<NATURE_WITHOUT_BARRIER>*>(this)->
            executeMode(wg);
    }
}

void
WorkGroup::executeWorkItem()
{
    callKernel((kernelentrypoint_t)kernel_.getEntryPoint(), workItem0_->nativeStackPtr());
}

void
WorkGroup::executeWithBarrier()
{
    kernelentrypoint_t entryPoint = (kernelentrypoint_t)kernel_.getEntryPoint();

    workingFiber_ = workItem0_;
    address workGroupStackPtr = workItem0_->nativeStackPtr();

    // Save the current stack context in case we execute a barrier.
    volatile size_t threadCounter = 0;
    bool barrier = !thread_.mainFiber().save();

    size_t tid = threadCounter++;
    WorkItem* workItem = (WorkItem*)((char*) workItem0_
        - tid * CLK_PRIVATE_MEMORY_SIZE);

    if (barrier) {
        WorkItem* prev = (WorkItem*)((char*) workItem
            + CLK_PRIVATE_MEMORY_SIZE);

        WINDOWS_ONLY(amd::Os::touchStackPages(
            (address) (workItem + 1), (address) prev));
        ::memcpy(workItem, prev, sizeof(WorkItem));

        clk_thread_info_block_t& tib = workItem->infoBlock();
        ++tib.local_id[0];
        if (unlikely(tib.local_id[0] >= tib.local_size[0])) {
            //
            // Compiling for Windows 64bit (only in release) introduces a bug,
            // which uses the same register for saving threadCounter and the
            // 0 value. Therefore "tib.local_id[i] = 0" was actually translated
            // to "tib.local_id[0] = threadCounter". To avoid this issue, and
            // still be able to store a 0 into tib.local_id[i], we trick the
            // compiler, by using the value in tib.local_id[3], which is always
            // initialized to 0.
            //
            tib.local_id[0] = tib.local_id[3];

            ++tib.local_id[1];
            if (unlikely(tib.local_id[1] >= tib.local_size[1])) {
                tib.local_id[1] = tib.local_id[3];

                ++tib.local_id[2];
            }
        }

        // Link the previous workitem to this one.
        prev->setNext(workItem);
        // If this is the last workitem, complete the ring.
        if (tid >= numWorkItems_ - 1) {
            workItem->setNext(workItem0_);
        }
    }

    // Execute thread0

    address workItemStackPtr = workItem->nativeStackPtr();
    callKernelProtectedReturn(entryPoint, workItemStackPtr);

    // Check if thread0 executed a barrier()
    if (threadCounter > 1) {
        workItem = (WorkItem*)workingFiber_;
        workingFiber_ = workingFiber_->next();

        tid = ((address)workItem0_ - (address)workItem)
            / CLK_PRIVATE_MEMORY_SIZE;
        if (tid == (numWorkItems_ - 1)) {
            // If we get here, we are done!
            return;
        }
        if (workItem->next() == &thread_.mainFiber()) {
            // Detected a deadlock
            command_.setException(CL_INVALID_KERNEL);
            return;
        }

        // Schedule the next workitem.
        workItem->next()->restore();
        ShouldNotReachHere();
    }

    // Execute thread1...threadN
    callKernelRange(entryPoint, workItemStackPtr, workItem->infoBlock());
}

void
WorkGroup::executeWithoutBarrier()
{
    kernelentrypoint_t entryPoint = (kernelentrypoint_t)kernel_.getEntryPoint();
    address workItemStackPtr = workItem0_->nativeStackPtr();

    // Execute thread0
    callKernel(entryPoint, workItemStackPtr);

    // Execute thread1...threadN
    callKernelRange(entryPoint, workItemStackPtr, workItem0_->infoBlock());
}

void
WorkGroup::callKernelRange(kernelentrypoint_t entryPoint,
                           address stackPtr,
                           clk_thread_info_block_t& tib)
{
    while (true) {
        ++tib.local_id[0];
        if (unlikely(tib.local_id[0] >= tib.local_size[0])) {
            tib.local_id[0] = 0;

            ++tib.local_id[1];
            if (unlikely(tib.local_id[1] >= tib.local_size[1])) {
                tib.local_id[1] = 0;

                ++tib.local_id[2];
                if (unlikely(tib.local_id[2] >= tib.local_size[2])) {
                    tib.local_id[2] = 0;

                    return;
                }
            }
        }

        callKernel(entryPoint, stackPtr);
    }
}

WorkItem::WorkItem(const amd::NDRangeContainer& sizes,
                   void* scratchMemPtr,
                   void* localMemPtr)
{
    const amd::NDRange& local = sizes.local();
    const amd::NDRange& global = sizes.global();
    const amd::NDRange& offset = sizes.offset();
    const size_t dims = sizes.dimensions();

    tib_.builtins = &Builtins::dispatchTable_;
    tib_.local_mem_base = localMemPtr;
    tib_.local_scratch = scratchMemPtr;
    tib_.table_base = (const void *)cpuTables;
    tib_.work_dim = (cl_uint) sizes.dimensions();

    for (size_t i = 0; i < dims; ++i) {
        tib_.global_offset[i] = offset[i];
        tib_.global_size[i] = global[i];
        tib_.local_size[i] =  local[i];
        tib_.enqueued_local_size[i] =  local[i];
        tib_.local_id[i] = 0;
        tib_.group_id[i] = 0;
    }

    // Fill the remaining dimensions.
    for (size_t i = dims; i < sizeof(tib_.global_size)/sizeof(size_t); ++i) {
        tib_.global_offset[i] =  0;
        tib_.global_size[i] = 1;
        tib_.local_size[i] =  1;
        tib_.enqueued_local_size[i] =  1;
        tib_.local_id[i] = 0;
        tib_.group_id[i] = 0;
    }
}

ALWAYSINLINE void
WorkItem::setGroupId(
    const amd::NDRange& rangeLimits, 
    const amd::NDRange& offset, 
    size_t n)
{
    const size_t dims = rangeLimits.dimensions();
    for (size_t i = 0; i < dims; ++i) {
        size_t lim = rangeLimits[i];
        size_t& val = tib_.group_id[i];
        val = n;
        if (n < lim) {
            tib_.global_offset[i] =
              offset[i] + val * tib_.enqueued_local_size[i];
            tib_.local_id[i] = 0;
            tib_.local_size[i] =
              std::min(tib_.enqueued_local_size[i],
                       tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));

            ++i;
            for (; i < dims; ++i) {
                tib_.global_offset[i] = offset[i];
                tib_.local_id[i] = 0;
                tib_.group_id[i] = 0;
            }
            break;
        }
        else {
            n /= lim;
            val -= n * lim;
            tib_.global_offset[i] =
              offset[i] + val * tib_.enqueued_local_size[i];
            tib_.local_id[i] = 0;
            tib_.local_size[i] =
              std::min(tib_.enqueued_local_size[i],
                       tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));
        }
    }
}

ALWAYSINLINE void
WorkItem::incrementGroupId(
    const amd::NDRange& rangeLimits, 
    const amd::NDRange& offset, 
    size_t n)
{
    const size_t dims = rangeLimits.dimensions();
    for (size_t i = 0; i < dims; ++i) {
        size_t lim = rangeLimits[i];
        size_t& val = tib_.group_id[i];
        val += n;
        if (val < lim) {
            tib_.global_offset[i] =
              offset[i] + val * tib_.enqueued_local_size[i];
            tib_.local_id[i] = 0;
            tib_.local_size[i] =
              std::min(tib_.enqueued_local_size[i],
                       tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));
            break;
        }
        else {
            n = val / lim;
            val -= n * lim;
            tib_.global_offset[i] =
              offset[i] + val * tib_.enqueued_local_size[i];
            tib_.local_id[i] = 0;
            tib_.local_size[i] =
              std::min(tib_.enqueued_local_size[i],
                       tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));
        }
    }
}

void
WorkItem::barrier(cl_mem_fence_flags flags)
{
    WorkItem* workItem = WorkItem::current();
    workItem->swap(workItem->next());
}

void Operation::cleanup()
{
    cl_int lastException = command().exception();
    cl_int status = (lastException != 0) ? lastException : CL_COMPLETE;

    Counter* counter = reinterpret_cast<Counter*>(command().data());
    if (counter == NULL) {
        command().setStatus(status);
    }
    else if (counter->decrement() == 0) {
        counter->event().setStatus(status);
    }
}

} // namespace cpu