rocm-systems/rocclr/runtime/device/cpu/cpucommand.hpp

//
// Copyright 2010 Advanced Micro Devices, Inc. All rights reserved.
//

#ifndef OPERATION_HPP_
#define OPERATION_HPP_

#include "top.hpp"
#include "device/cpu/cpudevice.hpp"
#include "device/cpu/cpukernel.hpp"
#include "platform/command.hpp"
#include "thread/thread.hpp"
#include "os/os.hpp"
#include "amdocl/cl_kernel.h"

#if defined(ATI_ARCH_ARM)
#include <setjmp.h>
#endif // ATI_ARCH_ARM

namespace cpu {

/*! \addtogroup CPU
 *  @{
 *
 *  \addtogroup CPUExec Execution environment
 *  @{
 */

//! A saved stack context
class StackContext : public amd::StackObject
{
private:
#if defined(ATI_ARCH_ARM)
    jmp_buf env_;
#elif defined(_WIN64)
    intptr_t __declspec(align(16)) regs_[32];
#else // !_WIN64
    intptr_t regs_[LP64_SWITCH(6,8)];
#endif // !_WIN64

public:
    //! Save the stack context. Return 0 if returning directly.
    inline intptr_t setjmp();

    //! Restore the stack context
    inline void longjmp(intptr_t val) const;
};

//! A thread fiber
class Fiber : public amd::StackObject
{
private:
    //! Next fiber in the thread.
    Fiber* next_;

    //! This fiber's saved state.
    StackContext context_;

public:
    //! Construct a new Fiber
    Fiber() : next_(NULL) { }

    //! Return the next fiber in the current thread.
    const Fiber* next() const { return next_; }
    //! Set the next fiber in the current thread.
    void setNext(Fiber* next) { next_ = next; }

    //! Save the state of this fiber. Return true if directly returning.
    ALWAYSINLINE bool save() { return context_.setjmp() == 0; }
    //! Restore this fiber from the saved context.
    void restore() const { context_.longjmp(1); }

    //! Switch to the given fiber.
    void swap(const Fiber* fiber) { if (save()) { fiber->restore(); } }
};



//! A CPU core operation (enqueued in the worker thread queue)
class Operation :  public amd::HeapObject
{
public:
    //! An atomic counter
    class Counter
    {
        // FIXME_lmoriche: recycle the counters, implement a thread local pool.
    private:
        amd::Event& event_;
        //! The atomic counter value.
        amd::Atomic<size_t> counter_;

    public:
        //! Initialize the counter with the given initial value.
        Counter(amd::Event& event, size_t initialValue) :
            event_(event), counter_(initialValue) { }
        //! Return the event associated with this counter.
        amd::Event& event() { return event_; }
        //! Decrement the counter and return the new value.
        size_t decrement() { return --counter_; }
    };

protected:
    amd::Command& command_; 

public:
    Operation(amd::Command& command) : command_(command)
    { }

    virtual ~Operation() {};

    virtual void clone(Operation* buf) = 0;

    void cleanup();

    amd::Command& command() { return command_;}

    virtual void execute() = 0;
};

/*! @}
 *  \defgroup CPUOperations Operations
 *  @{
 */

//! A work item instance
class WorkItem : public Fiber
{
private:
    //! Thread info block (must be the last field).
    clk_thread_info_block_t tib_;

private:
    //! Cannot be deleted (allocated with placement new).
    void operator delete(void*) { ShouldNotCallThis(); }

public:
    //! Initialize this workgroup.
    WorkItem(
        const amd::NDRangeContainer& size,
        void* scratchMemPtr,
        void* localMemPtr);

    //! Return the current WorkItem (based of the current stack pointer).
    static WorkItem* current() {
        return (WorkItem*)amd::alignUp((intptr_t) amd::Os::currentStackPtr(),
            CLK_PRIVATE_MEMORY_SIZE) - 1;
    }

    clk_thread_info_block_t& infoBlock() { return tib_; }

    //! Return the native stack pointer base for this workitem.
    address nativeStackPtr() const {
        address newSp = amd::alignDown((address) this - CPUKERNEL_STACK_ALIGN,
                                       CPUKERNEL_STACK_ALIGN);
        WINDOWS_ONLY(NOT_WIN64(newSp += sizeof(void*)));
        return newSp;
    }

    //! These functions are mapping "n" from 1d index to the required dimension
    inline void setGroupId(
        const amd::NDRange& rangeLimits, 
        const amd::NDRange& offset, 
        size_t n);
    inline void incrementGroupId(
        const amd::NDRange& rangeLimits, 
        const amd::NDRange& offset, 
        size_t n);

    //! Execute a thread synchronization barrier.
    static void barrier(cl_mem_fence_flags flags);
};

typedef void (*kernelentrypoint_t)(const void*);

//! Execute a workgroup (work-items).
class WorkGroup
{
private:
    amd::NDRangeKernelCommand& command_;
    const cpu::Kernel& kernel_;
    WorkerThread& thread_;
    address params_;
    WorkItem* const workItem0_;
    const Fiber* workingFiber_;
    size_t numWorkItems_;

public:
    WorkGroup(
        amd::NDRangeKernelCommand& parent,
        const cpu::Kernel& kernel, 
        WorkerThread& thread,
        address params,
        WorkItem* workItem0,
        const size_t numWorkItems) :
            command_(parent), 
            kernel_(kernel), 
            thread_(thread),
            params_(params),
            workItem0_(workItem0),
            numWorkItems_(numWorkItems)
    { }

    WorkItem* getBaseWorkItem() { return workItem0_; }
    WorkerThread& getWorkerThread() { return thread_; }

    void executeWorkItem(); // In case of 1 WorkItem
    void executeWithBarrier();
    void executeWithoutBarrier();

    void setNumWorkItems(size_t workItems) { numWorkItems_ = workItems; }
    size_t getNumWorkItems() { return numWorkItems_; }
private:
    void callKernelRange(
        kernelentrypoint_t entryPoint,
        address stackPtr,
        clk_thread_info_block_t& tib);
    inline void callKernel(
        kernelentrypoint_t entryPoint,
        address stackPtr);
    inline void callKernelProtectedReturn(
        kernelentrypoint_t entryPoint,
        address stackPtr);
};

class NDRangeKernelBatch : public Operation
{
protected:
    size_t coreId_;
    const size_t numWorkGroups_;
    const size_t numCores_;
    volatile size_t currentOpId_;
    const amd::NDRange groupIds_; //!< Number of groups in each dimensions
    VirtualCPU& virtualDevice_;

public:
    enum ExecutionOrder {
        ORDER_DEFAULT,
        ORDER_ROUND_ROBIN = ORDER_DEFAULT,
        //ORDER_LINEAR
    };

    enum ExecutionNature {
        NATURE_WITH_BARRIER,
        NATURE_WITHOUT_BARRIER,
        NATURE_1_WORK_ITEM,
        NATURE_WG_LEVEL_EXEC
    };

    NDRangeKernelBatch(
        amd::NDRangeKernelCommand& parent,
        VirtualCPU& virtualDevice,
        const amd::NDRange& groupIds, size_t numCores) :
            Operation(parent), 
            coreId_(0),
            numWorkGroups_(groupIds.product()),
            numCores_(numCores),
            currentOpId_(0),
            groupIds_(groupIds), 
            virtualDevice_(virtualDevice)
    { }

    virtual void clone(Operation* buf)
    {
        ::new(buf) NDRangeKernelBatch(static_cast<amd::NDRangeKernelCommand&>(command_), 
            virtualDevice_, groupIds_, numCores_);
        static_cast<NDRangeKernelBatch*>(buf)->setCoreId(coreId_);
    }

    virtual void execute();

    void setCoreId(size_t coreId) { coreId_ = coreId; currentOpId_ = coreId; }

    inline bool getNextOperationId(size_t& opId);
    inline size_t getNextOperationIds(size_t& opId, size_t count);

private:
    bool patchParameters(
        const cpu::Kernel& kernel,
        address params,
        address& localMemPtr,
        const address localMemLimit,
        size_t localMemSize) const;
};

class NativeFn : public Operation
{
public:
    NativeFn(amd::NativeFnCommand& parent) : Operation(parent)
    { }

    virtual void clone(Operation* buf)
    {
        ::new(buf) NativeFn(static_cast<amd::NativeFnCommand&>(command_));
    }

    virtual void execute();
};
#ifndef MAX
#define MAX(x,y) ((x)>=(y) ?(x) : (y))
#endif //MAX

#define MAX_OPERATION_ALLOC_SIZE (MAX(sizeof(NDRangeKernelBatch), sizeof(NativeFn)))

//! A thread bound to a cpu core.
class WorkerThread : public amd::Thread
{
private:
    Fiber mainFiber_; //!< main fiber for this worker thread.

    amd::Monitor queueLock_; //!< lock protecting the queue.
    volatile int waitingOp_;
    bool terminated_; //!< true if the thread is shutting down.
    
    //! Local memory storage
    address localDataStorage_;
    //! Size of the local memory.
    size_t localDataSize_;

    char operation_[MAX_OPERATION_ALLOC_SIZE];

    address baseWorkItemsStack_;
private:
    //! Awaits operations and execute them as they become ready.
    void loop();

public:
    //! Construct a new WorkerThread.
    WorkerThread(const cpu::Device& device);
    //! Destroy the worker thread.
    virtual ~WorkerThread();
    //! Cleanup the thread before termination.
    bool terminate();

    //! Return the main fiber for this thread.
    Fiber& mainFiber() { return mainFiber_; }
    //! Return the LDS for this thread
    address localDataStorage() const { return localDataStorage_; }
    //! Return the size of the local memory for this thread.
    size_t localDataSize() const { return localDataSize_; }

    address baseWorkItemsStack() { return baseWorkItemsStack_; }

    Operation* operation() { return reinterpret_cast<Operation*>(operation_); }
    bool isOperationValid() { return waitingOp_ > 0; }

    //! Enqueue a new operation to execute in this thread.
    void enqueue(Operation& op);
    //! Signal to start processing the commands in the queue.
    void flush() { amd::ScopedLock sl(queueLock_); queueLock_.notify(); }

    //! This thread's execution engine.
    void run(void* data) {
        loop();
    }

    bool isWorkerThread() const { return true; }

    //! Return the currently executing WorkerThread's instance.
    static WorkerThread* current()
    {
        return static_cast<WorkerThread*>(Thread::current());
    }
};

/*! @}
 *  @}
 */

extern "C" intptr_t _StackContext_setjmp(intptr_t* regs);

#if !defined(ATI_ARCH_ARM)
ALWAYSINLINE
#endif
intptr_t
StackContext::setjmp()
{
#if defined(ATI_ARCH_ARM)
    return ::setjmp(env_);
#else
    return _StackContext_setjmp(regs_);
#endif
}

extern "C" void _StackContext_longjmp(const intptr_t* env, intptr_t val);

ALWAYSINLINE void
StackContext::longjmp(intptr_t val) const
{
#if defined(ATI_ARCH_ARM)
    return ::longjmp(*const_cast<jmp_buf*>(&env_), val);
#else
    return _StackContext_longjmp(regs_, val);
#endif
}



extern "C" void _WorkGroup_callKernel( 
    address params,
    kernelentrypoint_t entryPoint,
    address stackPtr);

extern "C" void _WorkGroup_callKernelProtectedReturn( 
    address params,
    kernelentrypoint_t entryPoint,
    address stackPtr);


ALWAYSINLINE void
WorkGroup::callKernel(
    kernelentrypoint_t entryPoint,
    address stackPtr)
{
    _WorkGroup_callKernel(params_, entryPoint, stackPtr);
}

// This version support the case of changing the stack for fibers.
ALWAYSINLINE void
WorkGroup::callKernelProtectedReturn(
    kernelentrypoint_t entryPoint,
    address stackPtr)
{
    _WorkGroup_callKernelProtectedReturn(params_, entryPoint, stackPtr);
}


} // namespace cpu

#endif /*OPERATION_HPP_*/