initial commit
This commit is contained in:
foreman
@@ -0,0 +1,213 @@
|
||||
//
|
||||
// Copyright 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#include "device/cpu/cpubinary.hpp"
|
||||
#include "device/cpu/cpudevice.hpp"
|
||||
#include "device/cpu/cpuprogram.hpp"
|
||||
#include "utils/versions.hpp"
|
||||
#include "os/os.hpp"
|
||||
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
namespace cpu {
|
||||
|
||||
ClBinary::FeatureCheckResult
|
||||
ClBinary::checkFeatures()
|
||||
{
|
||||
/* Validate that all cpu features of loaded binary target (i.e. elf_target) exists in current target.
|
||||
* If some of elf_target features doesn't exist in current target we fail the build since we assume that elf LLVM-IR and binary are
|
||||
* target specific and can't be recompiled to current target*/
|
||||
uint16_t target = (uint16_t)dev().settings().cpuFeatures_;
|
||||
uint16_t elf_target;
|
||||
amd::OclElf::oclElfPlatform platform;
|
||||
if (!elfIn()->getTarget(elf_target, platform)){
|
||||
LogError("Loading OCL CPU binary: incorrect format");
|
||||
return ERROR;
|
||||
}
|
||||
uint64_t chip_options=0x0;
|
||||
if (platform == amd::OclElf::COMPLIB_PLATFORM) {
|
||||
// BIF 3.0
|
||||
uint32_t flag;
|
||||
if (!elfIn()->getFlags(flag)) {
|
||||
LogError("Loading OCL CPU binary: incorrect format");
|
||||
return ERROR;
|
||||
}
|
||||
aclTargetInfo tgtInfo = aclGetTargetInfoFromChipID(LP64_SWITCH("x86", "x86-64"), flag, NULL);
|
||||
chip_options = aclGetChipOptions(tgtInfo) ;
|
||||
if (((target & chip_options) != chip_options) ||
|
||||
((elf_target == EM_386) && (strcmp(LP64_SWITCH("x86", "x86-64"), "x86") != 0)) ||
|
||||
((elf_target == EM_X86_64) && (strcmp(LP64_SWITCH("x86", "x86-64"), "x86-64") != 0))){
|
||||
LogError("Loading OCL CPU binary: different target");
|
||||
return ERROR;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// BIF 2.0
|
||||
if ((platform != amd::OclElf::CPU_PLATFORM) ||
|
||||
((target & elf_target) != elf_target)) {
|
||||
LogError("Loading OCL CPU binary: different target");
|
||||
return ERROR;
|
||||
}
|
||||
}
|
||||
char* section;
|
||||
size_t sz;
|
||||
|
||||
/* If current target has more cpu features than the one for which the binary was (notice it must have all features as in elf_target
|
||||
* due to previous check), we can benefit from recompiling the LLVM-IR if exists in binary (if there are errors, ignore them !).*/
|
||||
if (((platform == amd::OclElf::CPU_PLATFORM) &&
|
||||
((target ^ elf_target) != 0)) ||
|
||||
((platform == amd::OclElf::COMPLIB_PLATFORM) &&
|
||||
((target ^ chip_options) != 0))) {
|
||||
if (elfIn_->getSection(amd::OclElf::LLVMIR, §ion, &sz)) {
|
||||
if ((section != NULL) && (sz > 0)) {
|
||||
// hasDLL being false to force recompiling
|
||||
RECOMPILE;
|
||||
}
|
||||
}
|
||||
}
|
||||
return OK;
|
||||
}
|
||||
|
||||
bool
|
||||
ClBinary::loadX86(Program& program, std::string& dllName, bool& hasDLL)
|
||||
{
|
||||
hasDLL = false;
|
||||
|
||||
std::string tempName = amd::Os::getTempFileName();
|
||||
|
||||
dllName = tempName
|
||||
+ "." WINDOWS_SWITCH("dll",MACOS_SWITCH("dyld","so"));
|
||||
|
||||
switch (checkFeatures()) {
|
||||
case ERROR:
|
||||
return false;
|
||||
case RECOMPILE:
|
||||
return true;
|
||||
case OK:
|
||||
// Fallthrough
|
||||
break;
|
||||
}
|
||||
|
||||
char* section;
|
||||
size_t sz;
|
||||
|
||||
if (!elfIn_->getSection(amd::OclElf::DLL, §ion, &sz)) {
|
||||
LogError("Loading OCL CPU binary: error occured!");
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((section == NULL) || (sz == 0)) {
|
||||
// hasDLL being false to force recompiling
|
||||
return true;
|
||||
}
|
||||
|
||||
std::fstream f;
|
||||
f.open(dllName.c_str(), (std::fstream::out | std::fstream::binary));
|
||||
|
||||
if (!f.is_open()) {
|
||||
#ifdef _WIN32
|
||||
amd::Os::unlink(tempName.c_str());
|
||||
#endif // _WIN32
|
||||
LogError("Loading OCL CPU binary: cannot open a file!");
|
||||
return false;
|
||||
}
|
||||
f.write(section, sz);
|
||||
f.close();
|
||||
|
||||
hasDLL = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
ClBinary::storeX86(Program& program, std::string& dllName)
|
||||
{
|
||||
std::fstream f;
|
||||
f.open(dllName.c_str(), (std::fstream::in | std::fstream::binary));
|
||||
if (!f.is_open()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
f.seekg(0, std::fstream::end);
|
||||
size_t x86CodeSize = f.tellg();
|
||||
f.seekg(0, std::fstream::beg);
|
||||
|
||||
if (saveISA()) {
|
||||
char* x86Code = new char[x86CodeSize];
|
||||
f.read(x86Code, x86CodeSize);
|
||||
elfOut_->addSection(amd::OclElf::DLL, x86Code, x86CodeSize);
|
||||
delete [] x86Code;
|
||||
}
|
||||
f.close();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
ClBinary::loadX86JIT(Program& program, bool& hasJITBinary)
|
||||
{
|
||||
hasJITBinary = false;
|
||||
|
||||
switch (checkFeatures()) {
|
||||
case ERROR:
|
||||
return false;
|
||||
case RECOMPILE:
|
||||
return true;
|
||||
case OK:
|
||||
// Fallthrough
|
||||
break;
|
||||
}
|
||||
|
||||
char* section;
|
||||
size_t sz;
|
||||
|
||||
if (!elfIn_->getSection(amd::OclElf::JITBINARY, §ion, &sz)) {
|
||||
LogError("Loading OCL CPU JIT binary: error occured!");
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((section == NULL) || (sz == 0)) {
|
||||
// force recompiling
|
||||
return true;
|
||||
}
|
||||
|
||||
program.setJITBinary(aclJITObjectImageCopy(section, sz));
|
||||
|
||||
hasJITBinary = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
void checkDifference(const char* buf1, const char* buf2, size_t size) {
|
||||
for(size_t i = 0; i < size; ++i) {
|
||||
if(buf1[i] != buf2[i]) {
|
||||
printf("Index %d different",(int)i);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
bool
|
||||
ClBinary::storeX86JIT(Program& program)
|
||||
{
|
||||
if (saveISA()) {
|
||||
aclJITObjectImage objectImage = program.getJITBinary();
|
||||
const char* x86CodePtr = aclJITObjectImageData(objectImage);
|
||||
size_t x86CodeSize = aclJITObjectImageSize(objectImage);
|
||||
|
||||
elfOut_->addSection(amd::OclElf::JITBINARY, x86CodePtr, x86CodeSize);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
ClBinary::storeX86Asm(const char* buffer, size_t size)
|
||||
{
|
||||
if (saveAS()) {
|
||||
elfOut_->addSection(amd::OclElf::ASTEXT, buffer, size);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace cpu
|
||||
@@ -0,0 +1,85 @@
|
||||
//
|
||||
// Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef CPUBINARY_HPP_
|
||||
#define CPUBINARY_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "device/device.hpp"
|
||||
#include "device/cpu/cpudevice.hpp"
|
||||
#include "elf/elf.hpp"
|
||||
|
||||
//! \namespace cpu CPU Device Implementation
|
||||
namespace cpu {
|
||||
|
||||
class Device;
|
||||
class Program;
|
||||
|
||||
//! \class CPU binary
|
||||
class ClBinary : public device::ClBinary
|
||||
{
|
||||
public:
|
||||
//! Constructor
|
||||
ClBinary(const Device& dev) : device::ClBinary(dev) {}
|
||||
|
||||
//! Destructor
|
||||
~ClBinary() {}
|
||||
|
||||
//! Loads x86 executable code
|
||||
bool loadX86(
|
||||
Program& prorgam, //!< CPU Program object
|
||||
std::string& dllName, //!< Dll name of the CPU binary
|
||||
bool& hasDLL //!< indicate if the OCL binary has DLL
|
||||
);
|
||||
|
||||
//! Stores x86 executable code
|
||||
bool storeX86(
|
||||
Program& program, //!< CPU Program object
|
||||
std::string& dllName //!< Dll name for the binary
|
||||
);
|
||||
|
||||
//! Loads x86 executable in-memory code
|
||||
bool loadX86JIT(
|
||||
Program& prorgam, //!< CPU Program object
|
||||
bool& hasJITBin //!< indicate if the OCL binary has JIT binary
|
||||
);
|
||||
|
||||
//! Stores x86 executable in-memory code
|
||||
bool storeX86JIT(
|
||||
Program& program //!< CPU Program object
|
||||
);
|
||||
|
||||
//! Set elf header information for CPU target
|
||||
bool setElfTarget() {
|
||||
uint32_t target = dev().settings().cpuFeatures_;
|
||||
assert (((0xFFFF8000 & target) == 0) && "ASIC target ID >= 2^15");
|
||||
uint16_t elf_target = (uint16_t)(0x7FFF & target);
|
||||
return elfOut()->setTarget(elf_target, amd::OclElf::CPU_PLATFORM);
|
||||
}
|
||||
|
||||
bool storeX86Asm(const char* buffer, size_t size);
|
||||
|
||||
private:
|
||||
|
||||
enum FeatureCheckResult {
|
||||
ERROR,
|
||||
RECOMPILE,
|
||||
OK
|
||||
};
|
||||
|
||||
FeatureCheckResult checkFeatures();
|
||||
|
||||
//! Disable default copy constructor
|
||||
ClBinary(const ClBinary&);
|
||||
|
||||
//! Disable default operator=
|
||||
ClBinary& operator=(const ClBinary&);
|
||||
|
||||
//! Returns the GPU device for this object
|
||||
const Device& dev() { return static_cast<const Device&>(dev_); }
|
||||
};
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif // CPUBINARY_HPP_
|
||||
@@ -0,0 +1,57 @@
|
||||
//
|
||||
// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#include "device/cpu/cpubuiltins.hpp"
|
||||
#include "device/cpu/cpucommand.hpp"
|
||||
|
||||
#include <amdocl/cl_kernel.h>
|
||||
#include <cstdio> // for printf
|
||||
#include <stdarg.h>
|
||||
|
||||
#define BUF_SIZE_PRINTF 4095
|
||||
//In the current implementation of printf in gcc 4.5.2 runtime libraries,inf/infinity and nan are not supported
|
||||
//The [-]infinity value is printed as [-]1.#INF00
|
||||
//The [-]nan value is printed as [-]1.#INF00
|
||||
//bufOutUpdate converts the all printed instanced of [-]1.#INF00 to inf,and
|
||||
// all printed instanced of [-]1.#IND00 to nan
|
||||
void bufOutUpdate(std::string& sBufOut,const char* strToReplace,const char* strReplace)
|
||||
{
|
||||
size_t foundIdx = 0;
|
||||
while ((foundIdx = sBufOut.find(strToReplace,foundIdx)) != std::string::npos) {
|
||||
sBufOut.replace(foundIdx,strlen(strToReplace),strReplace,strlen(strReplace));
|
||||
foundIdx += 3;
|
||||
}
|
||||
}
|
||||
int cpuprintf(const char* format,...)
|
||||
{
|
||||
char cBufOut[BUF_SIZE_PRINTF];
|
||||
std::string sBufOut;
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
//write to the buffer
|
||||
vsprintf(cBufOut,format,args);
|
||||
sBufOut = cBufOut;
|
||||
|
||||
//convert to correct infinity/nan representation
|
||||
bufOutUpdate(sBufOut,"1.#INF00","inf");
|
||||
bufOutUpdate(sBufOut,"1.#IND00","nan");
|
||||
bufOutUpdate(sBufOut,"1.#QNAN0","nan");
|
||||
int ret = amd::Os::printf("%s",sBufOut.c_str());
|
||||
fflush(stdout);
|
||||
va_end (args);
|
||||
return ret;
|
||||
}
|
||||
namespace cpu {
|
||||
|
||||
const clk_builtins_t
|
||||
Builtins::dispatchTable_ =
|
||||
{
|
||||
/* Synchronization functions */
|
||||
&WorkItem::barrier,
|
||||
/* AMD Only builtins: FIXME_lmoriche: remove or add an extension */
|
||||
NULL,
|
||||
cpuprintf
|
||||
};
|
||||
|
||||
} // namespace cpu
|
||||
@@ -0,0 +1,20 @@
|
||||
//
|
||||
// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef BUILTINS_HPP_
|
||||
#define BUILTINS_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "amdocl/cl_kernel.h"
|
||||
|
||||
namespace cpu {
|
||||
|
||||
struct Builtins : public amd::AllStatic
|
||||
{
|
||||
static const clk_builtins_t dispatchTable_;
|
||||
};
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif /*BUILTINS_HPP_*/
|
||||
@@ -0,0 +1,676 @@
|
||||
//
|
||||
// Copyright 2010 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#include "device/cpu/cpucommand.hpp"
|
||||
#include "device/cpu/cpubuiltins.hpp"
|
||||
#include "device/cpu/cpudevice.hpp"
|
||||
#include "device/cpu/cputables.hpp"
|
||||
#include "platform/command.hpp"
|
||||
#include "platform/commandqueue.hpp"
|
||||
#include "platform/program.hpp"
|
||||
#include "platform/kernel.hpp"
|
||||
#include "platform/sampler.hpp"
|
||||
#include "thread/thread.hpp"
|
||||
#include "os/os.hpp"
|
||||
#include "utils/util.hpp"
|
||||
|
||||
#include <amdocl/cl_kernel.h>
|
||||
|
||||
namespace cpu {
|
||||
|
||||
#define CPU_WORKER_THREAD_TOTAL_STACK_SIZE (CPU_WORKER_THREAD_STACK_SIZE + \
|
||||
CLK_PRIVATE_MEMORY_SIZE * (CPU_MAX_WORKGROUP_SIZE + 1))
|
||||
|
||||
WorkerThread::WorkerThread(const cpu::Device& device) :
|
||||
Thread("CPU Worker Thread", CPU_WORKER_THREAD_TOTAL_STACK_SIZE),
|
||||
queueLock_("WorkerThread::queueLock"), waitingOp_(0), terminated_(false)
|
||||
{
|
||||
localDataSize_ = (size_t) device.info().localMemSize_;
|
||||
localDataStorage_ = (address) amd::AlignedMemory::allocate(
|
||||
localDataSize_ + __CPU_SCRATCH_SIZE, sizeof(cl_long16)) +
|
||||
__CPU_SCRATCH_SIZE;
|
||||
|
||||
#if defined(__linux__) && defined(NUMA_SUPPORT)
|
||||
const nodemask_t* numaMask = device.getNumaMask();
|
||||
if (numaMask != NULL) {
|
||||
numa_bind(numaMask);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
WorkerThread::~WorkerThread()
|
||||
{
|
||||
guarantee(Thread::current() != this && "thread suicide!");
|
||||
amd::AlignedMemory::deallocate(localDataStorage_ - __CPU_SCRATCH_SIZE);
|
||||
}
|
||||
|
||||
bool
|
||||
WorkerThread::terminate()
|
||||
{
|
||||
terminated_ = true;
|
||||
|
||||
if (Thread::current() != this) {
|
||||
// FIXME_lmoriche: fix termination handshake
|
||||
while (state() < Thread::FINISHED) {
|
||||
flush();
|
||||
amd::Os::yield();
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
WorkerThread::enqueue(Operation& op)
|
||||
{
|
||||
while (waitingOp_ != 0) {
|
||||
amd::Os::yield();
|
||||
}
|
||||
op.clone(operation());
|
||||
++waitingOp_;
|
||||
}
|
||||
|
||||
void
|
||||
WorkerThread::loop()
|
||||
{
|
||||
baseWorkItemsStack_ = amd::alignDown(stackBase() -
|
||||
CPU_WORKER_THREAD_STACK_SIZE, CLK_PRIVATE_MEMORY_SIZE);
|
||||
#if defined(WIN32)
|
||||
amd::Os::touchStackPages(baseWorkItemsStack_, amd::Os::currentStackPtr());
|
||||
#endif // WINDOWS
|
||||
Operation *op = operation();
|
||||
|
||||
queueLock_.lock();
|
||||
while (true) {
|
||||
while (waitingOp_ == 0) {
|
||||
if (terminated_) {
|
||||
break;
|
||||
}
|
||||
queueLock_.wait();
|
||||
}
|
||||
if (terminated_) {
|
||||
break;
|
||||
}
|
||||
op->command().setStatus(CL_RUNNING);
|
||||
op->execute();
|
||||
op->cleanup();
|
||||
--waitingOp_;
|
||||
}
|
||||
queueLock_.unlock();
|
||||
}
|
||||
|
||||
void
|
||||
NativeFn::execute()
|
||||
{
|
||||
cl_int status = static_cast<amd::NativeFnCommand&>(command()).invoke();
|
||||
command().setStatus(status);
|
||||
}
|
||||
|
||||
static void
|
||||
nop() { /*Do nothing*/ }
|
||||
|
||||
|
||||
template <NDRangeKernelBatch::ExecutionNature NATURE,
|
||||
NDRangeKernelBatch::ExecutionOrder ORDER =
|
||||
NDRangeKernelBatch::ORDER_DEFAULT>
|
||||
class NDRangeKernelBatchMode : public NDRangeKernelBatch
|
||||
{
|
||||
private:
|
||||
void executeWorkGroup(WorkGroup& wg)
|
||||
{
|
||||
if (NATURE == NATURE_WG_LEVEL_EXEC) {
|
||||
wg.executeWorkItem();
|
||||
}
|
||||
else if ((NATURE == NATURE_1_WORK_ITEM) ||
|
||||
(wg.getNumWorkItems() == 1)) {
|
||||
wg.executeWorkItem();
|
||||
}
|
||||
else {
|
||||
wg.getBaseWorkItem()->setNext(&wg.getWorkerThread().mainFiber());
|
||||
if (NATURE == NATURE_WITHOUT_BARRIER) {
|
||||
wg.executeWithoutBarrier();
|
||||
}
|
||||
else { // NATURE == NATURE_WITH_BARRIER
|
||||
wg.executeWithBarrier();
|
||||
}
|
||||
}
|
||||
// Yield at the end of each workgroup to avoid starving GPU device
|
||||
amd::Os::yield();
|
||||
}
|
||||
|
||||
public:
|
||||
void executeMode(WorkGroup& wg)
|
||||
{
|
||||
const amd::NDRange& offset =
|
||||
static_cast<amd::NDRangeKernelCommand&>(command_).sizes().offset();
|
||||
WorkItem* workItem0 = wg.getBaseWorkItem();
|
||||
clk_builtins_t tableTask;
|
||||
size_t prevOpId = 0, opId = (size_t)-1;
|
||||
|
||||
if (NATURE == NATURE_1_WORK_ITEM) {
|
||||
tableTask = Builtins::dispatchTable_;
|
||||
|
||||
// If local size == 1 then barrier() becomes a nop.
|
||||
tableTask.barrier_ptr = (void (*)(cl_mem_fence_flags)) nop;
|
||||
workItem0->infoBlock().builtins = &tableTask;
|
||||
workItem0->setNext(&wg.getWorkerThread().mainFiber());
|
||||
}
|
||||
|
||||
while (getNextOperationId(opId)) {
|
||||
workItem0->incrementGroupId(groupIds_, offset, opId - prevOpId);
|
||||
uint workDims = workItem0->infoBlock().work_dim;
|
||||
size_t numWorkItems = workItem0->infoBlock().local_size[0] *
|
||||
(workDims >= 2 ? workItem0->infoBlock().local_size[1] : 1) *
|
||||
(workDims >= 3 ? workItem0->infoBlock().local_size[2] : 1);
|
||||
wg.setNumWorkItems(numWorkItems);
|
||||
if(numWorkItems == 1) {
|
||||
tableTask = Builtins::dispatchTable_;
|
||||
tableTask.barrier_ptr = (void (*)(cl_mem_fence_flags)) nop;
|
||||
workItem0->infoBlock().builtins = &tableTask;
|
||||
workItem0->setNext(&wg.getWorkerThread().mainFiber());
|
||||
executeWorkGroup(wg);
|
||||
tableTask.barrier_ptr = &WorkItem::barrier;
|
||||
} else {
|
||||
executeWorkGroup(wg);
|
||||
}
|
||||
prevOpId = opId;
|
||||
}
|
||||
|
||||
//#define DISABLE_TASK_STEALING
|
||||
#if !defined(DISABLE_TASK_STEALING) && 0
|
||||
size_t maxId = numCores_;
|
||||
size_t stolenId = coreId_ + 1;
|
||||
NDRangeKernelBatch* workingBatch = this;
|
||||
size_t numStolenIds = 1;
|
||||
const size_t maxStealingSize = 3;
|
||||
const size_t minAdaptiveStealingDiff = numCores_ * maxStealingSize;
|
||||
|
||||
while (true) {
|
||||
for (; stolenId < maxId; ++stolenId) {
|
||||
WorkerThread* worker = virtualDevice_.getWorkerThread(stolenId);
|
||||
|
||||
// In case were we have less operations than Worker Threads
|
||||
if (worker->isOperationValid()) {
|
||||
workingBatch = static_cast<NDRangeKernelBatch*>(
|
||||
worker->operation());
|
||||
|
||||
numStolenIds =
|
||||
workingBatch->getNextOperationIds(opId, numStolenIds);
|
||||
if (numStolenIds > 0) {
|
||||
do {
|
||||
for (size_t i = 0; i < numStolenIds; ++i) {
|
||||
workItem0->setGroupId(groupIds_, offset, opId);
|
||||
executeWorkGroup(wg);
|
||||
opId += numCores_;
|
||||
}
|
||||
|
||||
// adaptive stealing
|
||||
if (numWorkGroups_ - opId > minAdaptiveStealingDiff) {
|
||||
numStolenIds = maxStealingSize;
|
||||
}
|
||||
else {
|
||||
while (workingBatch->getNextOperationId(opId)) {
|
||||
workItem0->setGroupId(groupIds_, offset, opId);
|
||||
executeWorkGroup(wg);
|
||||
}
|
||||
break;
|
||||
}
|
||||
numStolenIds = workingBatch->getNextOperationIds(
|
||||
opId, numStolenIds);
|
||||
} while (numStolenIds > 0);
|
||||
}
|
||||
numStolenIds = 1;
|
||||
}
|
||||
} // for (stolenId..maxId)
|
||||
|
||||
if (stolenId == coreId_) {
|
||||
break;
|
||||
}
|
||||
|
||||
stolenId = 0;
|
||||
maxId = coreId_;
|
||||
} // while (true)
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
inline bool
|
||||
NDRangeKernelBatch::getNextOperationId(size_t& opId)
|
||||
{
|
||||
if (currentOpId_ >= numWorkGroups_) {
|
||||
return false;
|
||||
}
|
||||
opId = amd::AtomicOperation::add(numCores_, ¤tOpId_);
|
||||
return opId < numWorkGroups_;
|
||||
}
|
||||
|
||||
|
||||
inline size_t
|
||||
NDRangeKernelBatch::getNextOperationIds(size_t& opId, size_t count)
|
||||
{
|
||||
size_t topId = numCores_ * count;
|
||||
if (currentOpId_ >= numWorkGroups_) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
opId = amd::AtomicOperation::add(topId, ¤tOpId_);
|
||||
const size_t numWorkGroups = numWorkGroups_;
|
||||
if (opId >= numWorkGroups) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
topId += opId;
|
||||
if (topId >= (numWorkGroups + numCores_)) {
|
||||
count -= (topId - numWorkGroups) / numCores_;
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
// Process the parameters, allocate LDS.
|
||||
bool
|
||||
NDRangeKernelBatch::patchParameters(
|
||||
const cpu::Kernel& cpuKernel,
|
||||
address params,
|
||||
address& localMemPtr,
|
||||
const address localMemLimit,
|
||||
size_t localMemSize) const
|
||||
{
|
||||
amd::NDRangeKernelCommand& command =
|
||||
static_cast<amd::NDRangeKernelCommand&>(command_);
|
||||
|
||||
const amd::Device& device = command.queue()->device();
|
||||
|
||||
const amd::Kernel& kernel = command.kernel();
|
||||
const amd::KernelSignature& signature = kernel.signature();
|
||||
const amd::KernelParameters& kernelParam = kernel.parameters();
|
||||
|
||||
const_address cmdParams = command.parameters();
|
||||
|
||||
unsigned effectiveOffset = 0;
|
||||
|
||||
// DD -- on CPU device, real effective offset is NATIVELY aligned
|
||||
// Here all source arguments are in place, so we're safe just iterating
|
||||
for (size_t i = 0; i < signature.numParameters(); ++i) {
|
||||
const amd::KernelParameterDescriptor& desc = signature.at(i);
|
||||
const void* cmdParam = cmdParams + desc.offset_;
|
||||
void *param;
|
||||
size_t prmSize = cpuKernel.getArgSize(i);
|
||||
|
||||
// Align i'th parameter on multiple of its size. Parameter size is power of 2.
|
||||
size_t alignment = cpuKernel.getArgAlignment(i);
|
||||
effectiveOffset = amd::alignUp(effectiveOffset, std::min(alignment, size_t(16)));
|
||||
param = params + effectiveOffset;
|
||||
|
||||
if (desc.size_ == 0) {
|
||||
// __local memory parameter
|
||||
localMemPtr = amd::alignUp(localMemPtr, sizeof(cl_long16));
|
||||
|
||||
size_t length = *static_cast<const size_t*>(cmdParam);
|
||||
*static_cast<void**>(param) = localMemPtr;
|
||||
localMemPtr += length;
|
||||
|
||||
if (localMemPtr > localMemLimit) {
|
||||
command.setException(CL_MEM_OBJECT_ALLOCATION_FAILURE);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (desc.type_ == T_POINTER) {
|
||||
// __global memory parameter
|
||||
cl_mem_object_type pointer_type = CL_MEM_OBJECT_BUFFER;
|
||||
if (kernelParam.boundToSvmPointer(device, cmdParams, i)) {
|
||||
*reinterpret_cast<void**>(param) =
|
||||
*reinterpret_cast<void* const *>(cmdParam);
|
||||
}
|
||||
else {
|
||||
void* hostMemPtr = NULL;
|
||||
amd::Memory* memArg =
|
||||
*reinterpret_cast<amd::Memory* const *>(cmdParam);
|
||||
if (memArg != NULL) {
|
||||
hostMemPtr = memArg->getHostMem();
|
||||
if (hostMemPtr == NULL) {
|
||||
command.setException(CL_MEM_OBJECT_ALLOCATION_FAILURE);
|
||||
return false;
|
||||
}
|
||||
pointer_type = memArg->getType();
|
||||
}
|
||||
// For images on CPU devices, pass "struct {int4 p0; int4 p1}".
|
||||
// That allows an obvious implementation for
|
||||
// __amdil_get_image[23]d_params[01].
|
||||
// That makes the rest of the .bc implementation for
|
||||
// images relatively straight forward.
|
||||
if (pointer_type == CL_MEM_OBJECT_IMAGE1D ||
|
||||
pointer_type == CL_MEM_OBJECT_IMAGE2D ||
|
||||
pointer_type == CL_MEM_OBJECT_IMAGE3D ||
|
||||
pointer_type == CL_MEM_OBJECT_IMAGE1D_ARRAY ||
|
||||
pointer_type == CL_MEM_OBJECT_IMAGE1D_BUFFER ||
|
||||
pointer_type == CL_MEM_OBJECT_IMAGE2D_ARRAY) {
|
||||
amd::Image::Impl& impl = memArg->asImage()->getImpl();
|
||||
impl.reserved_ = hostMemPtr;
|
||||
*reinterpret_cast<void**>(param) = (void*)&impl;
|
||||
} else {
|
||||
*reinterpret_cast<void**>(param) = hostMemPtr;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (desc.type_ == T_SAMPLER) {
|
||||
// Switch from an Amd::Sampler to the 32bit integer
|
||||
// variable that is a clk_sampler.
|
||||
amd::Sampler* samplerArg =
|
||||
*reinterpret_cast<amd::Sampler* const *>(cmdParam);
|
||||
*reinterpret_cast<uint32_t*>(param) = (uint32_t)samplerArg->state();
|
||||
}
|
||||
else {
|
||||
::memcpy(param, cmdParam, desc.size_);
|
||||
}
|
||||
|
||||
effectiveOffset += cpuKernel.getArgSize(i);
|
||||
}
|
||||
|
||||
localMemPtr = amd::alignUp(localMemPtr, sizeof(cl_long16));
|
||||
if ((localMemPtr + localMemSize) > localMemLimit) {
|
||||
command.setException(CL_MEM_OBJECT_ALLOCATION_FAILURE);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
NDRangeKernelBatch::execute()
|
||||
{
|
||||
amd::NDRangeKernelCommand& command =
|
||||
static_cast<amd::NDRangeKernelCommand&>(command_);
|
||||
|
||||
const cpu::Kernel& kernel = static_cast<const cpu::Kernel&>(
|
||||
*command.kernel().getDeviceKernel(command.queue()->device()));
|
||||
|
||||
WorkerThread& thread = *WorkerThread::current();
|
||||
|
||||
const size_t numWorkItems = command.sizes().local().product();
|
||||
|
||||
address params = thread.baseWorkItemsStack();
|
||||
address localMemPtr = thread.localDataStorage();
|
||||
if (!patchParameters(kernel,
|
||||
params, localMemPtr, localMemPtr + thread.localDataSize(),
|
||||
kernel.workGroupInfo()->localMemSize_)) {
|
||||
return;
|
||||
}
|
||||
|
||||
WorkItem* workItem0 = ::new((WorkItem*)params - 1) WorkItem(
|
||||
command.sizes(), localMemPtr);
|
||||
|
||||
WorkGroup wg(command, kernel, thread, params, workItem0, numWorkItems);
|
||||
|
||||
if (numWorkItems == 1) {
|
||||
static_cast<NDRangeKernelBatchMode<NATURE_1_WORK_ITEM>*>(this)->
|
||||
executeMode(wg);
|
||||
}
|
||||
else if (kernel.hasBarrier()) {
|
||||
static_cast<NDRangeKernelBatchMode<NATURE_WITH_BARRIER>*>(this)->
|
||||
executeMode(wg);
|
||||
}
|
||||
else {
|
||||
static_cast<NDRangeKernelBatchMode<NATURE_WITHOUT_BARRIER>*>(this)->
|
||||
executeMode(wg);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
WorkGroup::executeWorkItem()
|
||||
{
|
||||
callKernel((kernelentrypoint_t)kernel_.getEntryPoint(), workItem0_->nativeStackPtr());
|
||||
}
|
||||
|
||||
void
|
||||
WorkGroup::executeWithBarrier()
|
||||
{
|
||||
kernelentrypoint_t entryPoint = (kernelentrypoint_t)kernel_.getEntryPoint();
|
||||
|
||||
workingFiber_ = workItem0_;
|
||||
address workGroupStackPtr = workItem0_->nativeStackPtr();
|
||||
|
||||
// Save the current stack context in case we execute a barrier.
|
||||
volatile size_t threadCounter = 0;
|
||||
bool barrier = !thread_.mainFiber().save();
|
||||
|
||||
size_t tid = threadCounter++;
|
||||
WorkItem* workItem = (WorkItem*)((char*) workItem0_
|
||||
- tid * CLK_PRIVATE_MEMORY_SIZE);
|
||||
|
||||
if (barrier) {
|
||||
WorkItem* prev = (WorkItem*)((char*) workItem
|
||||
+ CLK_PRIVATE_MEMORY_SIZE);
|
||||
|
||||
WINDOWS_ONLY(amd::Os::touchStackPages(
|
||||
(address) (workItem + 1), (address) prev));
|
||||
::memcpy(workItem, prev, sizeof(WorkItem));
|
||||
|
||||
clk_thread_info_block_t& tib = workItem->infoBlock();
|
||||
++tib.local_id[0];
|
||||
if (unlikely(tib.local_id[0] >= tib.local_size[0])) {
|
||||
//
|
||||
// Compiling for Windows 64bit (only in release) introduces a bug,
|
||||
// which uses the same register for saving threadCounter and the
|
||||
// 0 value. Therefore "tib.local_id[i] = 0" was actually translated
|
||||
// to "tib.local_id[0] = threadCounter". To avoid this issue, and
|
||||
// still be able to store a 0 into tib.local_id[i], we trick the
|
||||
// compiler, by using the value in tib.local_id[3], which is always
|
||||
// initialized to 0.
|
||||
//
|
||||
tib.local_id[0] = tib.local_id[3];
|
||||
|
||||
++tib.local_id[1];
|
||||
if (unlikely(tib.local_id[1] >= tib.local_size[1])) {
|
||||
tib.local_id[1] = tib.local_id[3];
|
||||
|
||||
++tib.local_id[2];
|
||||
}
|
||||
}
|
||||
|
||||
// Link the previous workitem to this one.
|
||||
prev->setNext(workItem);
|
||||
// If this is the last workitem, complete the ring.
|
||||
if (tid >= numWorkItems_ - 1) {
|
||||
workItem->setNext(workItem0_);
|
||||
}
|
||||
}
|
||||
|
||||
// Execute thread0
|
||||
|
||||
address workItemStackPtr = workItem->nativeStackPtr();
|
||||
callKernelProtectedReturn(entryPoint, workItemStackPtr);
|
||||
|
||||
// Check if thread0 executed a barrier()
|
||||
if (threadCounter > 1) {
|
||||
workItem = (WorkItem*)workingFiber_;
|
||||
workingFiber_ = workingFiber_->next();
|
||||
|
||||
tid = ((address)workItem0_ - (address)workItem)
|
||||
/ CLK_PRIVATE_MEMORY_SIZE;
|
||||
if (tid == (numWorkItems_ - 1)) {
|
||||
// If we get here, we are done!
|
||||
return;
|
||||
}
|
||||
if (workItem->next() == &thread_.mainFiber()) {
|
||||
// Detected a deadlock
|
||||
command_.setException(CL_INVALID_KERNEL);
|
||||
return;
|
||||
}
|
||||
|
||||
// Schedule the next workitem.
|
||||
workItem->next()->restore();
|
||||
ShouldNotReachHere();
|
||||
}
|
||||
|
||||
// Execute thread1...threadN
|
||||
callKernelRange(entryPoint, workItemStackPtr, workItem->infoBlock());
|
||||
}
|
||||
|
||||
void
|
||||
WorkGroup::executeWithoutBarrier()
|
||||
{
|
||||
kernelentrypoint_t entryPoint = (kernelentrypoint_t)kernel_.getEntryPoint();
|
||||
address workItemStackPtr = workItem0_->nativeStackPtr();
|
||||
|
||||
// Execute thread0
|
||||
callKernel(entryPoint, workItemStackPtr);
|
||||
|
||||
// Execute thread1...threadN
|
||||
callKernelRange(entryPoint, workItemStackPtr, workItem0_->infoBlock());
|
||||
}
|
||||
|
||||
void
|
||||
WorkGroup::callKernelRange(kernelentrypoint_t entryPoint,
|
||||
address stackPtr,
|
||||
clk_thread_info_block_t& tib)
|
||||
{
|
||||
while (true) {
|
||||
++tib.local_id[0];
|
||||
if (unlikely(tib.local_id[0] >= tib.local_size[0])) {
|
||||
tib.local_id[0] = 0;
|
||||
|
||||
++tib.local_id[1];
|
||||
if (unlikely(tib.local_id[1] >= tib.local_size[1])) {
|
||||
tib.local_id[1] = 0;
|
||||
|
||||
++tib.local_id[2];
|
||||
if (unlikely(tib.local_id[2] >= tib.local_size[2])) {
|
||||
tib.local_id[2] = 0;
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
callKernel(entryPoint, stackPtr);
|
||||
}
|
||||
}
|
||||
|
||||
WorkItem::WorkItem(const amd::NDRangeContainer& sizes, void* localMemPtr)
|
||||
{
|
||||
const amd::NDRange& local = sizes.local();
|
||||
const amd::NDRange& global = sizes.global();
|
||||
const amd::NDRange& offset = sizes.offset();
|
||||
const size_t dims = sizes.dimensions();
|
||||
|
||||
tib_.builtins = &Builtins::dispatchTable_;
|
||||
tib_.work_dim = (cl_uint) sizes.dimensions();
|
||||
tib_.local_mem_base = localMemPtr;
|
||||
tib_.table_base = (const void *)cpuTables;
|
||||
for (size_t i = 0; i < dims; ++i) {
|
||||
tib_.global_offset[i] = offset[i];
|
||||
tib_.global_size[i] = global[i];
|
||||
tib_.local_size[i] = local[i];
|
||||
tib_.enqueued_local_size[i] = local[i];
|
||||
tib_.local_id[i] = 0;
|
||||
tib_.group_id[i] = 0;
|
||||
}
|
||||
// Fill the remaining dimensions.
|
||||
for (size_t i = dims; i < sizeof(tib_.global_size)/sizeof(size_t); ++i) {
|
||||
tib_.global_offset[i] = 0;
|
||||
tib_.global_size[i] = 1;
|
||||
tib_.local_size[i] = 1;
|
||||
tib_.enqueued_local_size[i] = 1;
|
||||
tib_.local_id[i] = 0;
|
||||
tib_.group_id[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
ALWAYSINLINE void
|
||||
WorkItem::setGroupId(
|
||||
const amd::NDRange& rangeLimits,
|
||||
const amd::NDRange& offset,
|
||||
size_t n)
|
||||
{
|
||||
const size_t dims = rangeLimits.dimensions();
|
||||
for (size_t i = 0; i < dims; ++i) {
|
||||
size_t lim = rangeLimits[i];
|
||||
size_t& val = tib_.group_id[i];
|
||||
val = n;
|
||||
if (n < lim) {
|
||||
tib_.global_offset[i] =
|
||||
offset[i] + val * tib_.enqueued_local_size[i];
|
||||
tib_.local_id[i] = 0;
|
||||
tib_.local_size[i] =
|
||||
std::min(tib_.enqueued_local_size[i],
|
||||
tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));
|
||||
|
||||
++i;
|
||||
for (; i < dims; ++i) {
|
||||
tib_.global_offset[i] = offset[i];
|
||||
tib_.local_id[i] = 0;
|
||||
tib_.group_id[i] = 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else {
|
||||
n /= lim;
|
||||
val -= n * lim;
|
||||
tib_.global_offset[i] =
|
||||
offset[i] + val * tib_.enqueued_local_size[i];
|
||||
tib_.local_id[i] = 0;
|
||||
tib_.local_size[i] =
|
||||
std::min(tib_.enqueued_local_size[i],
|
||||
tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ALWAYSINLINE void
|
||||
WorkItem::incrementGroupId(
|
||||
const amd::NDRange& rangeLimits,
|
||||
const amd::NDRange& offset,
|
||||
size_t n)
|
||||
{
|
||||
const size_t dims = rangeLimits.dimensions();
|
||||
for (size_t i = 0; i < dims; ++i) {
|
||||
size_t lim = rangeLimits[i];
|
||||
size_t& val = tib_.group_id[i];
|
||||
val += n;
|
||||
if (val < lim) {
|
||||
tib_.global_offset[i] =
|
||||
offset[i] + val * tib_.enqueued_local_size[i];
|
||||
tib_.local_id[i] = 0;
|
||||
tib_.local_size[i] =
|
||||
std::min(tib_.enqueued_local_size[i],
|
||||
tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));
|
||||
break;
|
||||
}
|
||||
else {
|
||||
n = val / lim;
|
||||
val -= n * lim;
|
||||
tib_.global_offset[i] =
|
||||
offset[i] + val * tib_.enqueued_local_size[i];
|
||||
tib_.local_id[i] = 0;
|
||||
tib_.local_size[i] =
|
||||
std::min(tib_.enqueued_local_size[i],
|
||||
tib_.global_size[i] - (val * tib_.enqueued_local_size[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
WorkItem::barrier(cl_mem_fence_flags flags)
|
||||
{
|
||||
WorkItem* workItem = WorkItem::current();
|
||||
workItem->swap(workItem->next());
|
||||
}
|
||||
|
||||
void Operation::cleanup()
|
||||
{
|
||||
cl_int lastException = command().exception();
|
||||
cl_int status = (lastException != 0) ? lastException : CL_COMPLETE;
|
||||
|
||||
Counter* counter = reinterpret_cast<Counter*>(command().data());
|
||||
if (counter == NULL) {
|
||||
command().setStatus(status);
|
||||
}
|
||||
else if (counter->decrement() == 0) {
|
||||
counter->event().setStatus(status);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace cpu
|
||||
@@ -0,0 +1,425 @@
|
||||
//
|
||||
// Copyright 2010 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef OPERATION_HPP_
|
||||
#define OPERATION_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "device/cpu/cpudevice.hpp"
|
||||
#include "device/cpu/cpukernel.hpp"
|
||||
#include "platform/command.hpp"
|
||||
#include "thread/thread.hpp"
|
||||
#include "os/os.hpp"
|
||||
#include "amdocl/cl_kernel.h"
|
||||
#include "device/cpu/ring.hpp"
|
||||
|
||||
#if defined(ATI_ARCH_ARM)
|
||||
#include <setjmp.h>
|
||||
#endif // ATI_ARCH_ARM
|
||||
|
||||
namespace cpu {
|
||||
|
||||
/*! \addtogroup CPU
|
||||
* @{
|
||||
*
|
||||
* \addtogroup CPUExec Execution environment
|
||||
* @{
|
||||
*/
|
||||
|
||||
//! A saved stack context
|
||||
class StackContext : public amd::StackObject
|
||||
{
|
||||
private:
|
||||
#if defined(ATI_ARCH_ARM)
|
||||
jmp_buf env_;
|
||||
#elif defined(_WIN64)
|
||||
intptr_t __declspec(align(16)) regs_[32];
|
||||
#else // !_WIN64
|
||||
intptr_t regs_[LP64_SWITCH(6,8)];
|
||||
#endif // !_WIN64
|
||||
|
||||
public:
|
||||
//! Save the stack context. Return 0 if returning directly.
|
||||
inline intptr_t setjmp();
|
||||
|
||||
//! Restore the stack context
|
||||
inline void longjmp(intptr_t val) const;
|
||||
};
|
||||
|
||||
//! A thread fiber
|
||||
class Fiber : public amd::StackObject
|
||||
{
|
||||
private:
|
||||
//! Next fiber in the thread.
|
||||
Fiber* next_;
|
||||
|
||||
//! This fiber's saved state.
|
||||
StackContext context_;
|
||||
|
||||
public:
|
||||
//! Construct a new Fiber
|
||||
Fiber() : next_(NULL) { }
|
||||
|
||||
//! Return the next fiber in the current thread.
|
||||
const Fiber* next() const { return next_; }
|
||||
//! Set the next fiber in the current thread.
|
||||
void setNext(Fiber* next) { next_ = next; }
|
||||
|
||||
//! Save the state of this fiber. Return true if directly returning.
|
||||
ALWAYSINLINE bool save() { return context_.setjmp() == 0; }
|
||||
//! Restore this fiber from the saved context.
|
||||
void restore() const { context_.longjmp(1); }
|
||||
|
||||
//! Switch to the given fiber.
|
||||
void swap(const Fiber* fiber) { if (save()) { fiber->restore(); } }
|
||||
};
|
||||
|
||||
|
||||
|
||||
//! A CPU core operation (enqueued in the worker thread queue)
|
||||
class Operation : public amd::HeapObject
|
||||
{
|
||||
public:
|
||||
//! An atomic counter
|
||||
class Counter
|
||||
{
|
||||
// FIXME_lmoriche: recycle the counters, implement a thread local pool.
|
||||
private:
|
||||
amd::Event& event_;
|
||||
//! The atomic counter value.
|
||||
amd::Atomic<size_t> counter_;
|
||||
|
||||
public:
|
||||
//! Initialize the counter with the given initial value.
|
||||
Counter(amd::Event& event, size_t initialValue) :
|
||||
event_(event), counter_(initialValue) { }
|
||||
//! Return the event associated with this counter.
|
||||
amd::Event& event() { return event_; }
|
||||
//! Decrement the counter and return the new value.
|
||||
size_t decrement() { return --counter_; }
|
||||
};
|
||||
|
||||
protected:
|
||||
amd::Command& command_;
|
||||
|
||||
public:
|
||||
Operation(amd::Command& command) : command_(command)
|
||||
{ }
|
||||
|
||||
virtual ~Operation() {};
|
||||
|
||||
virtual void clone(Operation* buf) = 0;
|
||||
|
||||
void cleanup();
|
||||
|
||||
amd::Command& command() { return command_;}
|
||||
|
||||
virtual void execute() = 0;
|
||||
};
|
||||
|
||||
/*! @}
|
||||
* \defgroup CPUOperations Operations
|
||||
* @{
|
||||
*/
|
||||
|
||||
//! A work item instance
|
||||
class WorkItem : public Fiber
|
||||
{
|
||||
private:
|
||||
//! Thread info block (must be the last field).
|
||||
clk_thread_info_block_t tib_;
|
||||
|
||||
private:
|
||||
//! Cannot be deleted (allocated with placement new).
|
||||
void operator delete(void*) { ShouldNotCallThis(); }
|
||||
|
||||
public:
|
||||
//! Initialize this workgroup.
|
||||
WorkItem(const amd::NDRangeContainer& size, void* localMemPtr);
|
||||
|
||||
//! Return the current WorkItem (based of the current stack pointer).
|
||||
static WorkItem* current() {
|
||||
return (WorkItem*)amd::alignUp((intptr_t) amd::Os::currentStackPtr(),
|
||||
CLK_PRIVATE_MEMORY_SIZE) - 1;
|
||||
}
|
||||
|
||||
clk_thread_info_block_t& infoBlock() { return tib_; }
|
||||
|
||||
//! Return the native stack pointer base for this workitem.
|
||||
address nativeStackPtr() const {
|
||||
address newSp = amd::alignDown((address) this - CPUKERNEL_STACK_ALIGN,
|
||||
CPUKERNEL_STACK_ALIGN);
|
||||
WINDOWS_ONLY(NOT_WIN64(newSp += sizeof(void*)));
|
||||
return newSp;
|
||||
}
|
||||
|
||||
//! These functions are mapping "n" from 1d index to the required dimension
|
||||
inline void setGroupId(
|
||||
const amd::NDRange& rangeLimits,
|
||||
const amd::NDRange& offset,
|
||||
size_t n);
|
||||
inline void incrementGroupId(
|
||||
const amd::NDRange& rangeLimits,
|
||||
const amd::NDRange& offset,
|
||||
size_t n);
|
||||
|
||||
//! Execute a thread synchronization barrier.
|
||||
static void barrier(cl_mem_fence_flags flags);
|
||||
};
|
||||
|
||||
typedef void (*kernelentrypoint_t)(const void*);
|
||||
|
||||
//! Execute a workgroup (work-items).
|
||||
class WorkGroup
|
||||
{
|
||||
private:
|
||||
amd::NDRangeKernelCommand& command_;
|
||||
const cpu::Kernel& kernel_;
|
||||
WorkerThread& thread_;
|
||||
address params_;
|
||||
WorkItem* const workItem0_;
|
||||
const Fiber* workingFiber_;
|
||||
size_t numWorkItems_;
|
||||
|
||||
public:
|
||||
WorkGroup(
|
||||
amd::NDRangeKernelCommand& parent,
|
||||
const cpu::Kernel& kernel,
|
||||
WorkerThread& thread,
|
||||
address params,
|
||||
WorkItem* workItem0,
|
||||
const size_t numWorkItems) :
|
||||
command_(parent),
|
||||
kernel_(kernel),
|
||||
thread_(thread),
|
||||
params_(params),
|
||||
workItem0_(workItem0),
|
||||
numWorkItems_(numWorkItems)
|
||||
{ }
|
||||
|
||||
WorkItem* getBaseWorkItem() { return workItem0_; }
|
||||
WorkerThread& getWorkerThread() { return thread_; }
|
||||
|
||||
void executeWorkItem(); // In case of 1 WorkItem
|
||||
void executeWithBarrier();
|
||||
void executeWithoutBarrier();
|
||||
|
||||
void setNumWorkItems(size_t workItems) { numWorkItems_ = workItems; }
|
||||
size_t getNumWorkItems() { return numWorkItems_; }
|
||||
private:
|
||||
void callKernelRange(
|
||||
kernelentrypoint_t entryPoint,
|
||||
address stackPtr,
|
||||
clk_thread_info_block_t& tib);
|
||||
inline void callKernel(
|
||||
kernelentrypoint_t entryPoint,
|
||||
address stackPtr);
|
||||
inline void callKernelProtectedReturn(
|
||||
kernelentrypoint_t entryPoint,
|
||||
address stackPtr);
|
||||
};
|
||||
|
||||
class NDRangeKernelBatch : public Operation
|
||||
{
|
||||
protected:
|
||||
size_t coreId_;
|
||||
const size_t numWorkGroups_;
|
||||
const size_t numCores_;
|
||||
volatile size_t currentOpId_;
|
||||
const amd::NDRange groupIds_; //!< Number of groups in each dimensions
|
||||
VirtualCPU& virtualDevice_;
|
||||
|
||||
public:
|
||||
enum ExecutionOrder {
|
||||
ORDER_DEFAULT,
|
||||
ORDER_ROUND_ROBIN = ORDER_DEFAULT,
|
||||
//ORDER_LINEAR
|
||||
};
|
||||
|
||||
enum ExecutionNature {
|
||||
NATURE_WITH_BARRIER,
|
||||
NATURE_WITHOUT_BARRIER,
|
||||
NATURE_1_WORK_ITEM,
|
||||
NATURE_WG_LEVEL_EXEC
|
||||
};
|
||||
|
||||
NDRangeKernelBatch(
|
||||
amd::NDRangeKernelCommand& parent,
|
||||
VirtualCPU& virtualDevice,
|
||||
const amd::NDRange& groupIds, size_t numCores) :
|
||||
Operation(parent),
|
||||
coreId_(0),
|
||||
numWorkGroups_(groupIds.product()),
|
||||
numCores_(numCores),
|
||||
currentOpId_(0),
|
||||
groupIds_(groupIds),
|
||||
virtualDevice_(virtualDevice)
|
||||
{ }
|
||||
|
||||
virtual void clone(Operation* buf)
|
||||
{
|
||||
::new(buf) NDRangeKernelBatch(static_cast<amd::NDRangeKernelCommand&>(command_),
|
||||
virtualDevice_, groupIds_, numCores_);
|
||||
static_cast<NDRangeKernelBatch*>(buf)->setCoreId(coreId_);
|
||||
}
|
||||
|
||||
virtual void execute();
|
||||
|
||||
void setCoreId(size_t coreId) { coreId_ = coreId; currentOpId_ = coreId; }
|
||||
|
||||
inline bool getNextOperationId(size_t& opId);
|
||||
inline size_t getNextOperationIds(size_t& opId, size_t count);
|
||||
|
||||
private:
|
||||
bool patchParameters(
|
||||
const cpu::Kernel& kernel,
|
||||
address params,
|
||||
address& localMemPtr,
|
||||
const address localMemLimit,
|
||||
size_t localMemSize) const;
|
||||
};
|
||||
|
||||
class NativeFn : public Operation
|
||||
{
|
||||
public:
|
||||
NativeFn(amd::NativeFnCommand& parent) : Operation(parent)
|
||||
{ }
|
||||
|
||||
virtual void clone(Operation* buf)
|
||||
{
|
||||
::new(buf) NativeFn(static_cast<amd::NativeFnCommand&>(command_));
|
||||
}
|
||||
|
||||
virtual void execute();
|
||||
};
|
||||
#ifndef MAX
|
||||
#define MAX(x,y) ((x)>=(y) ?(x) : (y))
|
||||
#endif //MAX
|
||||
|
||||
#define MAX_OPERATION_ALLOC_SIZE (MAX(sizeof(NDRangeKernelBatch), sizeof(NativeFn)))
|
||||
|
||||
//! A thread bound to a cpu core.
|
||||
class WorkerThread : public amd::Thread
|
||||
{
|
||||
private:
|
||||
Fiber mainFiber_; //!< main fiber for this worker thread.
|
||||
|
||||
amd::Monitor queueLock_; //!< lock protecting the queue.
|
||||
volatile int waitingOp_;
|
||||
bool terminated_; //!< true if the thread is shutting down.
|
||||
|
||||
//! Local memory storage
|
||||
address localDataStorage_;
|
||||
//! Size of the local memory.
|
||||
size_t localDataSize_;
|
||||
|
||||
char operation_[MAX_OPERATION_ALLOC_SIZE];
|
||||
|
||||
address baseWorkItemsStack_;
|
||||
private:
|
||||
//! Awaits operations and execute them as they become ready.
|
||||
void loop();
|
||||
|
||||
public:
|
||||
//! Construct a new WorkerThread.
|
||||
WorkerThread(const cpu::Device& device);
|
||||
//! Destroy the worker thread.
|
||||
virtual ~WorkerThread();
|
||||
//! Cleanup the thread before termination.
|
||||
bool terminate();
|
||||
|
||||
//! Return the main fiber for this thread.
|
||||
Fiber& mainFiber() { return mainFiber_; }
|
||||
//! Return the LDS for this thread
|
||||
address localDataStorage() const { return localDataStorage_; }
|
||||
//! Return the size of the local memory for this thread.
|
||||
size_t localDataSize() const { return localDataSize_; }
|
||||
|
||||
address baseWorkItemsStack() { return baseWorkItemsStack_; }
|
||||
|
||||
Operation* operation() { return reinterpret_cast<Operation*>(operation_); }
|
||||
bool isOperationValid() { return waitingOp_ > 0; }
|
||||
|
||||
//! Enqueue a new operation to execute in this thread.
|
||||
void enqueue(Operation& op);
|
||||
//! Signal to start processing the commands in the queue.
|
||||
void flush() { amd::ScopedLock sl(queueLock_); queueLock_.notify(); }
|
||||
|
||||
//! This thread's execution engine.
|
||||
void run(void* data) {
|
||||
loop();
|
||||
}
|
||||
|
||||
//! Return the currently executing WorkerThread's instance.
|
||||
static WorkerThread* current()
|
||||
{
|
||||
return static_cast<WorkerThread*>(Thread::current());
|
||||
}
|
||||
};
|
||||
|
||||
/*! @}
|
||||
* @}
|
||||
*/
|
||||
|
||||
extern "C" intptr_t _StackContext_setjmp(intptr_t* regs);
|
||||
|
||||
#if !defined(ATI_ARCH_ARM)
|
||||
ALWAYSINLINE
|
||||
#endif
|
||||
intptr_t
|
||||
StackContext::setjmp()
|
||||
{
|
||||
#if defined(ATI_ARCH_ARM)
|
||||
return ::setjmp(env_);
|
||||
#else
|
||||
return _StackContext_setjmp(regs_);
|
||||
#endif
|
||||
}
|
||||
|
||||
extern "C" void _StackContext_longjmp(const intptr_t* env, intptr_t val);
|
||||
|
||||
ALWAYSINLINE void
|
||||
StackContext::longjmp(intptr_t val) const
|
||||
{
|
||||
#if defined(ATI_ARCH_ARM)
|
||||
return ::longjmp(*const_cast<jmp_buf*>(&env_), val);
|
||||
#else
|
||||
return _StackContext_longjmp(regs_, val);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
|
||||
extern "C" void _WorkGroup_callKernel(
|
||||
address params,
|
||||
kernelentrypoint_t entryPoint,
|
||||
address stackPtr);
|
||||
|
||||
extern "C" void _WorkGroup_callKernelProtectedReturn(
|
||||
address params,
|
||||
kernelentrypoint_t entryPoint,
|
||||
address stackPtr);
|
||||
|
||||
|
||||
ALWAYSINLINE void
|
||||
WorkGroup::callKernel(
|
||||
kernelentrypoint_t entryPoint,
|
||||
address stackPtr)
|
||||
{
|
||||
_WorkGroup_callKernel(params_, entryPoint, stackPtr);
|
||||
}
|
||||
|
||||
// This version support the case of changing the stack for fibers.
|
||||
ALWAYSINLINE void
|
||||
WorkGroup::callKernelProtectedReturn(
|
||||
kernelentrypoint_t entryPoint,
|
||||
address stackPtr)
|
||||
{
|
||||
_WorkGroup_callKernelProtectedReturn(params_, entryPoint, stackPtr);
|
||||
}
|
||||
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif /*OPERATION_HPP_*/
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,246 @@
|
||||
//
|
||||
// Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef CPUDEVICE_HPP_
|
||||
#define CPUDEVICE_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "device/device.hpp"
|
||||
#include "device/cpu/cpuvirtual.hpp"
|
||||
#include "device/cpu/cpusettings.hpp"
|
||||
#include "os/os.hpp"
|
||||
|
||||
#if defined(__linux__) && defined(NUMA_SUPPORT)
|
||||
#include <numa.h>
|
||||
#endif
|
||||
|
||||
#include "acl.h"
|
||||
|
||||
//! \namespace cpu CPU Device Implementation
|
||||
namespace cpu {
|
||||
|
||||
//! Maximum number of the supported samplers
|
||||
const static uint32_t MaxSamplers = 16;
|
||||
//! Maximum number of supported read images
|
||||
const static uint32_t MaxReadImage = 128;
|
||||
//! Maximum number of supported write images
|
||||
const static uint32_t MaxWriteImage = 64;
|
||||
//! Maximum number of supported read/write images
|
||||
const static uint32_t MaxReadWriteImage = 64;
|
||||
|
||||
/*! \addtogroup CPU CPU Device Implementation
|
||||
* @{
|
||||
*
|
||||
* \addtogroup CPUDevice Device
|
||||
*
|
||||
* \copydoc cpu::Device
|
||||
*
|
||||
* @{
|
||||
*/
|
||||
|
||||
//! A CPU device ordinal
|
||||
class Device : public amd::Device
|
||||
{
|
||||
protected:
|
||||
static aclCompiler* compiler_;
|
||||
public:
|
||||
aclCompiler* compiler() const { return compiler_; }
|
||||
|
||||
public:
|
||||
static bool init(void);
|
||||
|
||||
//! Shutdown CPU device
|
||||
static void tearDown();
|
||||
|
||||
//! Construct a new identifier
|
||||
Device(Device* parent = NULL) :
|
||||
amd::Device(parent),
|
||||
workerThreadsAffinity_(NULL)
|
||||
{}
|
||||
|
||||
virtual ~Device();
|
||||
|
||||
bool create();
|
||||
|
||||
virtual cl_int createSubDevices(
|
||||
device::CreateSubDevicesInfo& create_info,
|
||||
cl_uint num_entries,
|
||||
cl_device_id* devices,
|
||||
cl_uint* num_devices);
|
||||
|
||||
//! Instantiate a new virtual device
|
||||
virtual device::VirtualDevice* createVirtualDevice(
|
||||
bool profiling,
|
||||
bool interopQueue
|
||||
#if cl_amd_open_video
|
||||
, void* calVideoProperties = NULL
|
||||
#endif // cl_amd_open_video
|
||||
, uint deviceQueueSize = 0
|
||||
)
|
||||
{
|
||||
VirtualCPU* virtualCpu = new VirtualCPU(*this);
|
||||
if (virtualCpu != NULL && !virtualCpu->acceptingCommands()) {
|
||||
virtualCpu->terminate();
|
||||
delete virtualCpu;
|
||||
virtualCpu = NULL;
|
||||
}
|
||||
return virtualCpu;
|
||||
}
|
||||
|
||||
//! Compile the given source code.
|
||||
virtual device::Program* createProgram(int oclVer = 120);
|
||||
|
||||
//! Just returns NULL as CPU devices use the host memory
|
||||
virtual device::Memory* createMemory(amd::Memory& owner) const
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
//! Sampler object allocation
|
||||
virtual bool createSampler(
|
||||
const amd::Sampler& owner, //!< abstraction layer sampler object
|
||||
device::Sampler** sampler //!< device sampler object
|
||||
) const
|
||||
{
|
||||
// Just return NULL on CPU device
|
||||
*sampler = NULL;
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Reallocates device memory obje
|
||||
virtual bool reallocMemory(amd::Memory& owner) const
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Just returns NULL as CPU devices use the host memory
|
||||
virtual device::Memory* createView(
|
||||
amd::Memory& owner, //!< Owner memory object
|
||||
const device::Memory& parent //!< Parent device memory object for the view
|
||||
) const
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
//! Acquire external graphics API object in the host thread
|
||||
//! Needed for OpenGL objects on CPU device
|
||||
|
||||
//! Return true if initialized interoperability, otherwise false
|
||||
virtual bool bindExternalDevice(intptr_t type, void* pDevice, void* pContext, bool validateOnly)
|
||||
{
|
||||
return true; // On CPU always avail if pD3DDevice is not NULL
|
||||
}
|
||||
|
||||
virtual bool unbindExternalDevice(intptr_t type, void* pDevice, void* pContext, bool validateOnly)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Gets a pointer to a region of host-visible memory for use as the target
|
||||
//! of a non-blocking map for a given memory object
|
||||
virtual void* allocMapTarget(
|
||||
amd::Memory& mem, //!< Abstraction layer memory object
|
||||
const amd::Coord3D& origin, //!< The map location in memory
|
||||
const amd::Coord3D& region, //!< The map region in memory
|
||||
size_t* rowPitch = NULL, //!< Row pitch for the mapped memory
|
||||
size_t* slicePitch = NULL //!< Slice for the mapped memory
|
||||
);
|
||||
|
||||
//! Releases non-blocking map target memory
|
||||
virtual void freeMapTarget(amd::Memory& mem, void* target);
|
||||
|
||||
//! Empty implementation on a CPU device
|
||||
virtual bool globalFreeMemory(size_t* freeMemory) const { return false; }
|
||||
|
||||
//! Get CPU device settings
|
||||
const cpu::Settings& settings() const
|
||||
{ return reinterpret_cast<cpu::Settings&>(*settings_); }
|
||||
|
||||
bool hasAVXInstructions() const
|
||||
{ return (settings().cpuFeatures_ & Settings::AVXInstructions) ? true : false; }
|
||||
|
||||
bool hasFMA4Instructions() const
|
||||
{ return (settings().cpuFeatures_ & Settings::FMA4Instructions) ? true : false; }
|
||||
|
||||
static size_t getMaxWorkerThreadsNumber() { return maxWorkerThreads_; }
|
||||
|
||||
void setWorkerThreadsAffinity(
|
||||
cl_uint numWorkerThreads,
|
||||
const amd::Os::ThreadAffinityMask* threadsAffinityMask,
|
||||
uint& baseCoreId);
|
||||
|
||||
const amd::Os::ThreadAffinityMask* getWorkerThreadsAffinity() const
|
||||
{
|
||||
return workerThreadsAffinity_;
|
||||
}
|
||||
//! host memory alloc
|
||||
virtual void* svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_svm_mem_flags flags) const
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
//! host memory deallocation
|
||||
virtual void svmFree(void* ptr) const
|
||||
{
|
||||
return;
|
||||
}
|
||||
private:
|
||||
bool initSubDevice(
|
||||
device::Info& info,
|
||||
cl_uint maxComputeUnits,
|
||||
const device::CreateSubDevicesInfo& create_info);
|
||||
|
||||
cl_int partitionEqually(
|
||||
const device::CreateSubDevicesInfo& create_info,
|
||||
cl_uint num_entries,
|
||||
cl_device_id* devices,
|
||||
cl_uint* num_devices);
|
||||
|
||||
cl_int partitionByCounts(
|
||||
const device::CreateSubDevicesInfo& create_info,
|
||||
cl_uint num_entries,
|
||||
cl_device_id* devices,
|
||||
cl_uint* num_devices);
|
||||
|
||||
cl_int partitionByAffinityDomainNUMA(
|
||||
const device::CreateSubDevicesInfo& create_info,
|
||||
cl_uint num_entries,
|
||||
cl_device_id* devices,
|
||||
cl_uint* num_devices);
|
||||
|
||||
cl_int partitionByAffinityDomainCacheLevel(
|
||||
const device::CreateSubDevicesInfo& create_info,
|
||||
cl_uint num_entries,
|
||||
cl_device_id* devices,
|
||||
cl_uint* num_devices);
|
||||
|
||||
private:
|
||||
#if defined(__linux__) && defined(NUMA_SUPPORT)
|
||||
public:
|
||||
const nodemask_t* getNumaMask() const
|
||||
{
|
||||
return (info_.partitionCreateInfo_.type_ == device::PartitionType::BY_AFFINITY_DOMAIN &&
|
||||
info_.partitionCreateInfo_.byAffinityDomain_.numa_) ?
|
||||
numaMask_ : NULL;
|
||||
}
|
||||
|
||||
private:
|
||||
union {
|
||||
nodemask_t* numaMask_;
|
||||
amd::Os::ThreadAffinityMask* workerThreadsAffinity_; //!< As the number of compute units.
|
||||
};
|
||||
#else
|
||||
amd::Os::ThreadAffinityMask* workerThreadsAffinity_; //!< As the number of compute units.
|
||||
#endif
|
||||
|
||||
static size_t maxWorkerThreads_; //!< Maximum number of Worker Threads
|
||||
};
|
||||
|
||||
/*! @}
|
||||
* @}
|
||||
*/
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif // CPUDEVICE_HPP_
|
||||
@@ -0,0 +1,27 @@
|
||||
//
|
||||
// Copyright 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef CPUFEAT_HPP
|
||||
#define CPUFEAT_HPP
|
||||
|
||||
#define CPUFEAT_CX_SSE3 (1 << 0)
|
||||
#define CPUFEAT_CX_SSSE3 (1 << 9)
|
||||
#define CPUFEAT_CX_CMPXCHG16B (1 << 13)
|
||||
#define CPUFEAT_CX_SSE4_1 (1 << 19)
|
||||
#define CPUFEAT_CX_SSE4_2 (1 << 20)
|
||||
#define CPUFEAT_CX_POPCNT (1 << 23)
|
||||
#define CPUFEAT_CX_AES (1 << 25)
|
||||
#define CPUFEAT_CX_OSXSAVE (1 << 27)
|
||||
#define CPUFEAT_CX_AVX (1 << 28)
|
||||
|
||||
#define INTEL_CPUFEAT_CX_FMA3 (1 << 12)
|
||||
|
||||
#define AMD_CPUFEAT_CX_FMA4 (1 << 16)
|
||||
#define AMD_CPUFEAT_CX_XOP (1 << 11)
|
||||
#define AMD_CPUFEAT_CX_SSE4A (1 << 6)
|
||||
|
||||
#define CPUFEAT_DX_SSE (1 < 25)
|
||||
#define CPUFEAT_DX_SSE2 (1 << 26)
|
||||
|
||||
#endif // CPUFEAT_HPP
|
||||
@@ -0,0 +1,87 @@
|
||||
#
|
||||
# Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
|
||||
.text
|
||||
.globl _WorkGroup_callKernel
|
||||
#if defined(ATI_ARCH_X86)
|
||||
.type _WorkGroup_callKernel, @function
|
||||
_WorkGroup_callKernel:
|
||||
#if defined(_LP64)
|
||||
pushq %rbp
|
||||
movq %rsp, %rbp
|
||||
movq %rdx, %rsp // stackPtr
|
||||
call *%rsi
|
||||
movq %rbp, %rsp
|
||||
popq %rbp
|
||||
#else // _LP64
|
||||
pushl %ebp
|
||||
movl %esp, %ebp
|
||||
movl 0x10(%ebp), %esp // stackPtr
|
||||
movl 0x0C(%ebp), %edx // entryPoint
|
||||
movl 0x08(%ebp), %ecx // params
|
||||
movl %ecx, (%esp)
|
||||
call *%edx
|
||||
movl %ebp, %esp
|
||||
popl %ebp
|
||||
#endif // _LP64
|
||||
ret
|
||||
#elif defined(ATI_ARCH_ARM)
|
||||
.type _WorkGroup_callKernel, %function
|
||||
_WorkGroup_callKernel:
|
||||
bx lr
|
||||
#endif
|
||||
|
||||
.globl _WorkGroup_callKernelProtectedReturn
|
||||
#if defined(ATI_ARCH_X86)
|
||||
.type _WorkGroup_callKernelProtectedReturn, @function
|
||||
_WorkGroup_callKernelProtectedReturn:
|
||||
#if defined(_LP64)
|
||||
movq %rbp, %rax
|
||||
movq %rsp, %rbp
|
||||
movq %rdx, %rsp // stackPtr
|
||||
subq $CPUKERNEL_STACK_ALIGN, %rsp
|
||||
movq %rax, 0x08(%rsp) // save rbp
|
||||
movq %rbx, 0x00(%rsp) // save rbx
|
||||
movq (%rbp), %rbx // return address
|
||||
|
||||
call *%rsi
|
||||
|
||||
movq %rbx, %rdx
|
||||
movq %rbp, %rcx
|
||||
movq 0x00(%rsp), %rbx // load rbx
|
||||
movq 0x08(%rsp), %rbp // load rbp
|
||||
movq %rcx, %rsp
|
||||
addq $0x08, %rsp // skip return address
|
||||
jmp *%rdx
|
||||
#else // !_LP64
|
||||
movl %ebp, %eax
|
||||
movl %esp, %ebp
|
||||
movl 0x0C(%ebp), %esp // stackPtr
|
||||
subl $CPUKERNEL_STACK_ALIGN, %esp
|
||||
movl 0x04(%ebp), %ecx // params
|
||||
movl %eax, 0x08(%esp) // save ebp
|
||||
movl %ebx, 0x04(%esp) // save ebx
|
||||
movl %ecx, 0x00(%esp) // pass params
|
||||
movl 0x00(%ebp), %ebx // return address
|
||||
movl 0x08(%ebp), %edx // entryPoint
|
||||
|
||||
call *%edx
|
||||
|
||||
movl %ebx, %edx
|
||||
movl %ebp, %ecx
|
||||
movl 0x04(%esp), %ebx // load ebx
|
||||
movl 0x08(%esp), %ebp // load ebp
|
||||
movl %ecx, %esp
|
||||
addl $0x4, %esp // skip return address
|
||||
jmp *%edx
|
||||
#endif // !_LP64
|
||||
#elif defined(ATI_ARCH_ARM)
|
||||
.type _WorkGroup_callKernelProtectedReturn, %function
|
||||
_WorkGroup_callKernelProtectedReturn:
|
||||
bx lr
|
||||
#endif
|
||||
|
||||
|
||||
.section .note.GNU-stack,"",%progbits
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
;
|
||||
; Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
;
|
||||
|
||||
ifndef _WIN64
|
||||
.386
|
||||
.model flat, c
|
||||
endif ; !_WIN64
|
||||
|
||||
OPTION PROLOGUE:NONE
|
||||
OPTION EPILOGUE:NONE
|
||||
.code
|
||||
|
||||
ifndef _WIN64
|
||||
|
||||
_WorkGroup_callKernel proc
|
||||
push ebp
|
||||
mov ebp, esp
|
||||
mov esp, 10h[ebp] ; stackPtr
|
||||
mov edx, 0Ch[ebp] ; entryPoint
|
||||
push 08h[ebp] ; params
|
||||
call edx
|
||||
mov esp, ebp
|
||||
pop ebp
|
||||
ret
|
||||
_WorkGroup_callKernel endp
|
||||
|
||||
_WorkGroup_callKernelProtectedReturn proc
|
||||
mov eax, ebp
|
||||
mov ebp, esp
|
||||
mov esp, 0Ch[ebp] ; stackPtr
|
||||
sub esp, CPUKERNEL_STACK_ALIGN
|
||||
mov 04h[esp], eax ; save ebp
|
||||
mov 00h[esp], ebx ; save ebx
|
||||
mov ebx, 00h[ebp] ; return address
|
||||
mov edx, 08h[ebp] ; entryPoint
|
||||
|
||||
push 04h[ebp] ; params
|
||||
call edx
|
||||
|
||||
mov edx, ebx
|
||||
mov ecx, ebp
|
||||
mov ebx, 04h[esp] ; load ebx
|
||||
mov ebp, 08h[esp] ; load ebp
|
||||
mov esp, ecx
|
||||
add esp, 04h ; skip return address
|
||||
jmp edx
|
||||
_WorkGroup_callKernelProtectedReturn endp
|
||||
|
||||
else ; _WIN64
|
||||
|
||||
_WorkGroup_callKernel proc
|
||||
push rbp
|
||||
mov rbp, rsp
|
||||
mov rsp, r8 ; stackPtr
|
||||
call rdx
|
||||
mov rsp, rbp
|
||||
pop rbp
|
||||
ret
|
||||
_WorkGroup_callKernel endp
|
||||
|
||||
_WorkGroup_callKernelProtectedReturn proc
|
||||
mov rax, rbp
|
||||
mov rbp, rsp
|
||||
mov rsp, r8 ; stackPtr
|
||||
sub rsp, CPUKERNEL_STACK_ALIGN
|
||||
mov 08h[rsp], rax ; save rbp
|
||||
mov 00h[rsp], rbx ; save rbx
|
||||
mov rbx, [rbp] ; return address
|
||||
|
||||
call rdx
|
||||
|
||||
mov rdx, rbx
|
||||
mov rcx, rbp
|
||||
mov rbx, 00h[rsp] ; load rbx
|
||||
mov rbp, 08h[rsp] ; load rbp
|
||||
mov rsp, rcx
|
||||
add rsp, 08h ; skip return address
|
||||
jmp rdx
|
||||
_WorkGroup_callKernelProtectedReturn endp
|
||||
|
||||
endif ; _WIN64
|
||||
|
||||
end
|
||||
@@ -0,0 +1,71 @@
|
||||
//
|
||||
// Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef CPUKERNEL_HPP_
|
||||
#define CPUKERNEL_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "device/device.hpp"
|
||||
#include <amdocl/cl_kernel.h>
|
||||
|
||||
//! \namespace cpu CPU Device Implementation
|
||||
namespace cpu {
|
||||
|
||||
//! \class CPU kernel
|
||||
class Kernel : public device::Kernel
|
||||
{
|
||||
private:
|
||||
const void* entryPoint_; //!< entry for the kernel
|
||||
|
||||
std::vector< std::pair<size_t, size_t> > args_;
|
||||
public:
|
||||
uint nature_; //!< kernel's nature
|
||||
uint privateSize_; //!< WorkItem's private memory size (in bytes)
|
||||
|
||||
private:
|
||||
//! Disable default copy constructor
|
||||
Kernel(const Kernel&);
|
||||
//! Disable operator=
|
||||
Kernel& operator=(const Kernel&);
|
||||
|
||||
public:
|
||||
void addArg(size_t size, size_t alignment) {
|
||||
args_.push_back(std::pair<size_t, size_t>(size, alignment));
|
||||
}
|
||||
|
||||
size_t getArgSize(int argIndex) const {
|
||||
return args_[argIndex].first;
|
||||
}
|
||||
|
||||
size_t getArgAlignment(int argIndex) const {
|
||||
return args_[argIndex].second;
|
||||
}
|
||||
|
||||
//! Default constructor
|
||||
Kernel(const std::string& name)
|
||||
: device::Kernel(name), entryPoint_(NULL), nature_(0),
|
||||
privateSize_(CLK_PRIVATE_MEMORY_SIZE)
|
||||
{
|
||||
workGroupInfo_.size_ = CPU_MAX_WORKGROUP_SIZE;
|
||||
}
|
||||
|
||||
//! Default destructor
|
||||
~Kernel() {}
|
||||
|
||||
//! Returns the CPU kernel entry point
|
||||
const void* getEntryPoint() const { return entryPoint_; }
|
||||
|
||||
//! Sets the CPU kernel entry point
|
||||
void setEntryPoint(const void* entryPoint) { entryPoint_ = entryPoint; }
|
||||
|
||||
//! Returns true if the kernel has a call to barrier
|
||||
bool hasBarrier() const { return 0 != (nature_ & KN_HAS_BARRIER); }
|
||||
|
||||
//! Returns the private memory size of a single WorkItem
|
||||
uint getWorkItemPrivateMemSize() const { return privateSize_; }
|
||||
};
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif // CPUKERNEL_HPP_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,118 @@
|
||||
//
|
||||
// Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef CPUPROGRAM_HPP_
|
||||
#define CPUPROGRAM_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "device/device.hpp"
|
||||
#include "device/cpu/cpubinary.hpp"
|
||||
#include <string>
|
||||
|
||||
#include "jit.h"
|
||||
|
||||
// forward declaration
|
||||
namespace amd {
|
||||
namespace option {
|
||||
class Options;
|
||||
} // option
|
||||
} // amd
|
||||
|
||||
//! \namespace cpu CPU Device Implementation
|
||||
namespace cpu {
|
||||
|
||||
//! \class CPU program
|
||||
class Program : public device::Program
|
||||
{
|
||||
private:
|
||||
aclJITObjectImage JITBinary;
|
||||
std::string sourceFileName_; //!< The source image.
|
||||
void* handle_; // @todo: remove me
|
||||
|
||||
public:
|
||||
//! Default constructor
|
||||
Program(Device& cpuDev)
|
||||
: device::Program(cpuDev), JITBinary(NULL), handle_(NULL) {}
|
||||
|
||||
//! Default destructor
|
||||
~Program();
|
||||
|
||||
//! pre-compile setup for CPU
|
||||
virtual bool initBuild(amd::option::Options* options);
|
||||
|
||||
//! post-compile setup for CPU
|
||||
virtual bool finiBuild(bool isBuildGood);
|
||||
|
||||
//! Compiles CPU program
|
||||
virtual bool compileImpl(
|
||||
const std::string& sourceCode,
|
||||
const std::vector<const std::string*>& headers,
|
||||
const char** headerIncludeNames,
|
||||
amd::option::Options* options );
|
||||
|
||||
//! Links CPU program
|
||||
virtual bool linkImpl(amd::option::Options* options = NULL);
|
||||
|
||||
//! Links CPU programs
|
||||
virtual bool linkImpl(
|
||||
const std::vector<device::Program*>& inputPrograms,
|
||||
amd::option::Options* options = NULL,
|
||||
bool createLibrary = false);
|
||||
|
||||
virtual bool createBinary(amd::option::Options* options);
|
||||
|
||||
//! Returns the device object, associated with this program.
|
||||
const Device& device() {
|
||||
return static_cast<const Device&>(device::Program::device());
|
||||
}
|
||||
|
||||
/*! \brief Invokes the LLC compiler for the LLVM binary compilation
|
||||
* to x86 ASM text source code and ISA binary
|
||||
*
|
||||
* \return True if we successefully compiled a CPU program
|
||||
*/
|
||||
bool compileBinaryToISA(
|
||||
amd::option::Options* options //!< options for compilation
|
||||
);
|
||||
|
||||
//! Load the library into memory
|
||||
bool loadDllCode(amd::option::Options* options, bool addElfSymbols=false);
|
||||
|
||||
//! Initialize binary for CPU
|
||||
virtual bool initClBinary();
|
||||
|
||||
//! Release binary for CPU
|
||||
virtual void releaseClBinary();
|
||||
|
||||
ClBinary* clBinary() {
|
||||
return static_cast<ClBinary*>(device::Program::clBinary());
|
||||
}
|
||||
const ClBinary* clBinary() const {
|
||||
return static_cast<const ClBinary*>(device::Program::clBinary());
|
||||
}
|
||||
|
||||
aclJITObjectImage getJITBinary() { return this->JITBinary; }
|
||||
void setJITBinary(aclJITObjectImage JITBinary) { this->JITBinary = JITBinary; }
|
||||
|
||||
private:
|
||||
aclCompiler* compiler() { return static_cast<const Device&>(device()).compiler(); }
|
||||
|
||||
//! Disable default copy constructor
|
||||
Program(const Program&);
|
||||
|
||||
//! Disable operator=
|
||||
Program& operator=(const Program&);
|
||||
|
||||
std::string dllFileName_; //!< File name of the dll with kernels
|
||||
protected:
|
||||
virtual bool isElf(const char* bin) const {
|
||||
return amd::isElfHeader(bin, LP64_SWITCH(ELFCLASS32, ELFCLASS64));
|
||||
}
|
||||
|
||||
virtual const aclTargetInfo & info(const char * str = "");
|
||||
};
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif // CPUPROGRAM_HPP_
|
||||
@@ -0,0 +1,104 @@
|
||||
//
|
||||
// Copyright 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#include "device/cpu/cpusettings.hpp"
|
||||
#include "os/os.hpp"
|
||||
|
||||
namespace cpu {
|
||||
|
||||
bool
|
||||
Settings::create()
|
||||
{
|
||||
largeHostMemAlloc_ = true;
|
||||
|
||||
// This code is temporary until cl_khr_fp64 is unconditional
|
||||
if (flagIsDefault(CL_KHR_FP64) || CL_KHR_FP64) {
|
||||
enableExtension(ClKhrFp64);
|
||||
}
|
||||
|
||||
enableExtension(ClAmdFp64);
|
||||
enableExtension(ClKhrGlobalInt32BaseAtomics);
|
||||
enableExtension(ClKhrGlobalInt32ExtendedAtomics);
|
||||
enableExtension(ClKhrLocalInt32BaseAtomics);
|
||||
enableExtension(ClKhrLocalInt32ExtendedAtomics);
|
||||
|
||||
#ifdef _LP64
|
||||
enableExtension(ClKhrInt64BaseAtomics);
|
||||
enableExtension(ClKhrInt64ExtendedAtomics);
|
||||
#endif // _LP64
|
||||
enableExtension(ClKhrByteAddressableStore);
|
||||
enableExtension(ClKhrGlSharing);
|
||||
enableExtension(ClKhrGlEvent);
|
||||
enableExtension(ClExtDeviceFission);
|
||||
enableExtension(ClAmdDeviceAttributeQuery);
|
||||
enableExtension(ClAmdVec3);
|
||||
enableExtension(ClAmdMediaOps);
|
||||
enableExtension(ClAmdMediaOps2);
|
||||
enableExtension(ClAmdPopcnt);
|
||||
enableExtension(ClAmdPrintf);
|
||||
|
||||
// enableExtension(ClKhrSelectFpRoundingMode);
|
||||
enableExtension(ClKhr3DImageWrites);
|
||||
|
||||
// enableExtension(ClKhrFp16);
|
||||
|
||||
#if defined(_WIN32)
|
||||
enableExtension(ClKhrD3d10Sharing);
|
||||
#endif // _WIN32
|
||||
enableExtension(ClKhrSpir);
|
||||
|
||||
// Enable some OpenCL 2.0 extensions
|
||||
if (OPENCL_MAJOR >= 2) {
|
||||
partialDispatch_ = true;
|
||||
enableExtension(ClKhrSubGroups);
|
||||
supportDepthsRGB_ = true;
|
||||
}
|
||||
|
||||
// Map CPUID feature bits to our own feature bits
|
||||
const int sse2_features = CPUFEAT_DX_SSE | CPUFEAT_DX_SSE2;
|
||||
const int avx_features = CPUFEAT_CX_SSE3 | CPUFEAT_CX_SSSE3 |
|
||||
CPUFEAT_CX_SSE4_1 | CPUFEAT_CX_SSE4_2 |
|
||||
CPUFEAT_CX_POPCNT | CPUFEAT_CX_AVX |
|
||||
CPUFEAT_CX_OSXSAVE;
|
||||
const int fma3_features = INTEL_CPUFEAT_CX_FMA3;
|
||||
const int fma4_features = AMD_CPUFEAT_CX_FMA4 | AMD_CPUFEAT_CX_XOP;
|
||||
int regs[4];
|
||||
|
||||
#if defined(ATI_ARCH_X86)
|
||||
amd::Os::cpuid(regs, 0x0);
|
||||
bool isAmd = regs[1] == ('A' | ('u' << 8) | ('t' << 16) | ('h' << 24));
|
||||
bool isIntel = regs[1] == ('G' | ('e' << 8) | ('n' << 16) | ('u' << 24));
|
||||
|
||||
amd::Os::cpuid(regs, 0x1);
|
||||
|
||||
cpuFeatures_ = (regs[3] & sse2_features) == sse2_features ?
|
||||
SSE2Instructions : 0;
|
||||
|
||||
if ((regs[2] & avx_features) == avx_features) {
|
||||
// Check for state support
|
||||
uint64_t xcr0 = amd::Os::xgetbv(0);
|
||||
|
||||
// Check for SSE and YMM bits (1 and 2)
|
||||
if (((uint32_t)xcr0 & 0x6U) == 0x6U) {
|
||||
cpuFeatures_ |= AVXInstructions;
|
||||
|
||||
// Now check for FMA and XOP
|
||||
if (isIntel) {
|
||||
cpuFeatures_ |= (regs[2] & fma3_features) == fma3_features ?
|
||||
FMA3Instructions : 0;
|
||||
}
|
||||
|
||||
if (isAmd) {
|
||||
amd::Os::cpuid(regs, 0x80000001);
|
||||
cpuFeatures_ |= (regs[2] & fma4_features) == fma4_features ?
|
||||
FMA4Instructions : 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // ATI_ARCH_X86
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace cpu
|
||||
@@ -0,0 +1,44 @@
|
||||
//
|
||||
// Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef CPUSETTINGS_HPP_
|
||||
#define CPUSETTINGS_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "device/device.hpp"
|
||||
#include "device/cpu/cpufeat.hpp"
|
||||
|
||||
//! \namespace cpu CPU Device Implementation
|
||||
namespace cpu {
|
||||
|
||||
//! Device settings
|
||||
class Settings : public device::Settings
|
||||
{
|
||||
public:
|
||||
enum CpuFeatures {
|
||||
SSE2Instructions = 0x01,
|
||||
AVXInstructions = 0x02, // Processor reports SSSE3, SSE4_1, SSE4_2
|
||||
// POPCNT and AVX
|
||||
FMA3Instructions = 0x04, // Intel processor reports FMA3
|
||||
FMA4Instructions = 0x08 // AMD processor reports FMA4 and XOP
|
||||
};
|
||||
uint32_t cpuFeatures_; //!< CPU features
|
||||
|
||||
//! Default constructor
|
||||
Settings() { cpuFeatures_ = 0; }
|
||||
|
||||
//! Creates settings
|
||||
bool create();
|
||||
|
||||
private:
|
||||
//! Disable copy constructor
|
||||
Settings(const Settings&);
|
||||
|
||||
//! Disable assignment
|
||||
Settings& operator=(const Settings&);
|
||||
};
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif // CPUSETTINGS_HPP_
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,627 @@
|
||||
//
|
||||
// Copyright 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#include "device/cpu/cpuvirtual.hpp"
|
||||
#include "device/cpu/cpudevice.hpp"
|
||||
#include "device/cpu/cpucommand.hpp"
|
||||
#include "device/blit.hpp"
|
||||
#include "platform/command.hpp"
|
||||
#include "platform/commandqueue.hpp"
|
||||
#include "platform/memory.hpp"
|
||||
#include "platform/sampler.hpp"
|
||||
#include "os/os.hpp"
|
||||
|
||||
namespace cpu {
|
||||
|
||||
amd::Atomic<size_t> VirtualCPU::numWorkerThreads_(0);
|
||||
|
||||
VirtualCPU::VirtualCPU(Device& device)
|
||||
: device::VirtualDevice(device), acceptingCommands_(false)
|
||||
{
|
||||
const size_t numCores = device.info().maxComputeUnits_;
|
||||
|
||||
if ((numWorkerThreads_ += numCores) >= Device::getMaxWorkerThreadsNumber()) {
|
||||
numWorkerThreads_ -= numCores;
|
||||
cores_ = NULL;
|
||||
return;
|
||||
}
|
||||
|
||||
cores_ = new(std::nothrow) WorkerThread*[numCores];
|
||||
if (cores_ == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Clear memory for the worker threads
|
||||
memset(cores_, 0, numCores * sizeof(WorkerThread*));
|
||||
|
||||
#if defined(__linux__)
|
||||
const bool isNuma =
|
||||
#if defined(NUMA_SUPPORT)
|
||||
device.getNumaMask() == NULL;
|
||||
#else
|
||||
false;
|
||||
#endif // NUMA_SUPPORT
|
||||
const amd::Os::ThreadAffinityMask* affinityMask = isNuma ? NULL :
|
||||
#else
|
||||
const amd::Os::ThreadAffinityMask* affinityMask =
|
||||
#endif
|
||||
device.getWorkerThreadsAffinity();
|
||||
|
||||
uint coreId = affinityMask != NULL ? affinityMask->getFirstSet() : (uint)-1;
|
||||
|
||||
for (size_t i = 0; i < numCores; ++i) {
|
||||
WorkerThread* thread = cores_[i] = new WorkerThread(device);
|
||||
if (thread == NULL) {
|
||||
for (size_t j = 0; j < i; ++j) {
|
||||
cores_[j]->resume();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (thread->state() != amd::Thread::INITIALIZED) {
|
||||
return;
|
||||
}
|
||||
|
||||
#if defined(__linux__)
|
||||
if (!isNuma) {
|
||||
if (coreId == (uint)-1) {
|
||||
thread->setAffinity((uint) i);
|
||||
}
|
||||
else {
|
||||
thread->setAffinity(coreId);
|
||||
coreId = affinityMask->getNextSet(coreId);
|
||||
}
|
||||
}
|
||||
#else // On Windows we set an affinity mask and not a specific ID.
|
||||
if (coreId != (uint)-1) {
|
||||
thread->setAffinity(*affinityMask);
|
||||
}
|
||||
#endif
|
||||
thread->start();
|
||||
}
|
||||
|
||||
blitMgr_ = new device::HostBlitManager(*this);
|
||||
if ((NULL == blitMgr_) || !blitMgr_->create(device)) {
|
||||
LogError("Could not create BlitManager!");
|
||||
return;
|
||||
}
|
||||
|
||||
acceptingCommands_ = true;
|
||||
}
|
||||
|
||||
VirtualCPU::~VirtualCPU()
|
||||
{
|
||||
if (cores_ == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
delete blitMgr_;
|
||||
|
||||
const size_t numCores = device().info().maxComputeUnits_;
|
||||
for (size_t i = 0; i < numCores; ++i) {
|
||||
delete cores_[i];
|
||||
}
|
||||
numWorkerThreads_ -= numCores;
|
||||
delete[] cores_;
|
||||
}
|
||||
|
||||
bool
|
||||
VirtualCPU::terminate()
|
||||
{
|
||||
if (cores_ == NULL) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const size_t numCores = device().info().maxComputeUnits_;
|
||||
for (size_t i = 0; i < numCores; ++i) {
|
||||
if (cores_[i]) {
|
||||
cores_[i]->terminate();
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitReadMemory(amd::ReadMemoryCommand& vcmd)
|
||||
{
|
||||
vcmd.setStatus(CL_RUNNING);
|
||||
|
||||
bool result = false;
|
||||
device::Memory memory(vcmd.source());
|
||||
|
||||
// Ensure memory up-to-date
|
||||
vcmd.source().cacheWriteBack();
|
||||
|
||||
switch (vcmd.type()) {
|
||||
case CL_COMMAND_READ_BUFFER:
|
||||
result = blitMgr().readBuffer(memory, vcmd.destination(),
|
||||
vcmd.origin(), vcmd.size(), vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_READ_BUFFER_RECT:
|
||||
result = blitMgr().readBufferRect(memory,
|
||||
vcmd.destination(), vcmd.bufRect(), vcmd.hostRect(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_READ_IMAGE:
|
||||
result = blitMgr().readImage(memory, vcmd.destination(),
|
||||
vcmd.origin(), vcmd.size(), vcmd.rowPitch(), vcmd.slicePitch(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
default:
|
||||
LogError("Unsupported type for the read command");
|
||||
break;
|
||||
}
|
||||
|
||||
if (!result) {
|
||||
LogError("submitReadMemory failed!");
|
||||
vcmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
else {
|
||||
vcmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitWriteMemory(amd::WriteMemoryCommand& vcmd)
|
||||
{
|
||||
vcmd.setStatus(CL_RUNNING);
|
||||
|
||||
bool result = false;
|
||||
device::Memory memory(vcmd.destination());
|
||||
|
||||
// Ensure memory up-to-date
|
||||
vcmd.destination().cacheWriteBack();
|
||||
|
||||
// Process different write commands
|
||||
switch (vcmd.type()) {
|
||||
case CL_COMMAND_WRITE_BUFFER:
|
||||
result = blitMgr().writeBuffer(vcmd.source(), memory,
|
||||
vcmd.origin(), vcmd.size(), vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_WRITE_BUFFER_RECT:
|
||||
result = blitMgr().writeBufferRect(vcmd.source(), memory,
|
||||
vcmd.hostRect(), vcmd.bufRect(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_WRITE_IMAGE:
|
||||
result = blitMgr().writeImage(vcmd.source(), memory,
|
||||
vcmd.origin(), vcmd.size(), vcmd.rowPitch(), vcmd.slicePitch(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
default:
|
||||
LogError("Unsupported type for the write command");
|
||||
break;
|
||||
}
|
||||
|
||||
// Mark cache as clean (CPU works directly on backing store)
|
||||
vcmd.destination().signalWrite(NULL);
|
||||
|
||||
if (!result) {
|
||||
LogError("submitWriteMemory failed!");
|
||||
vcmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
else {
|
||||
vcmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
VirtualCPU::submitCopyMemory(amd::CopyMemoryCommand& vcmd)
|
||||
{
|
||||
vcmd.setStatus(CL_RUNNING);
|
||||
|
||||
// Ensure memory up-to-date
|
||||
vcmd.source().cacheWriteBack();
|
||||
vcmd.destination().cacheWriteBack();
|
||||
|
||||
// Translate memory references and ensure cache up-to-date
|
||||
device::Memory dstMemory(vcmd.destination());
|
||||
device::Memory srcMemory(vcmd.source());
|
||||
|
||||
bool result = false;
|
||||
|
||||
// Check if HW can be used for memory copy
|
||||
switch (vcmd.type()) {
|
||||
case CL_COMMAND_COPY_BUFFER:
|
||||
result = blitMgr().copyBuffer(srcMemory, dstMemory,
|
||||
vcmd.srcOrigin(), vcmd.dstOrigin(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_COPY_BUFFER_RECT:
|
||||
result = blitMgr().copyBufferRect(srcMemory, dstMemory,
|
||||
vcmd.srcRect(), vcmd.dstRect(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_COPY_IMAGE_TO_BUFFER:
|
||||
result = blitMgr().copyImageToBuffer(srcMemory, dstMemory,
|
||||
vcmd.srcOrigin(), vcmd.dstOrigin(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_COPY_BUFFER_TO_IMAGE:
|
||||
result = blitMgr().copyBufferToImage(srcMemory, dstMemory,
|
||||
vcmd.srcOrigin(), vcmd.dstOrigin(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_COPY_IMAGE:
|
||||
result = blitMgr().copyImage(srcMemory, dstMemory,
|
||||
vcmd.srcOrigin(), vcmd.dstOrigin(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
default:
|
||||
LogError("Unsupported command type for memory copy!");
|
||||
break;
|
||||
}
|
||||
|
||||
// Mark cache as clean (CPU works directly on backing store)
|
||||
vcmd.destination().signalWrite(NULL);
|
||||
|
||||
if (!result) {
|
||||
LogError("submitCopyMemory failed!");
|
||||
vcmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
else {
|
||||
vcmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitMapMemory(amd::MapMemoryCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
|
||||
if (cmd.mapFlags() & CL_MAP_READ
|
||||
|| cmd.mapFlags() & CL_MAP_WRITE) {
|
||||
LogInfo("cpu::VirtualCPU::submitMapMemory() CL_MAP_READ and CL_MAP_WRITE ignored");
|
||||
}
|
||||
|
||||
// Ensure memory up-to-date
|
||||
cmd.memory().cacheWriteBack();
|
||||
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitUnmapMemory(amd::UnmapMemoryCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
|
||||
// Mark cache as clean (CPU works directly on backing store)
|
||||
cmd.memory().signalWrite(NULL);
|
||||
|
||||
//! @todo:dgladdin: strictly speaking we should check that the mem object was mapped
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitFillMemory(amd::FillMemoryCommand& vcmd)
|
||||
{
|
||||
vcmd.setStatus(CL_RUNNING);
|
||||
|
||||
device::Memory memory(vcmd.memory());
|
||||
|
||||
vcmd.memory().cacheWriteBack();
|
||||
|
||||
bool result = false;
|
||||
|
||||
// Find the the right fill operation
|
||||
switch (vcmd.type()) {
|
||||
case CL_COMMAND_FILL_BUFFER:
|
||||
result = blitMgr().fillBuffer(memory, vcmd.pattern(),
|
||||
vcmd.patternSize(), vcmd.origin(), vcmd.size(),
|
||||
vcmd.isEntireMemory());
|
||||
break;
|
||||
case CL_COMMAND_FILL_IMAGE:
|
||||
result = blitMgr().fillImage(memory, vcmd.pattern(),
|
||||
vcmd.origin(), vcmd.size(), vcmd.isEntireMemory());
|
||||
break;
|
||||
default:
|
||||
LogError("Unsupported command type for FillMemory!");
|
||||
break;
|
||||
}
|
||||
|
||||
vcmd.memory().signalWrite(NULL);
|
||||
|
||||
if (!result) {
|
||||
LogError("submitFillMemory failed!");
|
||||
vcmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
else {
|
||||
vcmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
}
|
||||
|
||||
//! Helper function for forcing a cache sync for all kernel parameters
|
||||
static void syncAllParams(amd::NDRangeKernelCommand& cmd)
|
||||
{
|
||||
const amd::Kernel& kernel = cmd.kernel();
|
||||
const amd::KernelParameters& kernelParam = kernel.parameters();
|
||||
const amd::KernelSignature& signature = kernel.signature();
|
||||
const amd::Device& device = cmd.queue()->device();
|
||||
|
||||
for (size_t i = 0; i < signature.numParameters(); ++i) {
|
||||
const amd::KernelParameterDescriptor& desc = signature.at(i);
|
||||
if (desc.type_ == T_POINTER && desc.size_ > 0 &&
|
||||
!kernelParam.boundToSvmPointer(device, cmd.parameters(), i)) {
|
||||
address ptr = (address) (cmd.parameters() + desc.offset_);
|
||||
amd::Memory* memArg = *(amd::Memory**)ptr;
|
||||
|
||||
if (memArg != NULL) {
|
||||
memArg->cacheWriteBack();
|
||||
memArg->signalWrite(NULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::computeLocalSizes(amd::NDRangeKernelCommand& command,
|
||||
amd::NDRange& local) {
|
||||
bool uniformSize = (OPENCL_MAJOR < 2) ||
|
||||
command.kernel().getDeviceKernel(device())->getUniformWorkGroupSize();
|
||||
|
||||
const amd::NDRangeContainer& sizes = command.sizes();
|
||||
const size_t numCores = device().info().maxComputeUnits_;
|
||||
|
||||
const size_t globalSize1D = sizes.global().product();
|
||||
const size_t targetNumOperations =
|
||||
std::min(globalSize1D, numCores * 4);
|
||||
size_t localSize1D =
|
||||
std::min(globalSize1D / targetNumOperations,
|
||||
device().info().maxWorkGroupSize_);
|
||||
|
||||
for (size_t i = 0; i < local.dimensions(); ++i) {
|
||||
const size_t globalSize = sizes.global()[i];
|
||||
size_t localSize =
|
||||
std::min(std::min(localSize1D, globalSize),
|
||||
device().info().maxWorkItemSizes_[i]);
|
||||
|
||||
// local must exactly divide global if uniform size is required
|
||||
// For non uniform size, we could use the work group size hint
|
||||
if (uniformSize && globalSize % localSize != 0) {
|
||||
while (true) {
|
||||
//! @todo: lmoriche: find a better way
|
||||
if (globalSize % localSize == 0) break;
|
||||
--localSize;
|
||||
}
|
||||
}
|
||||
local[i] = localSize;
|
||||
localSize1D /= localSize;
|
||||
}
|
||||
|
||||
command.setLocalWorkSize(local);
|
||||
}
|
||||
|
||||
|
||||
static
|
||||
amd::NDRange computeRemainders(const amd::NDRange& global,
|
||||
const amd::NDRange& local)
|
||||
{
|
||||
amd::NDRange remainders(local.dimensions());
|
||||
|
||||
for (size_t i = 0; i < local.dimensions(); ++i) {
|
||||
remainders[i] = (global[i] % local[i] != 0) ? 1 : 0;
|
||||
}
|
||||
|
||||
return remainders;
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitKernel(amd::NDRangeKernelCommand& command)
|
||||
{
|
||||
const amd::NDRangeContainer& sizes = command.sizes();
|
||||
const size_t numCores = device().info().maxComputeUnits_;
|
||||
|
||||
amd::NDRange local = sizes.local();
|
||||
|
||||
if (local == 0) {
|
||||
computeLocalSizes(command, local);
|
||||
}
|
||||
amd::NDRange remainders = computeRemainders(sizes.global(), local);
|
||||
|
||||
// number of groups in each dimensions
|
||||
const amd::NDRange numGroups = (sizes.global() / local) + remainders;
|
||||
|
||||
size_t numOperations = numGroups.product();
|
||||
if (numOperations == 0) {
|
||||
command.setStatus(CL_COMPLETE);
|
||||
return;
|
||||
}
|
||||
|
||||
syncAllParams(command);
|
||||
// retain the command here instead of retaining in NDRangeKernelBatch' ctor
|
||||
command.retain();
|
||||
|
||||
size_t batchCount = std::min(numOperations, numCores);
|
||||
NDRangeKernelBatch batch(command, *this, numGroups, batchCount);
|
||||
|
||||
Operation::Counter counter(command, batchCount);
|
||||
command.setData(&counter);
|
||||
|
||||
for (size_t coreId = 0; coreId < batchCount; ++coreId) {
|
||||
batch.setCoreId(coreId);
|
||||
cores_[coreId]->enqueue(batch);
|
||||
cores_[coreId]->flush();
|
||||
}
|
||||
|
||||
command.awaitCompletion();
|
||||
command.release();
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitNativeFn(amd::NativeFnCommand& command)
|
||||
{
|
||||
NativeFn fn(command);
|
||||
cores_[0]->enqueue(fn);
|
||||
cores_[0]->flush();
|
||||
command.awaitCompletion();
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitMarker(amd::Marker& command)
|
||||
{
|
||||
command.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitAcquireExtObjects(amd::AcquireExtObjectsCommand& cmd)
|
||||
{
|
||||
//! @todo [odintsov]: create an AcquireExtObjectsOperation and enqueue it
|
||||
//! to a core when a core scheduler is around.
|
||||
//
|
||||
// cores_[0]->enqueue(new AcquireExtObjectsOperation(cmd));
|
||||
// the code below will be moved to AcquireExtObjectsOperation::execute()
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
|
||||
//
|
||||
// AcquireExtObjects execution starts here
|
||||
//
|
||||
bool bError = false;
|
||||
|
||||
//! Go through ext objects by one and call member function to execute
|
||||
//! a sequence of external graphics API commands for each external object
|
||||
for(std::vector<amd::Memory*>::const_iterator itr = cmd.getMemList().begin();
|
||||
itr != cmd.getMemList().end(); itr++) {
|
||||
if(*itr) {
|
||||
bError |= !((*itr)->mapExtObjectInCQThread());
|
||||
}
|
||||
}
|
||||
if(bError) {
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
else {
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitReleaseExtObjects(amd::ReleaseExtObjectsCommand& cmd)
|
||||
{
|
||||
//! @todo [odintsov]: create a ReleaseExtObjectsOperation and enqueue it
|
||||
//! to a core when a core scheduler is around.
|
||||
//
|
||||
// cores_[i]->enqueue(new ReleaseExtObjectsOperation(cmd));
|
||||
// the code below will be moved to ReleaseExtObjectsOperation::execute()
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
|
||||
bool bError = false;
|
||||
|
||||
for(std::vector<amd::Memory*>::const_iterator itr = cmd.getMemList().begin();
|
||||
itr != cmd.getMemList().end(); itr++) {
|
||||
if(*itr) {
|
||||
bError |= !((*itr)->unmapExtObjectInCQThread());
|
||||
}
|
||||
}
|
||||
if(bError) {
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
else {
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
}
|
||||
|
||||
void VirtualCPU::submitPerfCounter(amd::PerfCounterCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
LogError("We don't support HW perf counters on CPU");
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
|
||||
void VirtualCPU::submitThreadTraceMemObjects(amd::ThreadTraceMemObjectsCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
LogError("We don't support thread trace on CPU");
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
|
||||
void VirtualCPU::submitThreadTrace(amd::ThreadTraceCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
LogError("We don't support thread trace on CPU");
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::flush(amd::Command* list, bool wait)
|
||||
{
|
||||
amd::Command* head = list;
|
||||
|
||||
// Release all commands from the link list
|
||||
while (head != NULL) {
|
||||
amd::Command * it = head->getNext();
|
||||
head->release();
|
||||
head = it;
|
||||
}
|
||||
}
|
||||
|
||||
#if cl_amd_open_video
|
||||
void VirtualCPU::submitRunVideoProgram(amd::RunVideoProgramCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
|
||||
void VirtualCPU::submitSetVideoSession(amd::SetVideoSessionCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
#endif // cl_amd_open_video
|
||||
|
||||
void
|
||||
VirtualCPU::submitSignal(amd::SignalCommand & cmd)
|
||||
{
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitMakeBuffersResident(amd::MakeBuffersResidentCommand & cmd)
|
||||
{
|
||||
cmd.setStatus(CL_INVALID_OPERATION);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitSvmFreeMemory(amd::SvmFreeMemoryCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
if (cmd.pfnFreeFunc() == NULL) {
|
||||
// pointers allocated using clSVMAlloc
|
||||
for (cl_uint i = 0; i < cmd.svmPointers().size(); i++) {
|
||||
amd::SvmBuffer::free(cmd.context(), cmd.svmPointers()[i]);
|
||||
}
|
||||
}
|
||||
else {
|
||||
cmd.pfnFreeFunc()(as_cl(cmd.queue()->asCommandQueue()), cmd.svmPointers().size(),
|
||||
(void**) (&(cmd.svmPointers()[0])), cmd.userData());
|
||||
}
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitSvmCopyMemory(amd::SvmCopyMemoryCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
amd::SvmBuffer::memFill(cmd.dst(), cmd.src(), cmd.srcSize(), 1);
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitSvmFillMemory(amd::SvmFillMemoryCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_RUNNING);
|
||||
amd::SvmBuffer::memFill(cmd.dst(), cmd.pattern(), cmd.patternSize(), cmd.times());
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitSvmMapMemory(amd::SvmMapMemoryCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
void
|
||||
VirtualCPU::submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& cmd)
|
||||
{
|
||||
cmd.setStatus(CL_COMPLETE);
|
||||
}
|
||||
|
||||
} // namespace cpu
|
||||
@@ -0,0 +1,80 @@
|
||||
//
|
||||
// Copyright (c) 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef CPUVIRTUAL_HPP_
|
||||
#define CPUVIRTUAL_HPP_
|
||||
|
||||
#include "top.hpp"
|
||||
#include "device/device.hpp"
|
||||
#include "thread/atomic.hpp"
|
||||
#include "thread/thread.hpp"
|
||||
#include "platform/ndrange.hpp"
|
||||
|
||||
//! \namespace cpu CPU Device Implementation
|
||||
namespace cpu {
|
||||
|
||||
class WorkerThread;
|
||||
class Device;
|
||||
|
||||
class VirtualCPU : public device::VirtualDevice
|
||||
{
|
||||
private:
|
||||
WorkerThread** cores_; //!< Pointer to array of Worker threads
|
||||
static amd::Atomic<size_t> numWorkerThreads_; //!< Current Worker Threads number
|
||||
bool acceptingCommands_;
|
||||
|
||||
public:
|
||||
VirtualCPU(cpu::Device& device);
|
||||
~VirtualCPU();
|
||||
bool terminate();
|
||||
|
||||
WorkerThread* getWorkerThread(size_t id) { return cores_[id]; }
|
||||
|
||||
bool acceptingCommands() const { return acceptingCommands_; }
|
||||
|
||||
virtual void submitReadMemory(amd::ReadMemoryCommand& command);
|
||||
virtual void submitWriteMemory(amd::WriteMemoryCommand& command);
|
||||
virtual void submitCopyMemory(amd::CopyMemoryCommand& command);
|
||||
virtual void submitMapMemory(amd::MapMemoryCommand& command);
|
||||
virtual void submitUnmapMemory(amd::UnmapMemoryCommand& command);
|
||||
virtual void submitKernel(amd::NDRangeKernelCommand& command);
|
||||
virtual void submitNativeFn(amd::NativeFnCommand& command);
|
||||
virtual void submitMarker(amd::Marker& command);
|
||||
virtual void submitFillMemory(amd::FillMemoryCommand& command);
|
||||
virtual void submitMigrateMemObjects(amd::MigrateMemObjectsCommand& cmd) {}
|
||||
virtual void submitAcquireExtObjects(amd::AcquireExtObjectsCommand& cmd);
|
||||
virtual void submitReleaseExtObjects(amd::ReleaseExtObjectsCommand& cmd);
|
||||
virtual void submitPerfCounter(amd::PerfCounterCommand& cmd);
|
||||
virtual void submitThreadTraceMemObjects(amd::ThreadTraceMemObjectsCommand& cmd);
|
||||
virtual void submitThreadTrace(amd::ThreadTraceCommand& cmd);
|
||||
virtual void flush(amd::Command* list = NULL, bool wait = false);
|
||||
#if cl_amd_open_video
|
||||
virtual void submitRunVideoProgram(amd::RunVideoProgramCommand& cmd);
|
||||
virtual void submitSetVideoSession(amd::SetVideoSessionCommand& cmd);
|
||||
#endif // cl_amd_open_video
|
||||
virtual void submitSignal(amd::SignalCommand & cmd);
|
||||
virtual void submitMakeBuffersResident(amd::MakeBuffersResidentCommand & cmd);
|
||||
virtual void submitSvmFreeMemory(amd::SvmFreeMemoryCommand& cmd);
|
||||
virtual void submitSvmCopyMemory(amd::SvmCopyMemoryCommand& cmd);
|
||||
virtual void submitSvmFillMemory(amd::SvmFillMemoryCommand& cmd);
|
||||
virtual void submitSvmMapMemory(amd::SvmMapMemoryCommand& cmd);
|
||||
virtual void submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& cmd);
|
||||
|
||||
virtual void computeLocalSizes(amd::NDRangeKernelCommand& command,
|
||||
amd::NDRange& local);
|
||||
|
||||
static bool fillImage(
|
||||
amd::Image& image,
|
||||
address fillMem,
|
||||
const void* pattern,
|
||||
const amd::Coord3D& origin,
|
||||
const amd::Coord3D& region,
|
||||
size_t rowPitch,
|
||||
size_t slicePitch,
|
||||
size_t elementSize);
|
||||
};
|
||||
|
||||
} // namespace cpu
|
||||
|
||||
#endif // CPUVIRTUAL_HPP_
|
||||
@@ -0,0 +1,207 @@
|
||||
//
|
||||
// Copyright 2011 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
|
||||
#ifndef RING_BUFFER_HPP
|
||||
#define RING_BUFFER_HPP
|
||||
|
||||
#include "top.hpp"
|
||||
#include "thread/atomic.hpp"
|
||||
#include "os/alloc.hpp"
|
||||
// @brief Block-free ring buffer implemenation
|
||||
// @brief THE RING BUFFER SUPPORTS MULTIPLE CONSUMERS AND A SINGLE PRODUCER.
|
||||
// @tparam T Object-type to be saved within the ring buffer.
|
||||
namespace amd{
|
||||
template <typename T>
|
||||
class RingBuffer
|
||||
{
|
||||
public:
|
||||
///////////////////////////////////////
|
||||
// public initialization and cleanup //
|
||||
///////////////////////////////////////
|
||||
RingBuffer();
|
||||
~RingBuffer() { cleanup();}
|
||||
|
||||
bool
|
||||
initialize(unsigned short ringBufferSize);
|
||||
//////////////////////
|
||||
// public interface //
|
||||
///////////////////////
|
||||
|
||||
bool getNext(T & obj);
|
||||
bool insert(const T & obj);
|
||||
private:
|
||||
struct ABACounter{
|
||||
unsigned short tranactionId;
|
||||
unsigned short consumerIndex;
|
||||
};
|
||||
|
||||
union Consumer {
|
||||
ABACounter abaCounter;
|
||||
volatile int32_t interlockedVar;
|
||||
};
|
||||
|
||||
bool canInsert();
|
||||
|
||||
template <typename T2> T2 incrementIndex(T2 index) {
|
||||
index++;
|
||||
if (index == ringBufferSize_)
|
||||
index = 0;
|
||||
return index;
|
||||
}
|
||||
////////////////////////////////////
|
||||
// read only cache line for //
|
||||
// producer and consumer threads //
|
||||
////////////////////////////////////
|
||||
T * ringBuffer_;
|
||||
unsigned short ringBufferSize_;
|
||||
|
||||
char cachePad1_[64];
|
||||
/////////////////////////////////////
|
||||
// read/write cache line for //
|
||||
// producer thread //
|
||||
/////////////////////////////////////
|
||||
//! producer is an index in the ring buffer array
|
||||
volatile int32_t producer_;
|
||||
//! caches the amount of free space in the buffer. reduces cache misses
|
||||
//! by reducing access to 'm_consumer' from producer thread
|
||||
int32_t freeSpace_;
|
||||
|
||||
char cachePad2_[64];
|
||||
/////////////////////////////////////
|
||||
// read/write cache line for //
|
||||
// consumer threads //
|
||||
/////////////////////////////////////
|
||||
volatile Consumer consumer_;
|
||||
|
||||
/////////////////////////////////////
|
||||
// save the thread that inserts //
|
||||
// for checking multiple producers //
|
||||
/////////////////////////////////////
|
||||
Thread *producerThread_;
|
||||
|
||||
void cleanup();
|
||||
// do not allow copying
|
||||
RingBuffer(const RingBuffer&);
|
||||
RingBuffer& operator=(const RingBuffer&);
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
RingBuffer<T>::RingBuffer() :
|
||||
ringBuffer_(NULL),
|
||||
ringBufferSize_(0),
|
||||
producer_(0),
|
||||
freeSpace_(0),
|
||||
producerThread_(NULL)
|
||||
{
|
||||
consumer_.interlockedVar = 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool
|
||||
RingBuffer<T>::initialize(unsigned short ringBufferSize)
|
||||
{
|
||||
bool retVal = false;
|
||||
|
||||
cleanup();
|
||||
ringBuffer_ = new T [ringBufferSize];
|
||||
if (ringBuffer_)
|
||||
{
|
||||
ringBufferSize_ = ringBufferSize;
|
||||
retVal = true;
|
||||
}
|
||||
return retVal;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void
|
||||
RingBuffer<T>::cleanup()
|
||||
{
|
||||
if (ringBuffer_)
|
||||
{
|
||||
delete [] ringBuffer_;
|
||||
ringBuffer_ = NULL;
|
||||
}
|
||||
producer_ = 0;
|
||||
consumer_.interlockedVar = 0;
|
||||
freeSpace_ = 0;
|
||||
ringBufferSize_ = 0;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool
|
||||
RingBuffer<T>::insert(const T & obj)
|
||||
{
|
||||
#ifdef DEBUG
|
||||
// if this is the 1st insert, set producerThread_
|
||||
if (NULL == producerThread_) {
|
||||
producerThread_ = Thread::current();
|
||||
} else {
|
||||
assert(Thread::current() == producerThread_ && "not a single writer");
|
||||
}
|
||||
#endif //DEBUG
|
||||
bool retVal = false;
|
||||
if (canInsert())
|
||||
{
|
||||
ringBuffer_[producer_] = obj;
|
||||
|
||||
producer_ = incrementIndex(producer_);
|
||||
retVal = true;
|
||||
}
|
||||
return retVal;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool
|
||||
RingBuffer<T>::getNext( T & obj)
|
||||
{
|
||||
|
||||
Consumer consumer;
|
||||
consumer.interlockedVar = consumer_.interlockedVar;
|
||||
//cache the producer variable on the stack
|
||||
int producer = producer_;
|
||||
//while the buffer is not empty
|
||||
while (producer != consumer.abaCounter.consumerIndex)
|
||||
{
|
||||
obj = ringBuffer_[consumer.abaCounter.consumerIndex];
|
||||
Consumer newConsumer;
|
||||
newConsumer.abaCounter.consumerIndex = incrementIndex(consumer.abaCounter.consumerIndex);
|
||||
newConsumer.abaCounter.tranactionId = consumer.abaCounter.tranactionId+1;
|
||||
|
||||
if (consumer.interlockedVar == amd::AtomicOperation::compareAndSwap(consumer.interlockedVar,
|
||||
&(consumer_.interlockedVar),newConsumer.interlockedVar))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
consumer.interlockedVar = consumer_.interlockedVar;
|
||||
producer = producer_;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool
|
||||
RingBuffer<T>::canInsert()
|
||||
{
|
||||
if (freeSpace_ > 1)
|
||||
{
|
||||
freeSpace_--;
|
||||
return true;
|
||||
}
|
||||
//cache the volatile variable on the stack;
|
||||
int32_t consumer = consumer_.abaCounter.consumerIndex;
|
||||
|
||||
//there will alway be one unused cell in the array
|
||||
//to distinguish between the case it is completely full and completely empty
|
||||
freeSpace_ = consumer - producer_ - 1 ;
|
||||
if ( freeSpace_ <= -1 )
|
||||
{
|
||||
freeSpace_ = ringBufferSize_ + freeSpace_;
|
||||
}
|
||||
return (freeSpace_ > 0) ;
|
||||
}
|
||||
|
||||
}//NAMESPACE AMD
|
||||
|
||||
#endif // RING_BUFFER_HPP
|
||||
Reference in New Issue
Block a user