AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P4 to Git Change 1179091 by nhaustov@nhaustov_hsa on 2015/08/11 05:48:01

Browse Source 
ECR #333756 - Merge oclutils from runtime.

	Also updates temp file handling code to use pid/atomic which should fix Bug 10793.
	Note, changed the functions to use different prefix to avoid clash with runtime code.

	Reviewed by: Laurent Morichetti
	Testing: smoke, pre-checkin

Affected files ...

... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/alloc.cpp#2 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os.cpp#5 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os.hpp#6 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os_posix.cpp#10 edit
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os_win32.cpp#5 edit
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/atomic.hpp#4 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/monitor.cpp#2 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/monitor.hpp#2 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/semaphore.cpp#2 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/semaphore.hpp#3 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/thread.cpp#3 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/thread.hpp#3 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/debug.hpp#2 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/macros.hpp#3 integrate
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/traits.hpp#2 delete
... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/util.hpp#2 integrate
This commit is contained in:
foreman
2015-08-11 06:09:15 -04:00
parent e41da28605
commit 3607d18592
14 changed files with 442 additions and 853 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rocclr/compiler/lib/promotions/oclutils/os/alloc.cpp
+8 -1
View File
@@ -21,7 +21,14 @@ GuardedMemory::allocate(size_t size, size_t alignment, size_t guardSize)
{
size_t sizeToAllocate = guardSize + alignment;
sizeToAllocate += size + guardSize + Os::pageSize();
address userHostMemGuarded = Os::reserveMemory(sizeToAllocate, Os::MEM_PROT_RW);
sizeToAllocate = amd::alignUp(sizeToAllocate, Os::pageSize());
address userHostMemGuarded = Os::reserveMemory(NULL, sizeToAllocate);
if (!userHostMemGuarded || !Os::commitMemory(
userHostMemGuarded, sizeToAllocate, Os::MEM_PROT_RW)) {
return NULL;
}
address userHostMem = userHostMemGuarded + sizeToAllocate;
userHostMem = amd::alignDown(userHostMem - guardSize, Os::pageSize());
rocclr/compiler/lib/promotions/oclutils/os/os.hpp
+19 -6
View File
@@ -83,6 +83,8 @@ public:
};
private:
static const size_t FILE_PATH_MAX_LENGTH = 1024;
static size_t pageSize_; //!< The default os page size.
static int processorCount_; //!< The number of active processors.
@@ -134,6 +136,8 @@ public:
static void setThreadAffinity(const void* handle, const ThreadAffinityMask& mask);
//! Set the currently running thread's name.
static void setCurrentThreadName(const char* name);
//! Check if the thread is alive
static bool isThreadAlive(const Thread& osThread);
//! Sleep for n milli-seconds.
static void sleep(long n);
@@ -150,16 +154,22 @@ public:
//! Return the amount of host total physical memory in bytes.
static uint64_t hostTotalPhysicalMemory();
//! Reserve a chunk of memory (priv | anon | map on demand).
static address reserveMemory(size_t size, MemProt prot = MEM_PROT_NONE);
//! Reserve a chunk of memory (priv | anon | noreserve).
static address reserveMemory(address start, size_t size, size_t alignment = 0, MemProt prot = MEM_PROT_NONE);
//! Release a chunk of memory reserved with reserveMemory.
static bool releaseMemory(void* addr, size_t size);
//! Commit a chunk of memory previously reserved with reserveMemory.
static bool commitMemory(void* addr, size_t size, MemProt prot = MEM_PROT_NONE);
//! Uncommit a chunk of memory previously committed with commitMemory.
static bool uncommitMemory(void* addr, size_t size);
//! Set the page protections for the given memory region.
static bool protectMemory(void* addr, size_t size, MemProt prot);
//! Release a chunk of memory allocated with reserveMemory.
static bool releaseMemory(void* addr, size_t size);
//! Allocate an aligned chunk of memory.
static void* alignedMalloc(size_t size, size_t alignment);
//! Deallocate an aligned chunk of memory.
static void alignedFree(void* mem);
//! Platform-specific optimized memcpy()
static void* fastMemcpy(void *dest, const void *src, size_t n);
@@ -243,9 +253,12 @@ public:
//! Skip an IDIV (F6/F7) instruction and return a pointer to the next insn.
static bool skipIDIV(address& insn);
// return gloabal memory size to be assigned to device info
static size_t getPhysicalMemSize();
static size_t getPhysicalMemSize();
//! get Application file name
static std::string getAppFileName();
};
/*@}*/
rocclr/compiler/lib/promotions/oclutils/os/os_posix.cpp
+97 -37
View File
@@ -6,6 +6,7 @@
#include "os/os.hpp"
#include "thread/thread.hpp"
#include "utils/util.hpp"
#include <iostream>
#include <stdarg.h>
@@ -31,6 +32,7 @@
# define DT_GNU_HASH 0x6ffffef5
#endif // DT_GNU_HASH
#include <atomic>
#include <vector>
#include <string>
#include <sstream>
@@ -38,6 +40,7 @@
#include <cstdlib>
#include <cstdio> // for tempnam
#include <limits.h>
#include <memory>
#ifdef ANDROID
//#include <sys/ucontext.h>
@@ -95,30 +98,25 @@ divisionErrorHandler(int sig, siginfo_t* info, void* ptr)
#if defined(ATI_ARCH_X86)
insn = (address)uc->uc_mcontext.gregs[LP64_SWITCH(REG_EIP,REG_RIP)];
#else
assert(!"Unimplemented");
assert(!"Unimplemented");
#endif
if(Thread::current()->isWorkerThread()) {
if (Os::skipIDIV(insn)) {
#if defined(ATI_ARCH_X86)
uc->uc_mcontext.gregs[LP64_SWITCH(REG_EIP,REG_RIP)] = (greg_t)insn;
#else
assert(!"Unimplemented");
#endif
return;
}
}
// Call the chained signal handler
if (callOldSignalHandler(sig, info, ptr)) {
return;
}
// @todo: only handle exception in the generated code.
//
//if (!isKernelCode(insn)) {
// return;
//}
if (sig == SIGFPE && info->si_code == FPE_INTDIV) {
if (Os::skipIDIV(insn)) {
#if defined(ATI_ARCH_X86)
uc->uc_mcontext.gregs[LP64_SWITCH(REG_EIP,REG_RIP)] = (greg_t)insn;
#else
assert(!"Unimplemented");
#endif
return;
}
}
std::cerr << "Unhandled signal in divisionErrorHandler()" << std::endl;
::abort();
@@ -305,25 +303,76 @@ memProtToOsProt(Os::MemProt prot)
}
address
Os::reserveMemory(size_t size, MemProt prot)
Os::reserveMemory(address start, size_t size, size_t alignment, MemProt prot)
{
address mem = (address) ::mmap(NULL, size, memProtToOsProt(prot),
MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
size = alignUp(size, pageSize());
alignment = std::max(pageSize(), alignUp(alignment, pageSize()));
assert(isPowerOfTwo(alignment) && "not a power of 2");
assert(mem != NULL && "out of memory");
return mem;
size_t requested = size + alignment - pageSize();
address mem = (address) ::mmap(start, requested, memProtToOsProt(prot),
MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS, 0, 0);
// check for out of memory
if (mem == NULL) return NULL;
address aligned = alignUp(mem, alignment);
// return the unused leading pages to the free state
if (&aligned[0] != &mem[0]) {
assert(&aligned[0] > &mem[0] && "check this code");
if (::munmap(&mem[0], &aligned[0] - &mem[0]) != 0) {
assert(!"::munmap failed");
}
}
// return the unused trailing pages to the free state
if (&aligned[size] != &mem[requested]) {
assert(&aligned[size] < &mem[requested] && "check this code");
if (::munmap(&aligned[size], &mem[requested] - &aligned[size]) != 0) {
assert(!"::munmap failed");
}
}
return aligned;
}
bool
bool
Os::releaseMemory(void* addr, size_t size)
{
// Needs to calculate the size and actual address.
assert(isMultipleOf(addr, pageSize()) && "not page aligned!");
size = alignUp(size, pageSize());
return 0 == ::munmap(addr, size);
}
bool
Os::commitMemory(void* addr, size_t size, MemProt prot)
{
assert(isMultipleOf(addr, pageSize()) && "not page aligned!");
size = alignUp(size, pageSize());
return ::mmap(addr, size, memProtToOsProt(prot),
MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS,
-1, 0) != MAP_FAILED;
}
bool
Os::uncommitMemory(void* addr, size_t size)
{
assert(isMultipleOf(addr, pageSize()) && "not page aligned!");
size = alignUp(size, pageSize());
return ::mmap(addr, size, PROT_NONE,
MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE | MAP_ANONYMOUS,
-1, 0) != MAP_FAILED;
}
bool
Os::protectMemory(void* addr, size_t size, MemProt prot)
{
assert(isMultipleOf(addr, pageSize()) && "not page aligned!");
size = alignUp(size, pageSize());
return 0 == ::mprotect(addr, size, memProtToOsProt(prot));
}
@@ -406,6 +455,12 @@ Thread::entry(Thread* thread)
return thread->main();
}
bool
Os::isThreadAlive(const Thread& thread)
{
return true;
}
const void*
Os::createOsThread(amd::Thread* thread)
{
@@ -424,7 +479,7 @@ Os::createOsThread(amd::Thread* thread)
// We never plan the use join, so free the resources now.
::pthread_attr_setdetachstate(&threadAttr, PTHREAD_CREATE_DETACHED);
pthread_t handle = (pthread_t)NULL;
pthread_t handle = 0;
if (0 != ::pthread_create(&handle, &threadAttr,
(void* (*)(void*)) &Thread::entry, thread)) {
thread->setState(Thread::FAILED);
@@ -705,20 +760,13 @@ Os::getTempPath()
std::string
Os::getTempFileName()
{
std::string tempPath = getTempPath();
char* tempBuf = ::tempnam(tempPath.c_str(), "OCL");
static std::atomic_size_t counter(0);
if (tempBuf == NULL) {
static amd::Atomic<size_t> counter = 0;
std::string tempPath = getTempPath();
std::stringstream tempFileName;
std::stringstream ss;
ss << tempPath << "/OCL" << ::getpid() << 'T' << counter++;
return ss.str();
}
std::string tempFileName = tempBuf;
free(tempBuf);
return tempFileName;
tempFileName << tempPath << "/OCLC" << ::getpid() << 'T' << counter++;
return tempFileName.str();
}
int
@@ -823,6 +871,18 @@ size_t Os::getPhysicalMemSize()
return (size_t) si.totalram * si.mem_unit;
}
std::string Os::getAppFileName()
{
std::unique_ptr<char[]> buff(new char[FILE_PATH_MAX_LENGTH]());
if (readlink("/proc/self/exe", buff.get(), FILE_PATH_MAX_LENGTH) > 0) {
// Get filename without path and extension.
return std::string(basename(buff.get()));
}
return "";
}
} // namespace amd
#endif // !defined(_WIN32) && !defined(__CYGWIN__)
rocclr/compiler/lib/promotions/oclutils/os/os_win32.cpp
+111 -47
View File
@@ -27,6 +27,8 @@ BOOL (WINAPI *pfnGetNumaNodeProcessorMaskEx)(USHORT,PGROUP_AFFINITY) = NULL;
namespace amd {
static size_t allocationGranularity_;
static LONG WINAPI divExceptionFilter(struct _EXCEPTION_POINTERS* ep);
#ifdef _WIN64
@@ -56,6 +58,7 @@ Os::init()
SYSTEM_INFO si;
::GetSystemInfo(&si);
pageSize_ = si.dwPageSize;
allocationGranularity_ = (size_t) si.dwAllocationGranularity;
processorCount_ = si.dwNumberOfProcessors;
LARGE_INTEGER frequency;
@@ -285,26 +288,67 @@ memProtToOsProt(Os::MemProt prot)
}
address
Os::reserveMemory(size_t size, MemProt prot)
Os::reserveMemory(address start, size_t size, size_t alignment, MemProt prot)
{
// Needs to be COMMITed otherwise the protection will fail.
return (address)VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
size = alignUp(size, pageSize());
alignment = std::max(allocationGranularity_,
alignUp(alignment, allocationGranularity_));
assert(isPowerOfTwo(alignment) && "not a power of 2");
size_t requested = size + alignment - allocationGranularity_;
address mem, aligned;
do {
mem = (address)VirtualAlloc(start, requested,
MEM_RESERVE, memProtToOsProt(prot));
// check for out of memory.
if (mem == NULL) return NULL;
aligned = alignUp(mem, alignment);
// check for already aligned memory.
if (aligned == mem && size == requested) {
return mem;
}
// try to reserve the aligned address.
if (VirtualFree(mem, 0, MEM_RELEASE) == 0) {
assert(!"VirtualFree failed");
}
mem = (address)VirtualAlloc(aligned, size,
MEM_RESERVE, memProtToOsProt(prot));
assert((mem == NULL || mem == aligned) && "VirtualAlloc failed");
} while (mem != aligned);
return mem;
}
bool
bool
Os::releaseMemory(void* addr, size_t size)
{
BOOL error = VirtualFree(addr, 0, MEM_RELEASE);
return (error == 0) ? false : true;
return VirtualFree(addr, 0, MEM_RELEASE) != 0;
}
bool
Os::commitMemory(void* addr, size_t size, MemProt prot)
{
return VirtualAlloc(addr, size,
MEM_COMMIT, memProtToOsProt(prot)) != NULL;
}
bool
Os::uncommitMemory(void* addr, size_t size)
{
return VirtualFree(addr, size, MEM_DECOMMIT) != 0;
}
bool
Os::protectMemory(void* addr, size_t size, MemProt prot)
{
DWORD OldProtect;
BOOL error = VirtualProtect(addr, size, memProtToOsProt(prot), &OldProtect);
return (error == 0) ? false : true;
return VirtualProtect(addr, size, memProtToOsProt(prot), &OldProtect) != 0;
}
@@ -406,15 +450,11 @@ divExceptionFilter(struct _EXCEPTION_POINTERS* ep)
{
DWORD code = ep->ExceptionRecord->ExceptionCode;
if (code == EXCEPTION_INT_DIVIDE_BY_ZERO
|| code == EXCEPTION_INT_OVERFLOW) {
// @todo: only handle exception in the generated code.
//
//if (!isKernelCode(insn)) {
// return;
//}
if ((code == EXCEPTION_INT_DIVIDE_BY_ZERO ||
code == EXCEPTION_INT_OVERFLOW) &&
Thread::current()->isWorkerThread()) {
address insn = (address)ep->ContextRecord->LP64_SWITCH(Eip,Rip);
if (Os::skipIDIV(insn)) {
ep->ContextRecord->LP64_SWITCH(Eip,Rip) = (uintptr_t)insn;
return EXCEPTION_CONTINUE_EXECUTION;
@@ -427,25 +467,41 @@ void*
Thread::entry(Thread* thread)
{
void* ret = NULL;
// @todo: We only need this for CPU worker threads.
#if !defined(_WIN64)
if (true /*thread->isWorkerThread()*/) {
__try {
ret = thread->main();
}
__except(divExceptionFilter(GetExceptionInformation())) {
// nothing to do here.
}
}
else {
#else // _WIN64
{
#endif // _WIN64
__try {
ret = thread->main();
}
__except(divExceptionFilter(GetExceptionInformation())) {
// nothing to do here.
}
#else // _WIN64
ret = thread->main();
#endif // _WIN64
// The current thread exits, thus clear the pointer
#if defined(USE_DECLSPEC_THREAD)
details::thread_ = NULL;
#else // !USE_DECLSPEC_THREAD
TlsSetValue(details::threadIndex_, NULL);
#endif // !USE_DECLSPEC_THREAD
return ret;
}
bool
Os::isThreadAlive(const Thread& thread)
{
HANDLE handle = (HANDLE)(thread.handle());
DWORD exitCode = 0;
if (GetExitCodeThread(handle, &exitCode)) {
return exitCode == STILL_ACTIVE;
}
else {
// Could not get thread's exitcode
return false;
}
}
const void*
Os::createOsThread(Thread* thread)
{
@@ -653,14 +709,14 @@ Os::getTempPath()
// under windows directory, use . instead
std::string tempPathStr(tempPath);
char winPath[MAX_PATH];
ret = GetWindowsDirectory(winPath, MAX_PATH);
if (ret > 0) {
size_t len = strlen(winPath);
if (strlen(tempPath) >= len) {
tempPath[len] = 0;
if (_stricmp(tempPath, winPath) == 0) {
return std::string(".");
}
if (GetWindowsDirectory(winPath, MAX_PATH) > 0) {
// Need to check if tempPath is C:\Windows or C:\Windows\ //
if (tempPath[strlen(tempPath)-1] == '\\') {
tempPath[strlen(tempPath)-1] = '\0' ;
ret--;
}
if (_memicmp(tempPath, winPath, ret) == 0) {
return std::string(".");
}
}
return tempPathStr;
@@ -669,18 +725,13 @@ Os::getTempPath()
std::string
Os::getTempFileName()
{
char tempBuf[MAX_PATH];
std::string tempPath = getTempPath();
static std::atomic_size_t counter(0);
if (0 == GetTempFileName(tempPath.c_str(), "OCL", 0, tempBuf)) {
static amd::Atomic<size_t> counter = 0;
std::string tempPath = getTempPath();
std::stringstream tempFileName;
std::stringstream ss;
ss << tempPath << "\\OCL" << ::_getpid() << 'T' << counter++;
return ss.str();
}
return tempBuf;
tempFileName << tempPath << "\\OCLC" << ::_getpid() << 'T' << counter++;
return tempFileName.str();
}
int
@@ -1013,6 +1064,19 @@ size_t Os::getPhysicalMemSize()
return (size_t) statex.ullTotalPhys;
}
std::string Os::getAppFileName()
{
std::string strFileName;
char* buff = new char[FILE_PATH_MAX_LENGTH];
if (GetModuleFileNameA(NULL, buff, FILE_PATH_MAX_LENGTH) != 0) {
// Get filename without path and extension.
strFileName = strrchr(buff, '\\') ? strrchr(buff, '\\') + 1 : buff;
}
delete buff;
return strFileName;
}
} // namespace amd
rocclr/compiler/lib/promotions/oclutils/thread/atomic.hpp
+46 -167
View File
@@ -13,7 +13,8 @@
#define ATOMIC_HPP_
#include "top.hpp"
#include "utils/traits.hpp"
#include <type_traits>
#ifdef _WIN32
# include <intrin.h>
@@ -21,64 +22,15 @@
# include <emmintrin.h>
# include <xmmintrin.h>
#endif // !_WIN32
#include <atomic>
#include <utility>
namespace amd {
/*! \addtogroup Threads
* @{
*
* \defgroup MemOrder Memory ordering
* @{
*/
/*! \brief Memory order access operations.
*/
class MemoryOrder : AllStatic
{
public:
/*! \brief Execute a memory fence.
*
* Perform a serializing operation on loads and stores which guarantees
* that all memory operations dispatched prior to the fence will be
* globally visible before any other memory operation following the fence.
*/
static void fence() {
# if defined(ATI_ARCH_X86)
_mm_mfence();
# else // !ATI_ARCH_X86
__sync_synchronize();
# endif // !ATI_ARCH_X86
}
/*! \brief Execute a loads fence.
*
* Perform a serializing operation on loads which guarantees that all
* load from memory operations dispatched prior to the lfence will be
* globally visible before any other load following the lfence.
*/
static void lfence() {
# if defined(ATI_ARCH_X86)
_mm_lfence();
# else // !ATI_ARCH_X86
fence();
# endif // !ATI_ARCH_X86
}
/*! \brief Execute a stores fence.
*
* Perform a serializing operation on stores which guarantees that all
* store to memory operations dispatched prior to the sfence will be
* globally visible before any other store following the sfence.
*/
static void sfence() {
# if defined(ATI_ARCH_X86)
_mm_sfence();
# else // !ATI_ARCH_X86
fence();
# endif // !ATI_ARCH_X86
}
};
/*! @}
* \addtogroup Atomic Atomic Operations
* @{
*/
@@ -149,7 +101,7 @@ public:
* Atomically add \a inc to \a *dest and return the prior value.
*/
template <typename T>
static T add(typename make_arithmetic<T>::type inc, volatile T* dest)
static T add(T inc, volatile T* dest)
{
return Intrinsics<sizeof(T)>::add((T) inc, dest);
}
@@ -200,7 +152,7 @@ public:
* Atomically or \a mask to \a *dest and return the prior value.
*/
template <typename T>
static T _or(typename make_arithmetic<T>::type mask, volatile T* dest)
static T _or(T mask, volatile T* dest)
{
return Intrinsics<sizeof(T)>::_or((T) mask, dest);
}
@@ -210,7 +162,7 @@ public:
* Atomically or \a mask to \a *dest and return the prior value.
*/
template <typename T>
static T _and(typename make_arithmetic<T>::type mask, volatile T* dest)
static T _and(T mask, volatile T* dest)
{
return Intrinsics<sizeof(T)>::_and((T) mask, dest);
}
@@ -422,8 +374,9 @@ class Atomic
{
private:
typedef typename add_volatile<T>::type value_type;
value_type value_; //!< \brief The variable.
typedef typename std::remove_volatile<typename std::remove_pointer<
typename std::remove_reference<T>::type>::type>::type value_type;
typename std::add_volatile<T>::type value_; //!< \brief The variable.
public:
//! Construct a new %Atomic variable of type T.
@@ -444,50 +397,51 @@ public:
//! Return the %Atomic variable value.
T operator ->() const { return T(value_); }
//! Return the %Atomic variable's address.
typename add_pointer<value_type>::type operator &() { return &value_; }
typename std::add_pointer<typename std::add_volatile<value_type>::type>::
type operator &() { return &value_; }
//! Atomically add \a inc to this variable.
Atomic<T>& operator += (typename make_arithmetic<T>::type inc)
Atomic<T>& operator += (value_type inc)
{
if (is_pointer<T>::value) {
inc *= sizeof(typename remove_pointer<T>::type);
if (std::is_pointer<T>::value) {
inc *= sizeof(typename std::remove_pointer<T>::type);
}
AtomicOperation::add(inc, &value_);
return *this;
}
//! Atomically subtract \a inc to this variable.
Atomic<T>& operator -= (typename make_arithmetic<T>::type inc)
Atomic<T>& operator -= (value_type inc)
{
typename make_arithmetic<T>::type modifier = 0;
if (is_pointer<T>::value) {
inc *= sizeof(typename remove_pointer<T>::type);
value_type modifier = 0;
if (std::is_pointer<T>::value) {
inc *= sizeof(typename std::remove_pointer<T>::type);
}
AtomicOperation::add(modifier - inc, &value_);
return *this;
}
//! Atomically OR \a value to this variable.
Atomic<T>& operator |= (typename make_arithmetic<T>::type mask)
Atomic<T>& operator |= (value_type mask)
{
AtomicOperation::_or(mask, &value_);
return *this;
}
//! Atomically AND \a value to this variable.
Atomic<T>& operator &= (typename make_arithmetic<T>::type mask)
Atomic<T>& operator &= (value_type mask)
{
AtomicOperation::_and(mask, &value_);
return *this;
}
//! Atomically increment this variable and return its new value.
typename remove_reference<T>::type operator ++ ()
typename std::remove_reference<T>::type operator ++ ()
{
if (is_pointer<T>::value) {
typename make_arithmetic<T>::type inc = 1;
return AtomicOperation::add(
inc * sizeof(typename remove_pointer<T>::type), &value_) + 1;
if (std::is_pointer<T>::value) {
value_type inc = static_cast<value_type>(
sizeof(typename std::remove_pointer<T>::type));
return AtomicOperation::add(inc, &value_) + 1;
}
else {
return AtomicOperation::increment(&value_) + 1;
@@ -495,12 +449,13 @@ public:
}
//! Atomically decrement this variable and return its new value.
typename remove_reference<T>::type operator -- ()
typename std::remove_reference<T>::type operator -- ()
{
if (is_pointer<T>::value) {
typename make_arithmetic<T>::type inc = -1;
return AtomicOperation::add(
inc * sizeof(typename remove_pointer<T>::type), &value_) - 1;
if (std::is_pointer<T>::value) {
value_type inc = static_cast<value_type>(-
static_cast<typename std::make_signed<value_type>::type>(
sizeof(typename std::remove_pointer<T>::type)));
return AtomicOperation::add(inc, &value_) - 1;
}
else {
return AtomicOperation::decrement(&value_) - 1;
@@ -508,12 +463,12 @@ public:
}
//! Atomically increment this variable and return its previous value.
typename remove_reference<T>::type operator ++ (int)
typename std::remove_reference<T>::type operator ++ (int)
{
if (is_pointer<T>::value) {
typename make_arithmetic<T>::type inc = 1;
return AtomicOperation::add(
inc * sizeof(typename remove_pointer<T>::type), &value_);
if (std::is_pointer<T>::value) {
value_type inc = static_cast<value_type>(
sizeof(typename std::remove_pointer<T>::type));
return AtomicOperation::add(inc, &value_);
}
else {
return AtomicOperation::increment(&value_);
@@ -523,10 +478,11 @@ public:
//! Atomically decrement this variable and return its previous value.
T operator -- (int)
{
if (is_pointer<T>::value) {
typename make_arithmetic<T>::type inc = -1;
return AtomicOperation::add(
inc * sizeof(typename remove_pointer<T>::type), &value_);
if (std::is_pointer<T>::value) {
value_type inc = static_cast<value_type>(-
static_cast<typename std::make_signed<value_type>::type>(
sizeof(typename std::remove_pointer<T>::type)));
return AtomicOperation::add(inc, &value_);
}
else {
return AtomicOperation::decrement(&value_);
@@ -556,7 +512,7 @@ public:
*/
void storeRelease(T value)
{
MemoryOrder::fence();
std::atomic_thread_fence(std::memory_order_release);
value_ = value;
}
@@ -569,7 +525,7 @@ public:
T loadAcquire() const
{
T value = value_;
MemoryOrder::fence();
std::atomic_thread_fence(std::memory_order_acquire);
return value;
}
};
@@ -583,83 +539,6 @@ make_atomic(T& t)
}
template <typename T>
class AtomicMarkableReference
{
private:
static const intptr_t kMarkBitMask = 0x1;
private:
Atomic<T*> reference_;
private:
static intptr_t markMask(bool mark)
{
return mark ? kMarkBitMask : intptr_t(0);
}
public:
AtomicMarkableReference()
: reference_(NULL)
{ }
AtomicMarkableReference(T* ptr, bool mark = false)
: reference_((T*)((intptr_t) ptr | markMask(mark)))
{ }
bool compareAndSet(
T* expectedPtr, T* newPtr,
bool expectedMark, bool newMark)
{
return reference_.compareAndSet(
(T*)((intptr_t) expectedPtr | markMask(expectedMark)),
(T*)((intptr_t) newPtr | markMask(newMark)));
}
pair<T*,bool> swap(T* newPtr, bool newMark)
{
T* prev = reference_.swap(
(T*)((intptr_t) newPtr | markMask(newMark)));
return make_pair(
(T*) ((intptr_t) prev & ~kMarkBitMask),
((intptr_t) prev & kMarkBitMask) != 0);
}
bool tryMark(T* expectedPtr, bool newMark)
{
T* current = reference_;
if (((intptr_t) current & ~kMarkBitMask) != (intptr_t) expectedPtr) {
return false;
}
bool currentMark = ((intptr_t) current & kMarkBitMask) != 0;
return currentMark == newMark || reference_.compareAndSet(current,
(T*)((intptr_t) expectedPtr | markMask(newMark)));
}
bool isMarked() const
{
return ((intptr_t)(T*) reference_ & kMarkBitMask) != 0;
}
pair<T*,bool> get() const
{
T* current = reference_;
return make_pair(
(T*) ((intptr_t) current & ~kMarkBitMask),
((intptr_t) current & kMarkBitMask) != 0);
}
T* getReference() const
{
return (T*) ((intptr_t)(T*) reference_ & ~kMarkBitMask);
}
void set(T* ptr, bool mark)
{
reference_ = (T*)((intptr_t) ptr | markMask(mark));
}
};
/*! @}
* @}
*/
rocclr/compiler/lib/promotions/oclutils/thread/monitor.cpp
+56 -46
View File
@@ -9,11 +9,13 @@
#include "utils/util.hpp"
#include <cstring>
#include <tuple>
#include <utility>
namespace amd {
Monitor::Monitor(const char* name, bool recursive) :
contendersList_(NULL), onDeck_(NULL), waitersList_(NULL),
contendersList_(0), onDeck_(0), waitersList_(NULL),
owner_(NULL), recursive_(recursive)
{
const size_t maxNameLen = sizeof(name_);
@@ -66,19 +68,17 @@ Monitor::finishLock()
/* The lock is contended. Push the thread's semaphore onto
* the contention list.
*/
Semaphore& sem = thread->lockSemaphore();
sem.reset();
Semaphore& semaphore = thread->lockSemaphore();
semaphore.reset();
LinkedNode newHead;
newHead.setItem(&sem);
while (true) {
LinkedNode* head; bool isLocked;
newHead.setItem(&semaphore);
intptr_t head = contendersList_.load(std::memory_order_acquire);
for (;;) {
// The assumption is that lockWord is locked. Make sure we do not
// continue unless the lock bit is set.
tie(head, isLocked) = contendersList_.get();
if (!isLocked) {
if ((head & kLockBit) == 0) {
if (tryLock()) {
return;
}
@@ -86,8 +86,10 @@ Monitor::finishLock()
}
// Set the new contention list head if lockWord is unchanged.
newHead.setNext(head);
if (contendersList_.compareAndSet(head, &newHead, kLocked, kLocked)) {
newHead.setNext(reinterpret_cast<LinkedNode*>(head & ~kLockBit));
if (contendersList_.compare_exchange_weak(head,
reinterpret_cast<intptr_t>(&newHead) | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire)) {
break;
}
@@ -97,7 +99,7 @@ Monitor::finishLock()
int32_t spinCount = 0;
// Go to sleep until we become the on-deck thread.
while (onDeck_.getReference() != &sem) {
while ((onDeck_ & ~kLockBit) != reinterpret_cast<intptr_t>(&semaphore)) {
// First, be SMT friendly
if (spinCount < kMaxReadSpinIter) {
Os::spinPause();
@@ -108,7 +110,7 @@ Monitor::finishLock()
}
// now go to sleep
else {
sem.wait();
semaphore.wait();
}
spinCount++;
}
@@ -118,8 +120,9 @@ Monitor::finishLock()
// From now-on, we are the on-deck thread. It will stay that way until
// we successfuly acquire the lock.
//
while (true) {
assert(onDeck_.getReference() == &sem && "just checking");
for (;;) {
assert((onDeck_ & ~kLockBit) == reinterpret_cast<intptr_t>(&semaphore)
&& "just checking");
if (tryLock()) {
break;
}
@@ -136,13 +139,13 @@ Monitor::finishLock()
}
// now go to sleep
else {
sem.wait();
semaphore.wait();
}
spinCount++;
}
assert(newHead.next() == NULL && "Should not be linked");
onDeck_ = NULL;
onDeck_ = 0;
}
void
@@ -152,53 +155,58 @@ Monitor::finishUnlock()
// list waiting to acquire the lock. We need to select a successor and
// place it on-deck.
while (true) {
for (;;) {
// Grab the onDeck_ microlock to protect the next loop (make sure only
// one semaphore is removed from the contention list).
//
if (!onDeck_.compareAndSet(NULL, NULL, kUnlocked, kLocked)) {
intptr_t ptr = 0;
if (!onDeck_.compare_exchange_strong(ptr, ptr | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire)) {
return; // Somebody else has the microlock, let him select onDeck_
}
LinkedNode* head; bool isLocked;
while (true) {
tie(head, isLocked) = contendersList_.get();
if (head == NULL) {
intptr_t head = contendersList_.load(std::memory_order_acquire);
for (;;) {
if (head == 0) {
break; // There's nothing else to do.
}
if (isLocked) {
if ((head & kLockBit) != 0) {
// Somebody could have acquired then released the lock
// and failed to grab the onDeck_ microlock.
head = NULL;
head = 0;
break;
}
if (contendersList_.compareAndSet(
head, head->next(), kUnlocked, kUnlocked)) {
if (contendersList_.compare_exchange_weak(
head, reinterpret_cast<intptr_t>(
reinterpret_cast<LinkedNode*>(head)->next()),
std::memory_order_acq_rel, std::memory_order_acquire)) {
#ifdef ASSERT
head->setNext(NULL);
reinterpret_cast<LinkedNode*>(head)->setNext(NULL);
#endif // ASSERT
break;
}
}
Semaphore* sem = (head != NULL) ? head->item() : NULL;
onDeck_ = sem;
MemoryOrder::fence();
Semaphore* semaphore = (head != 0)
? reinterpret_cast<LinkedNode*>(head)->item()
: NULL;
onDeck_.store(reinterpret_cast<intptr_t>(semaphore),
std::memory_order_release);
//
// Release the onDeck_ microlock (end of critical region);
if (sem != NULL) {
sem->post();
if (semaphore != NULL) {
semaphore->post();
return;
}
// We do not have an on-deck thread (sem == NULL). Return if
// We do not have an on-deck thread (semaphore == NULL). Return if
// the contention list is empty or if the lock got acquired again.
tie(head, isLocked) = contendersList_.get();
if (isLocked || head == NULL) {
head = contendersList_;
if (head == 0 || (head & kLockBit) != 0) {
return;
}
}
@@ -228,7 +236,7 @@ Monitor::wait()
// Go to sleep until we become the on-deck thread.
int32_t spinCount = 0;
while (onDeck_.getReference() != &suspend) {
while ((onDeck_ & ~kLockBit) != reinterpret_cast<intptr_t>(&suspend)) {
// First, be SMT friendly
if (spinCount < kMaxReadSpinIter) {
Os::spinPause();
@@ -245,8 +253,9 @@ Monitor::wait()
}
spinCount = 0;
while (true) {
assert(onDeck_.getReference() == &suspend && "just checking");
for (;;) {
assert((onDeck_ & ~kLockBit) == reinterpret_cast<intptr_t>(&suspend)
&& "just checking");
if (trySpinLock()) {
break;
@@ -272,8 +281,7 @@ Monitor::wait()
// Restore the lock count (for recursive mutexes)
lockCount_ = lockCount;
onDeck_ = NULL;
MemoryOrder::fence();
onDeck_.store(0, std::memory_order_release);
}
void
@@ -288,11 +296,13 @@ Monitor::notify()
// Dequeue a waiter from the wait list and add it to the contention list.
waitersList_ = waiter->next();
while (true) {
LinkedNode* node = contendersList_.getReference();
waiter->setNext(node);
if (contendersList_.compareAndSet(node, waiter, kLocked, kLocked)) {
intptr_t node = contendersList_.load(std::memory_order_acquire);
for (;;) {
waiter->setNext(reinterpret_cast<LinkedNode*>(node & ~kLockBit));
if (contendersList_.compare_exchange_weak(node,
reinterpret_cast<intptr_t>(waiter) | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire)) {
break;
}
}
rocclr/compiler/lib/promotions/oclutils/thread/monitor.hpp
+60 -27
View File
@@ -6,10 +6,14 @@
#define MONITOR_HPP_
#include "top.hpp"
#include "atomic.hpp"
#include "thread/atomic.hpp"
#include "thread/semaphore.hpp"
#include "thread/thread.hpp"
#include <atomic>
#include <tuple>
#include <utility>
namespace amd {
/*! \addtogroup Threads
@@ -19,13 +23,46 @@ namespace amd {
* @{
*/
namespace details {
template <class T, class AllocClass = HeapObject>
struct SimplyLinkedNode : public AllocClass
{
typedef SimplyLinkedNode<T, AllocClass> Node;
protected:
std::atomic<Node*> next_; /*!< \brief The next element. */
T volatile item_;
public:
//! \brief Return the next element in the linked-list.
Node* next() const { return next_; }
//! \brief Return the item.
T item() const { return item_; }
//! \brief Set the next element pointer.
void setNext(Node* next) { next_ = next; }
//! \brief Set the item.
void setItem(T item) { item_ = item; }
//! \brief Swap the next element pointer.
Node* swapNext(Node* next) { return next_.swap(next); }
//! \brief Compare and set the next element pointer.
bool compareAndSetNext(Node* compare, Node* next)
{
return next_.compare_exchange_strong(compare, next);
}
};
} // namespace details
class Monitor : public HeapObject
{
typedef SimplyLinkedNode<Semaphore*,StackObject> LinkedNode;
typedef details::SimplyLinkedNode<Semaphore*,StackObject> LinkedNode;
private:
static const bool kUnlocked = false;
static const bool kLocked = true;
static const intptr_t kLockBit = 0x1;
static const int kMaxSpinIter = 55; //!< Total number of spin iterations.
static const int kMaxReadSpinIter = 50; //!< Read iterations before yielding
@@ -33,12 +70,12 @@ private:
/*! Linked list of semaphores the contending threads are waiting on
* and main lock.
*/
AtomicMarkableReference<LinkedNode> contendersList_;
std::atomic_intptr_t contendersList_;
//! The Mutex's name
char name_[64];
//! Semaphore of the next thread to contend for the lock.
AtomicMarkableReference<Semaphore> onDeck_;
std::atomic_intptr_t onDeck_;
//! Linked list of the suspended threads resume semaphores.
LinkedNode* volatile waitersList_;
@@ -63,7 +100,7 @@ protected:
*
* \note The user is responsible for the memory ordering.
*/
bool isLocked() const { return contendersList_.isMarked(); }
bool isLocked() const { return (contendersList_ & kLockBit) != 0; }
//! Return this monitor's owner thread (NULL if unlocked).
Thread* owner() const { return owner_; }
@@ -141,10 +178,9 @@ Monitor::tryLock()
Thread* thread = Thread::current();
assert(thread != NULL && "cannot lock() from (null)");
LinkedNode* ptr; bool isLocked;
tie(ptr, isLocked) = contendersList_.get();
intptr_t ptr = contendersList_.load(std::memory_order_acquire);
if (unlikely(isLocked)) {
if (unlikely((ptr & kLockBit) != 0)) {
if (recursive_ && thread == owner_) {
// Recursive lock: increment the lock count and return.
++lockCount_;
@@ -153,8 +189,8 @@ Monitor::tryLock()
return false; // Already locked!
}
if (unlikely(!contendersList_.compareAndSet(
ptr, ptr, kUnlocked, kLocked))) {
if (unlikely(!contendersList_.compare_exchange_weak(ptr, ptr | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire))) {
return false; // We failed the CAS from unlocked to locked.
}
@@ -189,23 +225,21 @@ Monitor::unlock()
setOwner(NULL);
while (true) {
LinkedNode* ptr = contendersList_.getReference();
// Clear the lock bit.
if (contendersList_.compareAndSet(ptr, ptr, kLocked, kUnlocked)) {
break; // We succeeded the CAS from locked to unlocked.
}
}
// Clear the lock bit.
intptr_t ptr = contendersList_.load(std::memory_order_acquire);
while (!contendersList_.compare_exchange_weak(ptr, ptr & ~kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire))
;
//
// We succeeded the CAS from locked to unlocked.
// This is the end of the critical region.
// Check if we have an on-deck thread that needs signaling.
Semaphore* onDeck; bool isMarked;
tie(onDeck, isMarked) = onDeck_.get();
if (onDeck != NULL) {
if (!isMarked) {
intptr_t onDeck = onDeck_;
if (onDeck != 0) {
if ((onDeck & kLockBit) == 0) {
// Only signal if it is unmarked.
onDeck->post();
reinterpret_cast<Semaphore*>(onDeck)->post();
}
return; // We are done.
}
@@ -215,9 +249,8 @@ Monitor::unlock()
// so return if the list is empty or if the lock got acquired again (it's
// somebody else's problem now!)
LinkedNode* head; bool isLocked;
amd::tie(head, isLocked) = contendersList_.get();
if (isLocked || head == NULL) {
intptr_t head = contendersList_;
if (head == 0 || (head & kLockBit) != 0) {
return;
}
rocclr/compiler/lib/promotions/oclutils/thread/semaphore.cpp
+8 -9
View File
@@ -3,7 +3,6 @@
//
#include "thread/semaphore.hpp"
#include "thread/atomic.hpp"
#include "thread/thread.hpp"
#if defined(_WIN32) || defined(__CYGWIN__)
@@ -16,8 +15,8 @@
namespace amd {
Semaphore::Semaphore()
: state_(0)
{
std::atomic_init(&state_, 0);
#ifdef _WIN32
handle_ = static_cast<void*>(CreateSemaphore(NULL, 0, LONG_MAX, NULL));
assert(handle_ != NULL && "CreateSemaphore failed");
@@ -44,18 +43,18 @@ Semaphore::~Semaphore()
void
Semaphore::post()
{
int state;
while (true) {
state = state_;
int state = state_.load(std::memory_order_relaxed);
for (;;) {
if (state > 0) {
// Do a load acquire.
MemoryOrder::fence();
if (state == state_) {
int newstate = state_.load(std::memory_order_acquire);
if (state == newstate) {
return;
}
state = newstate;
continue;
}
if (state_.compareAndSet(state, state+1)) {
if (state_.compare_exchange_weak(state, state+1,
std::memory_order_acq_rel, std::memory_order_acquire)) {
break;
}
}
rocclr/compiler/lib/promotions/oclutils/thread/semaphore.hpp
+5 -5
View File
@@ -6,9 +6,9 @@
#define SEMAPHORE_HPP_
#include "top.hpp"
#include "thread/atomic.hpp"
#include "utils/util.hpp"
#include <atomic>
#if defined(__linux__)
# include <semaphore.h>
#endif /*linux*/
@@ -29,14 +29,14 @@ class Thread;
class Semaphore : public HeapObject
{
private:
Atomic<int> state_; //!< This semaphore's value.
std::atomic_int state_; //!< This semaphore's value.
#ifdef _WIN32
void* handle_; //!< The semaphore object's handle.
char padding_[64-sizeof(void*)-sizeof(Atomic<int>)];
char padding_[64-sizeof(void*)-sizeof(std::atomic_int)];
#else // !_WIN32
sem_t sem_; //!< The semaphore object's identifier.
char padding_[64-sizeof(sem_t)-sizeof(Atomic<int>)];
char padding_[64-sizeof(sem_t)-sizeof(std::atomic_int)];
#endif /*!_WIN32*/
public:
@@ -52,7 +52,7 @@ public:
//! \brief Reset this semaphore.
void reset()
{
state_.swap(0);
state_.store(0, std::memory_order_release);
}
};
rocclr/compiler/lib/promotions/oclutils/thread/thread.hpp
+6 -3
View File
@@ -103,9 +103,6 @@ protected:
size_t stackSize = 0 /*use system default*/,
bool spawn = true /* create a new Os::thread */);
//! Destroy this thread.
virtual ~Thread();
public:
//! Return the currently running thread instance.
static inline Thread* current();
@@ -116,6 +113,9 @@ public:
//! Tear down the OsThread package.
static void tearDown();
//! Destroy this thread.
virtual ~Thread();
//! Return the thread's name
const std::string& name() const { return name_; }
@@ -131,6 +131,9 @@ public:
//! Return true is this is the host thread.
virtual bool isHostThread() const { return false; }
//! Return true if this is a worker thread.
virtual bool isWorkerThread() const { return false; }
//! Get the current thread state.
ThreadState state() const { return state_; }
rocclr/compiler/lib/promotions/oclutils/utils/debug.hpp
-11
View File
@@ -179,15 +179,4 @@ do { \
#define LogPrintfWarning(format, ...) Logf(amd::LOG_WARNING, format, __VA_ARGS__)
#define LogPrintfInfo(format, ...) Logf(amd::LOG_INFO, format, __VA_ARGS__)
//! Used by objects that have split constructors for sanity-checking
//! their construction state
enum ConstructionState {
CS_CONSTRUCTED, //!< alloc (regular) constructor called ok
CS_ACTIVE, //!< create constructor called ok
CS_DYING, //!< in destructor (use to trap races)
CS_BROKEN, //!< something went wrong during construction
};
#endif /*DEBUG_HPP_*/
rocclr/compiler/lib/promotions/oclutils/utils/macros.hpp
+6
View File
@@ -166,6 +166,12 @@
# define ALWAYSINLINE
#endif // !_MSC_VER
#ifdef BRAHMA
# define IS_BRAHMA true
#else
# define IS_BRAHMA false
#endif
//! \endcond
#endif // MACROS_HPP_
rocclr/compiler/lib/promotions/oclutils/utils/traits.hpp
-108
View File
@@ -1,108 +0,0 @@
//
// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved.
//
#ifndef TRAITS_HPP_
#define TRAITS_HPP_
namespace amd {
// Type traits:
//! \cond ignore
template <typename T>
struct is_pointer
{ static const bool value = false; };
template <typename T>
struct is_pointer<T*>
{ static const bool value = true; };
template <typename T>
struct remove_reference
{ typedef T type; };
template <typename T>
struct remove_reference<T&>
{ typedef T type; };
template <typename T>
struct remove_volatile
{ typedef T type; };
template <typename T>
struct remove_volatile<T volatile>
{ typedef T type; };
template <typename T>
struct remove_const
{ typedef T type; };
template <typename T>
struct remove_const<T const>
{ typedef T type; };
template <typename T>
struct remove_pointer
{ typedef T type; };
template <typename T>
struct remove_pointer<T*>
{ typedef T type; };
template <typename T>
struct add_const
{ typedef T const type; };
template <typename T>
struct add_const<T&>
{ typedef T& type; };
template <typename T>
struct add_volatile
{ typedef T volatile type; };
template <typename T>
struct add_volatile<T&>
{ typedef T& type; };
template <typename T>
struct add_pointer
{ typedef typename remove_reference<T>::type* type; };
template <typename T>
struct add_reference
{ typedef typename remove_reference<T>::type& type; };
template <>
struct add_reference<void>
{ typedef void type; };
template <>
struct add_reference<const void>
{ typedef const void type; };
template <>
struct add_reference<volatile void>
{ typedef volatile void type; };
template <>
struct add_reference<const volatile void>
{ typedef const volatile void type; };
template <typename T>
struct make_arithmetic
{ typedef typename remove_volatile<T>::type type; };
template <typename T>
struct make_arithmetic<T*>
{ typedef long int type; };
template <typename T>
struct make_arithmetic<T&>
{ typedef typename make_arithmetic<T>::type type; };
//! \endcond
} // namespace amd
#endif /* TRAITS_HPP_ */
rocclr/compiler/lib/promotions/oclutils/utils/util.hpp
+20 -386
View File
@@ -6,8 +6,8 @@
#define UTIL_HPP_
#include "top.hpp"
#include "thread/atomic.hpp"
#include <atomic>
#include <string>
namespace amd {
@@ -16,307 +16,6 @@ namespace amd {
* @{
*/
//! \cond ignore
template <int N>
struct PairElement;
template <>
struct PairElement<0>
{
template <class F, class S>
static inline F& get(pair<F,S>& p) { return p.first; }
template <class F, class S>
static inline const F& get(const pair<F,S>& p) { return p.first; }
};
template <>
struct PairElement<1>
{
template <class F, class S>
static inline S& get(pair<F,S>& p) { return p.second; }
template <class F, class S>
static inline const S& get(const pair<F,S>& p) { return p.second; }
};
// Forward declaration of the tuple_elements container class.
template <class H, class T>
struct TupleElementsContainer;
/*! \brief Return the type of the Nth element in the tuple.
*/
template <int N, class T>
struct TupleElementType
{
typedef typename T::tail_t next_element;
typedef typename TupleElementType<N-1,next_element>::type type;
};
// break the recursion
template <class T>
struct TupleElementType<0,T>
{
typedef Null next_element;
typedef typename T::head_t type;
};
/*! \brief Helper struct to extract the Nth element from a tuple
*/
template <int N>
struct TupleElementGetter
{
template <class R, class H, class T>
static R get(TupleElementsContainer<H,T>& t)
{
return TupleElementGetter<N-1>::template get<R>(t.tail);
}
template <class R, class H, class T>
static R get(const TupleElementsContainer<H,T>& t)
{
return TupleElementGetter<N-1>::template get<R>(t.tail);
}
};
// break the recursion
template <>
struct TupleElementGetter<0>
{
template <class R, class H, class T>
static R get(TupleElementsContainer<H,T>& t)
{
return t.head;
}
template <class R, class H, class T>
static R get(const TupleElementsContainer<H,T>& t)
{
return t.head;
}
};
/*! \brief Return the Nth element in the tuple.
*/
template <int N, class H, class T>
inline typename TupleElementType<N,TupleElementsContainer<H,T> >::type&
getTupleElement(TupleElementsContainer<H,T>& t)
{
return TupleElementGetter<N>::template get<
typename TupleElementType<N,TupleElementsContainer<H,T> >::type&,
H,T>(t);
}
template <int N, class H, class T>
inline const typename TupleElementType<N,TupleElementsContainer<H,T> >::type&
getTupleElement(const TupleElementsContainer<H, T>& t)
{
return TupleElementGetter<N>::template get<
const typename TupleElementType<N,TupleElementsContainer<H,T> >::type&,
H,T>(t);
}
/*! \brief The tuple elements struct
*/
template <class H, class T>
struct TupleElementsContainer
{
typedef H head_t;
typedef T tail_t;
head_t head; tail_t tail;
TupleElementsContainer() : head(), tail() { }
template <class T0, class T1, class T2, class T3>
TupleElementsContainer(T0& t0, T1& t1, T2& t2, T3& t3)
: head(t0), tail(t1, t2, t3, null())
{ }
template <class HH, class TT>
TupleElementsContainer& operator= (const TupleElementsContainer<HH,TT>& t)
{
head = t.head;
tail = t.tail;
return *this;
}
template <class F, class S>
TupleElementsContainer& operator= (const pair<F,S>& p)
{
head = p.first;
tail.head = p.second;
return *this;
}
template<int N>
typename TupleElementType<N, TupleElementsContainer>::type&
get() { return getTupleElement<N>(*this); }
};
// break the recursion
template <class H>
struct TupleElementsContainer<H, Null>
{
typedef H head_t;
typedef Null tail_t;
H head;
TupleElementsContainer() : head() { }
template <class T0>
TupleElementsContainer(T0& t0, const Null&, const Null&, const Null&)
: head(t0)
{ }
template <class HH>
TupleElementsContainer& operator = (
const TupleElementsContainer<HH,Null>& t)
{
head = t.head;
return *this;
}
template<int N>
typename TupleElementType<N, TupleElementsContainer>::type&
get() { return getTupleElement<N>(*this); }
};
/*! \brief Rebind the TupleElementsContainer type.
*/
template <class T0, class T1, class T2, class T3>
struct TupleElementsBinder
{
typedef TupleElementsContainer<
T0, typename TupleElementsBinder<T1, T2, T3, Null>::type
> type;
};
// break the recursion
template<>
struct TupleElementsBinder<Null, Null, Null, Null>
{ typedef Null type; };
//! \endcond
/*! \brief A simple N-element (1 to 4) tuple.
*/
template <class T0 = Null, class T1 = Null, class T2 = Null, class T3 = Null>
class tuple : public TupleElementsBinder<T0, T1, T2, T3>::type
{
private:
typedef typename TupleElementsBinder<T0, T1, T2, T3>::type base_t;
public:
tuple() { }
tuple(T0 t0) : base_t(t0, null(), null(), null()) { }
tuple(T0 t0, T1 t1) : base_t(t0, t1, null(), null()) { }
tuple(T0 t0, T1 t1, T2 t2) : base_t(t0, t1, t2, null()) { }
tuple(T0 t0, T1 t1, T2 t2, T3 t3) : base_t(t0, t1, t2, t3) { }
template <class H, class T>
tuple(const TupleElementsContainer<H,T>& te) : base_t(te)
{ }
template <class H, class T>
tuple& operator = (const TupleElementsContainer<H,T>& te)
{
base_t::operator = (te);
return *this;
}
template <class F, class S>
tuple& operator = (const pair<F,S>& p)
{
base_t::operator = (p);
return *this;
}
};
// tuple / pair element getters.
template <int N, class H, class T>
inline typename TupleElementType<N, TupleElementsContainer<H,T> >::type&
get(TupleElementsContainer<H,T>& te)
{
return getTupleElement<N>(te);
}
template <int N, class H, class T>
inline const typename TupleElementType<N, TupleElementsContainer<H,T> >::type&
get(const TupleElementsContainer<H,T>& te)
{
return getTupleElement<N>(te);
}
template <int N, class F, class S>
inline typename TupleElementType<N, tuple<F,S> >::type&
get(pair<F,S>& p)
{
return PairElement<N>::get(p);
}
template <int N, class F, class S>
inline const typename TupleElementType<N, tuple<F,S> >::type&
get(const pair<F,S>& p)
{
return PairElement<N>::get(p);
}
// Some tuple helpers (make_tuple() and tie())
template <class T0>
inline tuple<T0>
make_tuple(const T0& t0)
{
return tuple<T0>(t0);
}
template <class T0, class T1>
inline tuple<T0, T1>
make_tuple(const T0& t0, const T1& t1)
{
return tuple<T0, T1>(t0, t1);
}
template <class T0, class T1, class T2>
inline tuple<T0, T1, T2>
make_tuple(const T0& t0, const T1& t1, const T2& t2)
{
return tuple<T0, T1, T2>(t0, t1, t2);
}
template <class T0, class T1, class T2, class T3>
inline tuple<T0, T1, T2, T3>
make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3)
{
return tuple<T0, T1, T2, T3>(t0, t1, t2, t3);
}
template <class T0>
inline tuple<T0&>
tie(T0& t0)
{
return tuple<T0&>(t0);
}
template <class T0, class T1>
inline tuple<T0&, T1&>
tie(T0& t0, T1& t1)
{
return tuple<T0&, T1&>(t0, t1);
}
template <class T0, class T1, class T2>
inline tuple<T0&, T1&, T2&>
tie(T0& t0, T1& t1, T2& t2)
{
return tuple<T0&, T1&, T2&>(t0, t1, t2);
}
template <class T0, class T1, class T2, class T3>
inline tuple<T0&, T1&, T2&, T3&>
tie(T0& t0, T1& t1, T2& t2, T3& t3)
{
return tuple<T0&, T1&, T2&, T3&>(t0, t1, t2, t3);
}
//! \brief Check if the given value \a val is a power of 2.
template <typename T>
static inline bool
@@ -471,94 +170,22 @@ alignUp(T* value, size_t alignment)
return (T*) alignDown((intptr_t) (value + alignment - 1), alignment);
}
template <class T, class AllocClass = HeapObject>
struct SimplyLinkedNode : public AllocClass
template<typename T>
inline bool isMultipleOf(T value, size_t alignment)
{
typedef SimplyLinkedNode<T, AllocClass> Node;
protected:
Atomic<Node*> next_; /*!< \brief The next element. */
T volatile item_;
public:
//! \brief Return the next element in the linked-list.
Node* next() const { return next_; }
//! \brief Return the item.
T item() const { return item_; }
//! \brief Set the next element pointer.
void setNext(Node* next) { next_ = next; }
//! \brief Set the item.
void setItem(T item) { item_ = item; }
//! \brief Swap the next element pointer.
Node* swapNext(Node* next) { return next_.swap(next); }
//! \brief Compare and set the next element pointer.
bool compareAndSetNext(Node* compare, Node* next)
{
return next_.compareAndSet(compare, next);
if (isPowerOfTwo(alignment)) {
// fast path, using logical operators
return alignUp(value, alignment) == value;
}
};
return value % alignment == 0;
}
/* For the implementation of a doubly-linked list, check:
* Lock-Free and Practical
* Deques and Doubly Linked
* Lists using Single-Word
* Compare-And-Swap
*
* Hakan Sundell, Philippas Tsigas
* Department of Computing Science
* Chalmers Univ. of Technol. and Goteborg Univ.
*/
template <class T, class AllocClass = HeapObject>
struct DoublyLinkedNode
template<typename T>
inline bool isMultipleOf(T* value, size_t alignment)
{
typedef SimplyLinkedNode<T, AllocClass> Node;
protected:
Atomic<Node*> prev_; //!< The previous element.
Atomic<Node*> next_; //!< The next element.
T volatile item_;
public:
//! \brief Return the previous element in the linked-list.
Node* prev() const { return prev_; }
//! \brief Return the next element in the linked-list.
Node* next() const { return next_; }
//! \brief Return the item.
T item() const { return item_; }
//! \brief Set the previous element pointer.
void setPrev(Node* prev) { prev_ = prev; }
//! \brief Set the next element pointer.
void setNext(Node* next) { next_ = next; }
//! \brief Set the item.
void setItem(T item) { item_ = item; }
//! \brief Swap the previous element pointer.
Node* swapPrev(Node* prev)
{
return prev_.swap(prev);
}
//! \brief Swap the next element pointer.
Node* swapNext( Node* next)
{
return next_.swap(next);
}
//! \brief Compare and set the previous element pointer.
bool compareAndSetPrev(Node* compare, Node* prev)
{
return prev_.compareAndSet(compare, prev, false, false);
}
//! \brief Compare and set the next element pointer.
bool compareAndSetNext(Node* compare, Node* next)
{
return next_.compareAndSet(compare, next, false, false);
}
};
intptr_t ptr = reinterpret_cast<intptr_t>(value);
return isMultipleOf(ptr, alignment);
}
template <class Reference, class Value>
struct DeviceMap {
@@ -642,6 +269,13 @@ inline uint leastBitSet(T value)
leastBitSet32((uint32_t)value);
}
static inline bool Is32Bits() {
return LP64_SWITCH(true, false);
}
static inline bool Is64Bits() {
return LP64_SWITCH(false, true);
}
/*@}*/} // namespace amd
#endif /*UTIL_HPP_*/