From 3607d1859299c8bd8684d971552d3b12a96b6e23 Mon Sep 17 00:00:00 2001 From: foreman Date: Tue, 11 Aug 2015 06:09:15 -0400 Subject: [PATCH] P4 to Git Change 1179091 by nhaustov@nhaustov_hsa on 2015/08/11 05:48:01 ECR #333756 - Merge oclutils from runtime. Also updates temp file handling code to use pid/atomic which should fix Bug 10793. Note, changed the functions to use different prefix to avoid clash with runtime code. Reviewed by: Laurent Morichetti Testing: smoke, pre-checkin Affected files ... ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/alloc.cpp#2 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os.cpp#5 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os.hpp#6 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os_posix.cpp#10 edit ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/os/os_win32.cpp#5 edit ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/atomic.hpp#4 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/monitor.cpp#2 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/monitor.hpp#2 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/semaphore.cpp#2 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/semaphore.hpp#3 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/thread.cpp#3 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/thread/thread.hpp#3 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/debug.hpp#2 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/macros.hpp#3 integrate ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/traits.hpp#2 delete ... //depot/stg/opencl/drivers/opencl/compiler/lib/promotions/oclutils/utils/util.hpp#2 integrate --- .../lib/promotions/oclutils/os/alloc.cpp | 9 +- .../lib/promotions/oclutils/os/os.hpp | 25 +- .../lib/promotions/oclutils/os/os_posix.cpp | 134 ++++-- .../lib/promotions/oclutils/os/os_win32.cpp | 158 +++++-- .../lib/promotions/oclutils/thread/atomic.hpp | 213 ++------- .../promotions/oclutils/thread/monitor.cpp | 102 +++-- .../promotions/oclutils/thread/monitor.hpp | 87 ++-- .../promotions/oclutils/thread/semaphore.cpp | 17 +- .../promotions/oclutils/thread/semaphore.hpp | 10 +- .../lib/promotions/oclutils/thread/thread.hpp | 9 +- .../lib/promotions/oclutils/utils/debug.hpp | 11 - .../lib/promotions/oclutils/utils/macros.hpp | 6 + .../lib/promotions/oclutils/utils/traits.hpp | 108 ----- .../lib/promotions/oclutils/utils/util.hpp | 406 +----------------- 14 files changed, 442 insertions(+), 853 deletions(-) delete mode 100644 rocclr/compiler/lib/promotions/oclutils/utils/traits.hpp diff --git a/rocclr/compiler/lib/promotions/oclutils/os/alloc.cpp b/rocclr/compiler/lib/promotions/oclutils/os/alloc.cpp index 8751e80635..53e5f61499 100644 --- a/rocclr/compiler/lib/promotions/oclutils/os/alloc.cpp +++ b/rocclr/compiler/lib/promotions/oclutils/os/alloc.cpp @@ -21,7 +21,14 @@ GuardedMemory::allocate(size_t size, size_t alignment, size_t guardSize) { size_t sizeToAllocate = guardSize + alignment; sizeToAllocate += size + guardSize + Os::pageSize(); - address userHostMemGuarded = Os::reserveMemory(sizeToAllocate, Os::MEM_PROT_RW); + + sizeToAllocate = amd::alignUp(sizeToAllocate, Os::pageSize()); + address userHostMemGuarded = Os::reserveMemory(NULL, sizeToAllocate); + if (!userHostMemGuarded || !Os::commitMemory( + userHostMemGuarded, sizeToAllocate, Os::MEM_PROT_RW)) { + return NULL; + } + address userHostMem = userHostMemGuarded + sizeToAllocate; userHostMem = amd::alignDown(userHostMem - guardSize, Os::pageSize()); diff --git a/rocclr/compiler/lib/promotions/oclutils/os/os.hpp b/rocclr/compiler/lib/promotions/oclutils/os/os.hpp index 249838420a..e361a96f4f 100644 --- a/rocclr/compiler/lib/promotions/oclutils/os/os.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/os/os.hpp @@ -83,6 +83,8 @@ public: }; private: + static const size_t FILE_PATH_MAX_LENGTH = 1024; + static size_t pageSize_; //!< The default os page size. static int processorCount_; //!< The number of active processors. @@ -134,6 +136,8 @@ public: static void setThreadAffinity(const void* handle, const ThreadAffinityMask& mask); //! Set the currently running thread's name. static void setCurrentThreadName(const char* name); + //! Check if the thread is alive + static bool isThreadAlive(const Thread& osThread); //! Sleep for n milli-seconds. static void sleep(long n); @@ -150,16 +154,22 @@ public: //! Return the amount of host total physical memory in bytes. static uint64_t hostTotalPhysicalMemory(); - //! Reserve a chunk of memory (priv | anon | map on demand). - static address reserveMemory(size_t size, MemProt prot = MEM_PROT_NONE); + //! Reserve a chunk of memory (priv | anon | noreserve). + static address reserveMemory(address start, size_t size, size_t alignment = 0, MemProt prot = MEM_PROT_NONE); + //! Release a chunk of memory reserved with reserveMemory. + static bool releaseMemory(void* addr, size_t size); + //! Commit a chunk of memory previously reserved with reserveMemory. + static bool commitMemory(void* addr, size_t size, MemProt prot = MEM_PROT_NONE); + //! Uncommit a chunk of memory previously committed with commitMemory. + static bool uncommitMemory(void* addr, size_t size); //! Set the page protections for the given memory region. static bool protectMemory(void* addr, size_t size, MemProt prot); - //! Release a chunk of memory allocated with reserveMemory. - static bool releaseMemory(void* addr, size_t size); + //! Allocate an aligned chunk of memory. static void* alignedMalloc(size_t size, size_t alignment); //! Deallocate an aligned chunk of memory. static void alignedFree(void* mem); + //! Platform-specific optimized memcpy() static void* fastMemcpy(void *dest, const void *src, size_t n); @@ -243,9 +253,12 @@ public: //! Skip an IDIV (F6/F7) instruction and return a pointer to the next insn. static bool skipIDIV(address& insn); - + // return gloabal memory size to be assigned to device info - static size_t getPhysicalMemSize(); + static size_t getPhysicalMemSize(); + + //! get Application file name + static std::string getAppFileName(); }; /*@}*/ diff --git a/rocclr/compiler/lib/promotions/oclutils/os/os_posix.cpp b/rocclr/compiler/lib/promotions/oclutils/os/os_posix.cpp index 97cb9ba02e..c29f73bcaa 100644 --- a/rocclr/compiler/lib/promotions/oclutils/os/os_posix.cpp +++ b/rocclr/compiler/lib/promotions/oclutils/os/os_posix.cpp @@ -6,6 +6,7 @@ #include "os/os.hpp" #include "thread/thread.hpp" +#include "utils/util.hpp" #include #include @@ -31,6 +32,7 @@ # define DT_GNU_HASH 0x6ffffef5 #endif // DT_GNU_HASH +#include #include #include #include @@ -38,6 +40,7 @@ #include #include // for tempnam #include +#include #ifdef ANDROID //#include @@ -95,30 +98,25 @@ divisionErrorHandler(int sig, siginfo_t* info, void* ptr) #if defined(ATI_ARCH_X86) insn = (address)uc->uc_mcontext.gregs[LP64_SWITCH(REG_EIP,REG_RIP)]; #else - assert(!"Unimplemented"); + assert(!"Unimplemented"); #endif + if(Thread::current()->isWorkerThread()) { + if (Os::skipIDIV(insn)) { +#if defined(ATI_ARCH_X86) + uc->uc_mcontext.gregs[LP64_SWITCH(REG_EIP,REG_RIP)] = (greg_t)insn; +#else + assert(!"Unimplemented"); +#endif + return; + } + } + // Call the chained signal handler if (callOldSignalHandler(sig, info, ptr)) { return; } - // @todo: only handle exception in the generated code. - // - //if (!isKernelCode(insn)) { - // return; - //} - - if (sig == SIGFPE && info->si_code == FPE_INTDIV) { - if (Os::skipIDIV(insn)) { -#if defined(ATI_ARCH_X86) - uc->uc_mcontext.gregs[LP64_SWITCH(REG_EIP,REG_RIP)] = (greg_t)insn; -#else - assert(!"Unimplemented"); -#endif - return; - } - } std::cerr << "Unhandled signal in divisionErrorHandler()" << std::endl; ::abort(); @@ -305,25 +303,76 @@ memProtToOsProt(Os::MemProt prot) } address -Os::reserveMemory(size_t size, MemProt prot) +Os::reserveMemory(address start, size_t size, size_t alignment, MemProt prot) { - address mem = (address) ::mmap(NULL, size, memProtToOsProt(prot), - MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + size = alignUp(size, pageSize()); + alignment = std::max(pageSize(), alignUp(alignment, pageSize())); + assert(isPowerOfTwo(alignment) && "not a power of 2"); - assert(mem != NULL && "out of memory"); - return mem; + size_t requested = size + alignment - pageSize(); + address mem = (address) ::mmap(start, requested, memProtToOsProt(prot), + MAP_PRIVATE | MAP_NORESERVE | MAP_ANONYMOUS, 0, 0); + + // check for out of memory + if (mem == NULL) return NULL; + + address aligned = alignUp(mem, alignment); + + // return the unused leading pages to the free state + if (&aligned[0] != &mem[0]) { + assert(&aligned[0] > &mem[0] && "check this code"); + if (::munmap(&mem[0], &aligned[0] - &mem[0]) != 0) { + assert(!"::munmap failed"); + } + } + // return the unused trailing pages to the free state + if (&aligned[size] != &mem[requested]) { + assert(&aligned[size] < &mem[requested] && "check this code"); + if (::munmap(&aligned[size], &mem[requested] - &aligned[size]) != 0) { + assert(!"::munmap failed"); + } + } + + return aligned; } -bool +bool Os::releaseMemory(void* addr, size_t size) { - // Needs to calculate the size and actual address. + assert(isMultipleOf(addr, pageSize()) && "not page aligned!"); + size = alignUp(size, pageSize()); + return 0 == ::munmap(addr, size); } +bool +Os::commitMemory(void* addr, size_t size, MemProt prot) +{ + assert(isMultipleOf(addr, pageSize()) && "not page aligned!"); + size = alignUp(size, pageSize()); + + return ::mmap(addr, size, memProtToOsProt(prot), + MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS, + -1, 0) != MAP_FAILED; +} + +bool +Os::uncommitMemory(void* addr, size_t size) +{ + assert(isMultipleOf(addr, pageSize()) && "not page aligned!"); + size = alignUp(size, pageSize()); + + return ::mmap(addr, size, PROT_NONE, + MAP_PRIVATE | MAP_FIXED | MAP_NORESERVE | MAP_ANONYMOUS, + -1, 0) != MAP_FAILED; +} + bool Os::protectMemory(void* addr, size_t size, MemProt prot) { + assert(isMultipleOf(addr, pageSize()) && "not page aligned!"); + size = alignUp(size, pageSize()); + return 0 == ::mprotect(addr, size, memProtToOsProt(prot)); } @@ -406,6 +455,12 @@ Thread::entry(Thread* thread) return thread->main(); } +bool +Os::isThreadAlive(const Thread& thread) +{ + return true; +} + const void* Os::createOsThread(amd::Thread* thread) { @@ -424,7 +479,7 @@ Os::createOsThread(amd::Thread* thread) // We never plan the use join, so free the resources now. ::pthread_attr_setdetachstate(&threadAttr, PTHREAD_CREATE_DETACHED); - pthread_t handle = (pthread_t)NULL; + pthread_t handle = 0; if (0 != ::pthread_create(&handle, &threadAttr, (void* (*)(void*)) &Thread::entry, thread)) { thread->setState(Thread::FAILED); @@ -705,20 +760,13 @@ Os::getTempPath() std::string Os::getTempFileName() { - std::string tempPath = getTempPath(); - char* tempBuf = ::tempnam(tempPath.c_str(), "OCL"); + static std::atomic_size_t counter(0); - if (tempBuf == NULL) { - static amd::Atomic counter = 0; + std::string tempPath = getTempPath(); + std::stringstream tempFileName; - std::stringstream ss; - ss << tempPath << "/OCL" << ::getpid() << 'T' << counter++; - return ss.str(); - } - - std::string tempFileName = tempBuf; - free(tempBuf); - return tempFileName; + tempFileName << tempPath << "/OCLC" << ::getpid() << 'T' << counter++; + return tempFileName.str(); } int @@ -823,6 +871,18 @@ size_t Os::getPhysicalMemSize() return (size_t) si.totalram * si.mem_unit; } +std::string Os::getAppFileName() +{ + std::unique_ptr buff(new char[FILE_PATH_MAX_LENGTH]()); + + if (readlink("/proc/self/exe", buff.get(), FILE_PATH_MAX_LENGTH) > 0) { + // Get filename without path and extension. + return std::string(basename(buff.get())); + } + + return ""; +} + } // namespace amd #endif // !defined(_WIN32) && !defined(__CYGWIN__) diff --git a/rocclr/compiler/lib/promotions/oclutils/os/os_win32.cpp b/rocclr/compiler/lib/promotions/oclutils/os/os_win32.cpp index f627ea8bf6..b07c4ed276 100644 --- a/rocclr/compiler/lib/promotions/oclutils/os/os_win32.cpp +++ b/rocclr/compiler/lib/promotions/oclutils/os/os_win32.cpp @@ -27,6 +27,8 @@ BOOL (WINAPI *pfnGetNumaNodeProcessorMaskEx)(USHORT,PGROUP_AFFINITY) = NULL; namespace amd { +static size_t allocationGranularity_; + static LONG WINAPI divExceptionFilter(struct _EXCEPTION_POINTERS* ep); #ifdef _WIN64 @@ -56,6 +58,7 @@ Os::init() SYSTEM_INFO si; ::GetSystemInfo(&si); pageSize_ = si.dwPageSize; + allocationGranularity_ = (size_t) si.dwAllocationGranularity; processorCount_ = si.dwNumberOfProcessors; LARGE_INTEGER frequency; @@ -285,26 +288,67 @@ memProtToOsProt(Os::MemProt prot) } address -Os::reserveMemory(size_t size, MemProt prot) +Os::reserveMemory(address start, size_t size, size_t alignment, MemProt prot) { - // Needs to be COMMITed otherwise the protection will fail. - return (address)VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); + size = alignUp(size, pageSize()); + alignment = std::max(allocationGranularity_, + alignUp(alignment, allocationGranularity_)); + assert(isPowerOfTwo(alignment) && "not a power of 2"); + + size_t requested = size + alignment - allocationGranularity_; + address mem, aligned; + do { + mem = (address)VirtualAlloc(start, requested, + MEM_RESERVE, memProtToOsProt(prot)); + + // check for out of memory. + if (mem == NULL) return NULL; + + aligned = alignUp(mem, alignment); + + // check for already aligned memory. + if (aligned == mem && size == requested) { + return mem; + } + + // try to reserve the aligned address. + if (VirtualFree(mem, 0, MEM_RELEASE) == 0) { + assert(!"VirtualFree failed"); + } + + mem = (address)VirtualAlloc(aligned, size, + MEM_RESERVE, memProtToOsProt(prot)); + assert((mem == NULL || mem == aligned) && "VirtualAlloc failed"); + + } while (mem != aligned); + + return mem; } -bool +bool Os::releaseMemory(void* addr, size_t size) { - BOOL error = VirtualFree(addr, 0, MEM_RELEASE); - return (error == 0) ? false : true; + return VirtualFree(addr, 0, MEM_RELEASE) != 0; } +bool +Os::commitMemory(void* addr, size_t size, MemProt prot) +{ + return VirtualAlloc(addr, size, + MEM_COMMIT, memProtToOsProt(prot)) != NULL; +} + +bool +Os::uncommitMemory(void* addr, size_t size) +{ + return VirtualFree(addr, size, MEM_DECOMMIT) != 0; +} bool Os::protectMemory(void* addr, size_t size, MemProt prot) { DWORD OldProtect; - BOOL error = VirtualProtect(addr, size, memProtToOsProt(prot), &OldProtect); - return (error == 0) ? false : true; + return VirtualProtect(addr, size, memProtToOsProt(prot), &OldProtect) != 0; } @@ -406,15 +450,11 @@ divExceptionFilter(struct _EXCEPTION_POINTERS* ep) { DWORD code = ep->ExceptionRecord->ExceptionCode; - if (code == EXCEPTION_INT_DIVIDE_BY_ZERO - || code == EXCEPTION_INT_OVERFLOW) { - // @todo: only handle exception in the generated code. - // - //if (!isKernelCode(insn)) { - // return; - //} - + if ((code == EXCEPTION_INT_DIVIDE_BY_ZERO || + code == EXCEPTION_INT_OVERFLOW) && + Thread::current()->isWorkerThread()) { address insn = (address)ep->ContextRecord->LP64_SWITCH(Eip,Rip); + if (Os::skipIDIV(insn)) { ep->ContextRecord->LP64_SWITCH(Eip,Rip) = (uintptr_t)insn; return EXCEPTION_CONTINUE_EXECUTION; @@ -427,25 +467,41 @@ void* Thread::entry(Thread* thread) { void* ret = NULL; - // @todo: We only need this for CPU worker threads. #if !defined(_WIN64) - if (true /*thread->isWorkerThread()*/) { - __try { - ret = thread->main(); - } - __except(divExceptionFilter(GetExceptionInformation())) { - // nothing to do here. - } - } - else { -#else // _WIN64 - { -#endif // _WIN64 + __try { ret = thread->main(); } + __except(divExceptionFilter(GetExceptionInformation())) { + // nothing to do here. + } +#else // _WIN64 + ret = thread->main(); +#endif // _WIN64 + + // The current thread exits, thus clear the pointer +#if defined(USE_DECLSPEC_THREAD) + details::thread_ = NULL; +#else // !USE_DECLSPEC_THREAD + TlsSetValue(details::threadIndex_, NULL); +#endif // !USE_DECLSPEC_THREAD return ret; } +bool +Os::isThreadAlive(const Thread& thread) +{ + HANDLE handle = (HANDLE)(thread.handle()); + + DWORD exitCode = 0; + if (GetExitCodeThread(handle, &exitCode)) { + return exitCode == STILL_ACTIVE; + } + else { + // Could not get thread's exitcode + return false; + } +} + const void* Os::createOsThread(Thread* thread) { @@ -653,14 +709,14 @@ Os::getTempPath() // under windows directory, use . instead std::string tempPathStr(tempPath); char winPath[MAX_PATH]; - ret = GetWindowsDirectory(winPath, MAX_PATH); - if (ret > 0) { - size_t len = strlen(winPath); - if (strlen(tempPath) >= len) { - tempPath[len] = 0; - if (_stricmp(tempPath, winPath) == 0) { - return std::string("."); - } + if (GetWindowsDirectory(winPath, MAX_PATH) > 0) { + // Need to check if tempPath is C:\Windows or C:\Windows\ // + if (tempPath[strlen(tempPath)-1] == '\\') { + tempPath[strlen(tempPath)-1] = '\0' ; + ret--; + } + if (_memicmp(tempPath, winPath, ret) == 0) { + return std::string("."); } } return tempPathStr; @@ -669,18 +725,13 @@ Os::getTempPath() std::string Os::getTempFileName() { - char tempBuf[MAX_PATH]; - std::string tempPath = getTempPath(); + static std::atomic_size_t counter(0); - if (0 == GetTempFileName(tempPath.c_str(), "OCL", 0, tempBuf)) { - static amd::Atomic counter = 0; + std::string tempPath = getTempPath(); + std::stringstream tempFileName; - std::stringstream ss; - ss << tempPath << "\\OCL" << ::_getpid() << 'T' << counter++; - return ss.str(); - } - - return tempBuf; + tempFileName << tempPath << "\\OCLC" << ::_getpid() << 'T' << counter++; + return tempFileName.str(); } int @@ -1013,6 +1064,19 @@ size_t Os::getPhysicalMemSize() return (size_t) statex.ullTotalPhys; } +std::string Os::getAppFileName() +{ + std::string strFileName; + char* buff = new char[FILE_PATH_MAX_LENGTH]; + + if (GetModuleFileNameA(NULL, buff, FILE_PATH_MAX_LENGTH) != 0) { + // Get filename without path and extension. + strFileName = strrchr(buff, '\\') ? strrchr(buff, '\\') + 1 : buff; + } + + delete buff; + return strFileName; +} } // namespace amd diff --git a/rocclr/compiler/lib/promotions/oclutils/thread/atomic.hpp b/rocclr/compiler/lib/promotions/oclutils/thread/atomic.hpp index 144f0b6dde..8477ece1e0 100644 --- a/rocclr/compiler/lib/promotions/oclutils/thread/atomic.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/thread/atomic.hpp @@ -13,7 +13,8 @@ #define ATOMIC_HPP_ #include "top.hpp" -#include "utils/traits.hpp" + +#include #ifdef _WIN32 # include @@ -21,64 +22,15 @@ # include # include #endif // !_WIN32 + +#include +#include + namespace amd { /*! \addtogroup Threads * @{ * - * \defgroup MemOrder Memory ordering - * @{ - */ - -/*! \brief Memory order access operations. - */ -class MemoryOrder : AllStatic -{ -public: - /*! \brief Execute a memory fence. - * - * Perform a serializing operation on loads and stores which guarantees - * that all memory operations dispatched prior to the fence will be - * globally visible before any other memory operation following the fence. - */ - static void fence() { -# if defined(ATI_ARCH_X86) - _mm_mfence(); -# else // !ATI_ARCH_X86 - __sync_synchronize(); -# endif // !ATI_ARCH_X86 - } - - /*! \brief Execute a loads fence. - * - * Perform a serializing operation on loads which guarantees that all - * load from memory operations dispatched prior to the lfence will be - * globally visible before any other load following the lfence. - */ - static void lfence() { -# if defined(ATI_ARCH_X86) - _mm_lfence(); -# else // !ATI_ARCH_X86 - fence(); -# endif // !ATI_ARCH_X86 - } - - /*! \brief Execute a stores fence. - * - * Perform a serializing operation on stores which guarantees that all - * store to memory operations dispatched prior to the sfence will be - * globally visible before any other store following the sfence. - */ - static void sfence() { -# if defined(ATI_ARCH_X86) - _mm_sfence(); -# else // !ATI_ARCH_X86 - fence(); -# endif // !ATI_ARCH_X86 - } -}; - -/*! @} * \addtogroup Atomic Atomic Operations * @{ */ @@ -149,7 +101,7 @@ public: * Atomically add \a inc to \a *dest and return the prior value. */ template - static T add(typename make_arithmetic::type inc, volatile T* dest) + static T add(T inc, volatile T* dest) { return Intrinsics::add((T) inc, dest); } @@ -200,7 +152,7 @@ public: * Atomically or \a mask to \a *dest and return the prior value. */ template - static T _or(typename make_arithmetic::type mask, volatile T* dest) + static T _or(T mask, volatile T* dest) { return Intrinsics::_or((T) mask, dest); } @@ -210,7 +162,7 @@ public: * Atomically or \a mask to \a *dest and return the prior value. */ template - static T _and(typename make_arithmetic::type mask, volatile T* dest) + static T _and(T mask, volatile T* dest) { return Intrinsics::_and((T) mask, dest); } @@ -422,8 +374,9 @@ class Atomic { private: - typedef typename add_volatile::type value_type; - value_type value_; //!< \brief The variable. + typedef typename std::remove_volatile::type>::type>::type value_type; + typename std::add_volatile::type value_; //!< \brief The variable. public: //! Construct a new %Atomic variable of type T. @@ -444,50 +397,51 @@ public: //! Return the %Atomic variable value. T operator ->() const { return T(value_); } //! Return the %Atomic variable's address. - typename add_pointer::type operator &() { return &value_; } + typename std::add_pointer::type>:: + type operator &() { return &value_; } //! Atomically add \a inc to this variable. - Atomic& operator += (typename make_arithmetic::type inc) + Atomic& operator += (value_type inc) { - if (is_pointer::value) { - inc *= sizeof(typename remove_pointer::type); + if (std::is_pointer::value) { + inc *= sizeof(typename std::remove_pointer::type); } AtomicOperation::add(inc, &value_); return *this; } //! Atomically subtract \a inc to this variable. - Atomic& operator -= (typename make_arithmetic::type inc) + Atomic& operator -= (value_type inc) { - typename make_arithmetic::type modifier = 0; - if (is_pointer::value) { - inc *= sizeof(typename remove_pointer::type); + value_type modifier = 0; + if (std::is_pointer::value) { + inc *= sizeof(typename std::remove_pointer::type); } AtomicOperation::add(modifier - inc, &value_); return *this; } //! Atomically OR \a value to this variable. - Atomic& operator |= (typename make_arithmetic::type mask) + Atomic& operator |= (value_type mask) { AtomicOperation::_or(mask, &value_); return *this; } //! Atomically AND \a value to this variable. - Atomic& operator &= (typename make_arithmetic::type mask) + Atomic& operator &= (value_type mask) { AtomicOperation::_and(mask, &value_); return *this; } //! Atomically increment this variable and return its new value. - typename remove_reference::type operator ++ () + typename std::remove_reference::type operator ++ () { - if (is_pointer::value) { - typename make_arithmetic::type inc = 1; - return AtomicOperation::add( - inc * sizeof(typename remove_pointer::type), &value_) + 1; + if (std::is_pointer::value) { + value_type inc = static_cast( + sizeof(typename std::remove_pointer::type)); + return AtomicOperation::add(inc, &value_) + 1; } else { return AtomicOperation::increment(&value_) + 1; @@ -495,12 +449,13 @@ public: } //! Atomically decrement this variable and return its new value. - typename remove_reference::type operator -- () + typename std::remove_reference::type operator -- () { - if (is_pointer::value) { - typename make_arithmetic::type inc = -1; - return AtomicOperation::add( - inc * sizeof(typename remove_pointer::type), &value_) - 1; + if (std::is_pointer::value) { + value_type inc = static_cast(- + static_cast::type>( + sizeof(typename std::remove_pointer::type))); + return AtomicOperation::add(inc, &value_) - 1; } else { return AtomicOperation::decrement(&value_) - 1; @@ -508,12 +463,12 @@ public: } //! Atomically increment this variable and return its previous value. - typename remove_reference::type operator ++ (int) + typename std::remove_reference::type operator ++ (int) { - if (is_pointer::value) { - typename make_arithmetic::type inc = 1; - return AtomicOperation::add( - inc * sizeof(typename remove_pointer::type), &value_); + if (std::is_pointer::value) { + value_type inc = static_cast( + sizeof(typename std::remove_pointer::type)); + return AtomicOperation::add(inc, &value_); } else { return AtomicOperation::increment(&value_); @@ -523,10 +478,11 @@ public: //! Atomically decrement this variable and return its previous value. T operator -- (int) { - if (is_pointer::value) { - typename make_arithmetic::type inc = -1; - return AtomicOperation::add( - inc * sizeof(typename remove_pointer::type), &value_); + if (std::is_pointer::value) { + value_type inc = static_cast(- + static_cast::type>( + sizeof(typename std::remove_pointer::type))); + return AtomicOperation::add(inc, &value_); } else { return AtomicOperation::decrement(&value_); @@ -556,7 +512,7 @@ public: */ void storeRelease(T value) { - MemoryOrder::fence(); + std::atomic_thread_fence(std::memory_order_release); value_ = value; } @@ -569,7 +525,7 @@ public: T loadAcquire() const { T value = value_; - MemoryOrder::fence(); + std::atomic_thread_fence(std::memory_order_acquire); return value; } }; @@ -583,83 +539,6 @@ make_atomic(T& t) } -template -class AtomicMarkableReference -{ -private: - static const intptr_t kMarkBitMask = 0x1; - -private: - Atomic reference_; - -private: - static intptr_t markMask(bool mark) - { - return mark ? kMarkBitMask : intptr_t(0); - } - -public: - AtomicMarkableReference() - : reference_(NULL) - { } - - AtomicMarkableReference(T* ptr, bool mark = false) - : reference_((T*)((intptr_t) ptr | markMask(mark))) - { } - - bool compareAndSet( - T* expectedPtr, T* newPtr, - bool expectedMark, bool newMark) - { - return reference_.compareAndSet( - (T*)((intptr_t) expectedPtr | markMask(expectedMark)), - (T*)((intptr_t) newPtr | markMask(newMark))); - } - - pair swap(T* newPtr, bool newMark) - { - T* prev = reference_.swap( - (T*)((intptr_t) newPtr | markMask(newMark))); - return make_pair( - (T*) ((intptr_t) prev & ~kMarkBitMask), - ((intptr_t) prev & kMarkBitMask) != 0); - } - - bool tryMark(T* expectedPtr, bool newMark) - { - T* current = reference_; - if (((intptr_t) current & ~kMarkBitMask) != (intptr_t) expectedPtr) { - return false; - } - bool currentMark = ((intptr_t) current & kMarkBitMask) != 0; - return currentMark == newMark || reference_.compareAndSet(current, - (T*)((intptr_t) expectedPtr | markMask(newMark))); - } - - bool isMarked() const - { - return ((intptr_t)(T*) reference_ & kMarkBitMask) != 0; - } - - pair get() const - { - T* current = reference_; - return make_pair( - (T*) ((intptr_t) current & ~kMarkBitMask), - ((intptr_t) current & kMarkBitMask) != 0); - } - - T* getReference() const - { - return (T*) ((intptr_t)(T*) reference_ & ~kMarkBitMask); - } - - void set(T* ptr, bool mark) - { - reference_ = (T*)((intptr_t) ptr | markMask(mark)); - } -}; - /*! @} * @} */ diff --git a/rocclr/compiler/lib/promotions/oclutils/thread/monitor.cpp b/rocclr/compiler/lib/promotions/oclutils/thread/monitor.cpp index 571d5e2a18..9a3d7de433 100644 --- a/rocclr/compiler/lib/promotions/oclutils/thread/monitor.cpp +++ b/rocclr/compiler/lib/promotions/oclutils/thread/monitor.cpp @@ -9,11 +9,13 @@ #include "utils/util.hpp" #include +#include +#include namespace amd { Monitor::Monitor(const char* name, bool recursive) : - contendersList_(NULL), onDeck_(NULL), waitersList_(NULL), + contendersList_(0), onDeck_(0), waitersList_(NULL), owner_(NULL), recursive_(recursive) { const size_t maxNameLen = sizeof(name_); @@ -66,19 +68,17 @@ Monitor::finishLock() /* The lock is contended. Push the thread's semaphore onto * the contention list. */ - Semaphore& sem = thread->lockSemaphore(); - sem.reset(); + Semaphore& semaphore = thread->lockSemaphore(); + semaphore.reset(); LinkedNode newHead; - newHead.setItem(&sem); - - while (true) { - LinkedNode* head; bool isLocked; + newHead.setItem(&semaphore); + intptr_t head = contendersList_.load(std::memory_order_acquire); + for (;;) { // The assumption is that lockWord is locked. Make sure we do not // continue unless the lock bit is set. - tie(head, isLocked) = contendersList_.get(); - if (!isLocked) { + if ((head & kLockBit) == 0) { if (tryLock()) { return; } @@ -86,8 +86,10 @@ Monitor::finishLock() } // Set the new contention list head if lockWord is unchanged. - newHead.setNext(head); - if (contendersList_.compareAndSet(head, &newHead, kLocked, kLocked)) { + newHead.setNext(reinterpret_cast(head & ~kLockBit)); + if (contendersList_.compare_exchange_weak(head, + reinterpret_cast(&newHead) | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) { break; } @@ -97,7 +99,7 @@ Monitor::finishLock() int32_t spinCount = 0; // Go to sleep until we become the on-deck thread. - while (onDeck_.getReference() != &sem) { + while ((onDeck_ & ~kLockBit) != reinterpret_cast(&semaphore)) { // First, be SMT friendly if (spinCount < kMaxReadSpinIter) { Os::spinPause(); @@ -108,7 +110,7 @@ Monitor::finishLock() } // now go to sleep else { - sem.wait(); + semaphore.wait(); } spinCount++; } @@ -118,8 +120,9 @@ Monitor::finishLock() // From now-on, we are the on-deck thread. It will stay that way until // we successfuly acquire the lock. // - while (true) { - assert(onDeck_.getReference() == &sem && "just checking"); + for (;;) { + assert((onDeck_ & ~kLockBit) == reinterpret_cast(&semaphore) + && "just checking"); if (tryLock()) { break; } @@ -136,13 +139,13 @@ Monitor::finishLock() } // now go to sleep else { - sem.wait(); + semaphore.wait(); } spinCount++; } assert(newHead.next() == NULL && "Should not be linked"); - onDeck_ = NULL; + onDeck_ = 0; } void @@ -152,53 +155,58 @@ Monitor::finishUnlock() // list waiting to acquire the lock. We need to select a successor and // place it on-deck. - while (true) { + for (;;) { // Grab the onDeck_ microlock to protect the next loop (make sure only // one semaphore is removed from the contention list). // - if (!onDeck_.compareAndSet(NULL, NULL, kUnlocked, kLocked)) { + intptr_t ptr = 0; + if (!onDeck_.compare_exchange_strong(ptr, ptr | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) { return; // Somebody else has the microlock, let him select onDeck_ } - LinkedNode* head; bool isLocked; - while (true) { - tie(head, isLocked) = contendersList_.get(); - - if (head == NULL) { + intptr_t head = contendersList_.load(std::memory_order_acquire); + for (;;) { + if (head == 0) { break; // There's nothing else to do. } - if (isLocked) { + if ((head & kLockBit) != 0) { // Somebody could have acquired then released the lock // and failed to grab the onDeck_ microlock. - head = NULL; + head = 0; break; } - if (contendersList_.compareAndSet( - head, head->next(), kUnlocked, kUnlocked)) { + if (contendersList_.compare_exchange_weak( + head, reinterpret_cast( + reinterpret_cast(head)->next()), + std::memory_order_acq_rel, std::memory_order_acquire)) { #ifdef ASSERT - head->setNext(NULL); + reinterpret_cast(head)->setNext(NULL); #endif // ASSERT break; } } - Semaphore* sem = (head != NULL) ? head->item() : NULL; - onDeck_ = sem; - MemoryOrder::fence(); + Semaphore* semaphore = (head != 0) + ? reinterpret_cast(head)->item() + : NULL; + + onDeck_.store(reinterpret_cast(semaphore), + std::memory_order_release); // // Release the onDeck_ microlock (end of critical region); - if (sem != NULL) { - sem->post(); + if (semaphore != NULL) { + semaphore->post(); return; } - // We do not have an on-deck thread (sem == NULL). Return if + // We do not have an on-deck thread (semaphore == NULL). Return if // the contention list is empty or if the lock got acquired again. - tie(head, isLocked) = contendersList_.get(); - if (isLocked || head == NULL) { + head = contendersList_; + if (head == 0 || (head & kLockBit) != 0) { return; } } @@ -228,7 +236,7 @@ Monitor::wait() // Go to sleep until we become the on-deck thread. int32_t spinCount = 0; - while (onDeck_.getReference() != &suspend) { + while ((onDeck_ & ~kLockBit) != reinterpret_cast(&suspend)) { // First, be SMT friendly if (spinCount < kMaxReadSpinIter) { Os::spinPause(); @@ -245,8 +253,9 @@ Monitor::wait() } spinCount = 0; - while (true) { - assert(onDeck_.getReference() == &suspend && "just checking"); + for (;;) { + assert((onDeck_ & ~kLockBit) == reinterpret_cast(&suspend) + && "just checking"); if (trySpinLock()) { break; @@ -272,8 +281,7 @@ Monitor::wait() // Restore the lock count (for recursive mutexes) lockCount_ = lockCount; - onDeck_ = NULL; - MemoryOrder::fence(); + onDeck_.store(0, std::memory_order_release); } void @@ -288,11 +296,13 @@ Monitor::notify() // Dequeue a waiter from the wait list and add it to the contention list. waitersList_ = waiter->next(); - while (true) { - LinkedNode* node = contendersList_.getReference(); - waiter->setNext(node); - if (contendersList_.compareAndSet(node, waiter, kLocked, kLocked)) { + intptr_t node = contendersList_.load(std::memory_order_acquire); + for (;;) { + waiter->setNext(reinterpret_cast(node & ~kLockBit)); + if (contendersList_.compare_exchange_weak(node, + reinterpret_cast(waiter) | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) { break; } } diff --git a/rocclr/compiler/lib/promotions/oclutils/thread/monitor.hpp b/rocclr/compiler/lib/promotions/oclutils/thread/monitor.hpp index 5525a18dfa..fac75b72d8 100644 --- a/rocclr/compiler/lib/promotions/oclutils/thread/monitor.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/thread/monitor.hpp @@ -6,10 +6,14 @@ #define MONITOR_HPP_ #include "top.hpp" -#include "atomic.hpp" +#include "thread/atomic.hpp" #include "thread/semaphore.hpp" #include "thread/thread.hpp" +#include +#include +#include + namespace amd { /*! \addtogroup Threads @@ -19,13 +23,46 @@ namespace amd { * @{ */ +namespace details { + +template +struct SimplyLinkedNode : public AllocClass +{ + typedef SimplyLinkedNode Node; + +protected: + std::atomic next_; /*!< \brief The next element. */ + T volatile item_; + +public: + //! \brief Return the next element in the linked-list. + Node* next() const { return next_; } + //! \brief Return the item. + T item() const { return item_; } + + //! \brief Set the next element pointer. + void setNext(Node* next) { next_ = next; } + //! \brief Set the item. + void setItem(T item) { item_ = item; } + + //! \brief Swap the next element pointer. + Node* swapNext(Node* next) { return next_.swap(next); } + + //! \brief Compare and set the next element pointer. + bool compareAndSetNext(Node* compare, Node* next) + { + return next_.compare_exchange_strong(compare, next); + } +}; + +} // namespace details + class Monitor : public HeapObject { - typedef SimplyLinkedNode LinkedNode; + typedef details::SimplyLinkedNode LinkedNode; private: - static const bool kUnlocked = false; - static const bool kLocked = true; + static const intptr_t kLockBit = 0x1; static const int kMaxSpinIter = 55; //!< Total number of spin iterations. static const int kMaxReadSpinIter = 50; //!< Read iterations before yielding @@ -33,12 +70,12 @@ private: /*! Linked list of semaphores the contending threads are waiting on * and main lock. */ - AtomicMarkableReference contendersList_; + std::atomic_intptr_t contendersList_; //! The Mutex's name char name_[64]; //! Semaphore of the next thread to contend for the lock. - AtomicMarkableReference onDeck_; + std::atomic_intptr_t onDeck_; //! Linked list of the suspended threads resume semaphores. LinkedNode* volatile waitersList_; @@ -63,7 +100,7 @@ protected: * * \note The user is responsible for the memory ordering. */ - bool isLocked() const { return contendersList_.isMarked(); } + bool isLocked() const { return (contendersList_ & kLockBit) != 0; } //! Return this monitor's owner thread (NULL if unlocked). Thread* owner() const { return owner_; } @@ -141,10 +178,9 @@ Monitor::tryLock() Thread* thread = Thread::current(); assert(thread != NULL && "cannot lock() from (null)"); - LinkedNode* ptr; bool isLocked; - tie(ptr, isLocked) = contendersList_.get(); + intptr_t ptr = contendersList_.load(std::memory_order_acquire); - if (unlikely(isLocked)) { + if (unlikely((ptr & kLockBit) != 0)) { if (recursive_ && thread == owner_) { // Recursive lock: increment the lock count and return. ++lockCount_; @@ -153,8 +189,8 @@ Monitor::tryLock() return false; // Already locked! } - if (unlikely(!contendersList_.compareAndSet( - ptr, ptr, kUnlocked, kLocked))) { + if (unlikely(!contendersList_.compare_exchange_weak(ptr, ptr | kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire))) { return false; // We failed the CAS from unlocked to locked. } @@ -189,23 +225,21 @@ Monitor::unlock() setOwner(NULL); - while (true) { - LinkedNode* ptr = contendersList_.getReference(); - // Clear the lock bit. - if (contendersList_.compareAndSet(ptr, ptr, kLocked, kUnlocked)) { - break; // We succeeded the CAS from locked to unlocked. - } - } + // Clear the lock bit. + intptr_t ptr = contendersList_.load(std::memory_order_acquire); + while (!contendersList_.compare_exchange_weak(ptr, ptr & ~kLockBit, + std::memory_order_acq_rel, std::memory_order_acquire)) + ; // + // We succeeded the CAS from locked to unlocked. // This is the end of the critical region. // Check if we have an on-deck thread that needs signaling. - Semaphore* onDeck; bool isMarked; - tie(onDeck, isMarked) = onDeck_.get(); - if (onDeck != NULL) { - if (!isMarked) { + intptr_t onDeck = onDeck_; + if (onDeck != 0) { + if ((onDeck & kLockBit) == 0) { // Only signal if it is unmarked. - onDeck->post(); + reinterpret_cast(onDeck)->post(); } return; // We are done. } @@ -215,9 +249,8 @@ Monitor::unlock() // so return if the list is empty or if the lock got acquired again (it's // somebody else's problem now!) - LinkedNode* head; bool isLocked; - amd::tie(head, isLocked) = contendersList_.get(); - if (isLocked || head == NULL) { + intptr_t head = contendersList_; + if (head == 0 || (head & kLockBit) != 0) { return; } diff --git a/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.cpp b/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.cpp index f6d1a0955b..235ca45d9a 100644 --- a/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.cpp +++ b/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.cpp @@ -3,7 +3,6 @@ // #include "thread/semaphore.hpp" -#include "thread/atomic.hpp" #include "thread/thread.hpp" #if defined(_WIN32) || defined(__CYGWIN__) @@ -16,8 +15,8 @@ namespace amd { Semaphore::Semaphore() - : state_(0) { + std::atomic_init(&state_, 0); #ifdef _WIN32 handle_ = static_cast(CreateSemaphore(NULL, 0, LONG_MAX, NULL)); assert(handle_ != NULL && "CreateSemaphore failed"); @@ -44,18 +43,18 @@ Semaphore::~Semaphore() void Semaphore::post() { - int state; - while (true) { - state = state_; + int state = state_.load(std::memory_order_relaxed); + for (;;) { if (state > 0) { - // Do a load acquire. - MemoryOrder::fence(); - if (state == state_) { + int newstate = state_.load(std::memory_order_acquire); + if (state == newstate) { return; } + state = newstate; continue; } - if (state_.compareAndSet(state, state+1)) { + if (state_.compare_exchange_weak(state, state+1, + std::memory_order_acq_rel, std::memory_order_acquire)) { break; } } diff --git a/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.hpp b/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.hpp index ede63ff2b5..93c8c0f209 100644 --- a/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/thread/semaphore.hpp @@ -6,9 +6,9 @@ #define SEMAPHORE_HPP_ #include "top.hpp" -#include "thread/atomic.hpp" #include "utils/util.hpp" +#include #if defined(__linux__) # include #endif /*linux*/ @@ -29,14 +29,14 @@ class Thread; class Semaphore : public HeapObject { private: - Atomic state_; //!< This semaphore's value. + std::atomic_int state_; //!< This semaphore's value. #ifdef _WIN32 void* handle_; //!< The semaphore object's handle. - char padding_[64-sizeof(void*)-sizeof(Atomic)]; + char padding_[64-sizeof(void*)-sizeof(std::atomic_int)]; #else // !_WIN32 sem_t sem_; //!< The semaphore object's identifier. - char padding_[64-sizeof(sem_t)-sizeof(Atomic)]; + char padding_[64-sizeof(sem_t)-sizeof(std::atomic_int)]; #endif /*!_WIN32*/ public: @@ -52,7 +52,7 @@ public: //! \brief Reset this semaphore. void reset() { - state_.swap(0); + state_.store(0, std::memory_order_release); } }; diff --git a/rocclr/compiler/lib/promotions/oclutils/thread/thread.hpp b/rocclr/compiler/lib/promotions/oclutils/thread/thread.hpp index 11dad2b562..c2b5ca274c 100644 --- a/rocclr/compiler/lib/promotions/oclutils/thread/thread.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/thread/thread.hpp @@ -103,9 +103,6 @@ protected: size_t stackSize = 0 /*use system default*/, bool spawn = true /* create a new Os::thread */); - //! Destroy this thread. - virtual ~Thread(); - public: //! Return the currently running thread instance. static inline Thread* current(); @@ -116,6 +113,9 @@ public: //! Tear down the OsThread package. static void tearDown(); + //! Destroy this thread. + virtual ~Thread(); + //! Return the thread's name const std::string& name() const { return name_; } @@ -131,6 +131,9 @@ public: //! Return true is this is the host thread. virtual bool isHostThread() const { return false; } + //! Return true if this is a worker thread. + virtual bool isWorkerThread() const { return false; } + //! Get the current thread state. ThreadState state() const { return state_; } diff --git a/rocclr/compiler/lib/promotions/oclutils/utils/debug.hpp b/rocclr/compiler/lib/promotions/oclutils/utils/debug.hpp index d0302afeaa..c5cb91f8f0 100644 --- a/rocclr/compiler/lib/promotions/oclutils/utils/debug.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/utils/debug.hpp @@ -179,15 +179,4 @@ do { \ #define LogPrintfWarning(format, ...) Logf(amd::LOG_WARNING, format, __VA_ARGS__) #define LogPrintfInfo(format, ...) Logf(amd::LOG_INFO, format, __VA_ARGS__) -//! Used by objects that have split constructors for sanity-checking -//! their construction state - -enum ConstructionState { - CS_CONSTRUCTED, //!< alloc (regular) constructor called ok - CS_ACTIVE, //!< create constructor called ok - CS_DYING, //!< in destructor (use to trap races) - CS_BROKEN, //!< something went wrong during construction -}; - #endif /*DEBUG_HPP_*/ - diff --git a/rocclr/compiler/lib/promotions/oclutils/utils/macros.hpp b/rocclr/compiler/lib/promotions/oclutils/utils/macros.hpp index e7765f8384..d3cb6a2b60 100644 --- a/rocclr/compiler/lib/promotions/oclutils/utils/macros.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/utils/macros.hpp @@ -166,6 +166,12 @@ # define ALWAYSINLINE #endif // !_MSC_VER +#ifdef BRAHMA +# define IS_BRAHMA true +#else +# define IS_BRAHMA false +#endif + //! \endcond #endif // MACROS_HPP_ diff --git a/rocclr/compiler/lib/promotions/oclutils/utils/traits.hpp b/rocclr/compiler/lib/promotions/oclutils/utils/traits.hpp deleted file mode 100644 index c63f936b8e..0000000000 --- a/rocclr/compiler/lib/promotions/oclutils/utils/traits.hpp +++ /dev/null @@ -1,108 +0,0 @@ -// -// Copyright (c) 2008 Advanced Micro Devices, Inc. All rights reserved. -// - -#ifndef TRAITS_HPP_ -#define TRAITS_HPP_ - -namespace amd { - -// Type traits: - -//! \cond ignore -template -struct is_pointer -{ static const bool value = false; }; - -template -struct is_pointer -{ static const bool value = true; }; - -template -struct remove_reference -{ typedef T type; }; - -template -struct remove_reference -{ typedef T type; }; - -template -struct remove_volatile -{ typedef T type; }; - -template -struct remove_volatile -{ typedef T type; }; - -template -struct remove_const -{ typedef T type; }; - -template -struct remove_const -{ typedef T type; }; - -template -struct remove_pointer -{ typedef T type; }; - -template -struct remove_pointer -{ typedef T type; }; - -template -struct add_const -{ typedef T const type; }; - -template -struct add_const -{ typedef T& type; }; - -template -struct add_volatile -{ typedef T volatile type; }; - -template -struct add_volatile -{ typedef T& type; }; - -template -struct add_pointer -{ typedef typename remove_reference::type* type; }; - -template -struct add_reference -{ typedef typename remove_reference::type& type; }; - -template <> -struct add_reference -{ typedef void type; }; - -template <> -struct add_reference -{ typedef const void type; }; - -template <> -struct add_reference -{ typedef volatile void type; }; - -template <> -struct add_reference -{ typedef const volatile void type; }; - -template -struct make_arithmetic -{ typedef typename remove_volatile::type type; }; - -template -struct make_arithmetic -{ typedef long int type; }; - -template -struct make_arithmetic -{ typedef typename make_arithmetic::type type; }; -//! \endcond - -} // namespace amd - -#endif /* TRAITS_HPP_ */ diff --git a/rocclr/compiler/lib/promotions/oclutils/utils/util.hpp b/rocclr/compiler/lib/promotions/oclutils/utils/util.hpp index 2ff53a302c..efe0d41288 100644 --- a/rocclr/compiler/lib/promotions/oclutils/utils/util.hpp +++ b/rocclr/compiler/lib/promotions/oclutils/utils/util.hpp @@ -6,8 +6,8 @@ #define UTIL_HPP_ #include "top.hpp" -#include "thread/atomic.hpp" +#include #include namespace amd { @@ -16,307 +16,6 @@ namespace amd { * @{ */ -//! \cond ignore -template -struct PairElement; - -template <> -struct PairElement<0> -{ - template - static inline F& get(pair& p) { return p.first; } - template - static inline const F& get(const pair& p) { return p.first; } -}; - -template <> -struct PairElement<1> -{ - template - static inline S& get(pair& p) { return p.second; } - template - static inline const S& get(const pair& p) { return p.second; } -}; - -// Forward declaration of the tuple_elements container class. -template -struct TupleElementsContainer; - -/*! \brief Return the type of the Nth element in the tuple. - */ -template -struct TupleElementType -{ - typedef typename T::tail_t next_element; - typedef typename TupleElementType::type type; -}; - -// break the recursion -template -struct TupleElementType<0,T> -{ - typedef Null next_element; - typedef typename T::head_t type; -}; - -/*! \brief Helper struct to extract the Nth element from a tuple - */ -template -struct TupleElementGetter -{ - template - static R get(TupleElementsContainer& t) - { - return TupleElementGetter::template get(t.tail); - } - template - static R get(const TupleElementsContainer& t) - { - return TupleElementGetter::template get(t.tail); - } -}; - -// break the recursion -template <> -struct TupleElementGetter<0> -{ - template - static R get(TupleElementsContainer& t) - { - return t.head; - } - template - static R get(const TupleElementsContainer& t) - { - return t.head; - } -}; - -/*! \brief Return the Nth element in the tuple. - */ -template -inline typename TupleElementType >::type& -getTupleElement(TupleElementsContainer& t) -{ - return TupleElementGetter::template get< - typename TupleElementType >::type&, - H,T>(t); -} - -template -inline const typename TupleElementType >::type& -getTupleElement(const TupleElementsContainer& t) -{ - return TupleElementGetter::template get< - const typename TupleElementType >::type&, - H,T>(t); -} - -/*! \brief The tuple elements struct - */ -template -struct TupleElementsContainer -{ - typedef H head_t; - typedef T tail_t; - - head_t head; tail_t tail; - - TupleElementsContainer() : head(), tail() { } - - template - TupleElementsContainer(T0& t0, T1& t1, T2& t2, T3& t3) - : head(t0), tail(t1, t2, t3, null()) - { } - - template - TupleElementsContainer& operator= (const TupleElementsContainer& t) - { - head = t.head; - tail = t.tail; - return *this; - } - - template - TupleElementsContainer& operator= (const pair& p) - { - head = p.first; - tail.head = p.second; - return *this; - } - - template - typename TupleElementType::type& - get() { return getTupleElement(*this); } -}; - -// break the recursion -template -struct TupleElementsContainer -{ - typedef H head_t; - typedef Null tail_t; - - H head; - - TupleElementsContainer() : head() { } - - template - TupleElementsContainer(T0& t0, const Null&, const Null&, const Null&) - : head(t0) - { } - - template - TupleElementsContainer& operator = ( - const TupleElementsContainer& t) - { - head = t.head; - return *this; - } - - template - typename TupleElementType::type& - get() { return getTupleElement(*this); } -}; - -/*! \brief Rebind the TupleElementsContainer type. - */ -template -struct TupleElementsBinder -{ - typedef TupleElementsContainer< - T0, typename TupleElementsBinder::type - > type; -}; - -// break the recursion -template<> -struct TupleElementsBinder -{ typedef Null type; }; -//! \endcond - -/*! \brief A simple N-element (1 to 4) tuple. - */ -template -class tuple : public TupleElementsBinder::type -{ -private: - typedef typename TupleElementsBinder::type base_t; - -public: - tuple() { } - tuple(T0 t0) : base_t(t0, null(), null(), null()) { } - tuple(T0 t0, T1 t1) : base_t(t0, t1, null(), null()) { } - tuple(T0 t0, T1 t1, T2 t2) : base_t(t0, t1, t2, null()) { } - tuple(T0 t0, T1 t1, T2 t2, T3 t3) : base_t(t0, t1, t2, t3) { } - - template - tuple(const TupleElementsContainer& te) : base_t(te) - { } - - template - tuple& operator = (const TupleElementsContainer& te) - { - base_t::operator = (te); - return *this; - } - - template - tuple& operator = (const pair& p) - { - base_t::operator = (p); - return *this; - } -}; - -// tuple / pair element getters. - -template -inline typename TupleElementType >::type& -get(TupleElementsContainer& te) -{ - return getTupleElement(te); -} - -template -inline const typename TupleElementType >::type& -get(const TupleElementsContainer& te) -{ - return getTupleElement(te); -} - -template -inline typename TupleElementType >::type& -get(pair& p) -{ - return PairElement::get(p); -} - -template -inline const typename TupleElementType >::type& -get(const pair& p) -{ - return PairElement::get(p); -} - -// Some tuple helpers (make_tuple() and tie()) - -template -inline tuple -make_tuple(const T0& t0) -{ - return tuple(t0); -} - -template -inline tuple -make_tuple(const T0& t0, const T1& t1) -{ - return tuple(t0, t1); -} - -template -inline tuple -make_tuple(const T0& t0, const T1& t1, const T2& t2) -{ - return tuple(t0, t1, t2); -} - -template -inline tuple -make_tuple(const T0& t0, const T1& t1, const T2& t2, const T3& t3) -{ - return tuple(t0, t1, t2, t3); -} - -template -inline tuple -tie(T0& t0) -{ - return tuple(t0); -} - -template -inline tuple -tie(T0& t0, T1& t1) -{ - return tuple(t0, t1); -} - -template -inline tuple -tie(T0& t0, T1& t1, T2& t2) -{ - return tuple(t0, t1, t2); -} - -template -inline tuple -tie(T0& t0, T1& t1, T2& t2, T3& t3) -{ - return tuple(t0, t1, t2, t3); -} - //! \brief Check if the given value \a val is a power of 2. template static inline bool @@ -471,94 +170,22 @@ alignUp(T* value, size_t alignment) return (T*) alignDown((intptr_t) (value + alignment - 1), alignment); } -template -struct SimplyLinkedNode : public AllocClass +template +inline bool isMultipleOf(T value, size_t alignment) { - typedef SimplyLinkedNode Node; - -protected: - Atomic next_; /*!< \brief The next element. */ - T volatile item_; - -public: - //! \brief Return the next element in the linked-list. - Node* next() const { return next_; } - //! \brief Return the item. - T item() const { return item_; } - - //! \brief Set the next element pointer. - void setNext(Node* next) { next_ = next; } - //! \brief Set the item. - void setItem(T item) { item_ = item; } - - //! \brief Swap the next element pointer. - Node* swapNext(Node* next) { return next_.swap(next); } - - //! \brief Compare and set the next element pointer. - bool compareAndSetNext(Node* compare, Node* next) - { - return next_.compareAndSet(compare, next); + if (isPowerOfTwo(alignment)) { + // fast path, using logical operators + return alignUp(value, alignment) == value; } -}; + return value % alignment == 0; +} -/* For the implementation of a doubly-linked list, check: - * Lock-Free and Practical - * Deques and Doubly Linked - * Lists using Single-Word - * Compare-And-Swap - * - * Hakan Sundell, Philippas Tsigas - * Department of Computing Science - * Chalmers Univ. of Technol. and Goteborg Univ. - */ - -template -struct DoublyLinkedNode +template +inline bool isMultipleOf(T* value, size_t alignment) { - typedef SimplyLinkedNode Node; - -protected: - Atomic prev_; //!< The previous element. - Atomic next_; //!< The next element. - T volatile item_; - -public: - //! \brief Return the previous element in the linked-list. - Node* prev() const { return prev_; } - //! \brief Return the next element in the linked-list. - Node* next() const { return next_; } - //! \brief Return the item. - T item() const { return item_; } - - //! \brief Set the previous element pointer. - void setPrev(Node* prev) { prev_ = prev; } - //! \brief Set the next element pointer. - void setNext(Node* next) { next_ = next; } - //! \brief Set the item. - void setItem(T item) { item_ = item; } - - //! \brief Swap the previous element pointer. - Node* swapPrev(Node* prev) - { - return prev_.swap(prev); - } - //! \brief Swap the next element pointer. - Node* swapNext( Node* next) - { - return next_.swap(next); - } - - //! \brief Compare and set the previous element pointer. - bool compareAndSetPrev(Node* compare, Node* prev) - { - return prev_.compareAndSet(compare, prev, false, false); - } - //! \brief Compare and set the next element pointer. - bool compareAndSetNext(Node* compare, Node* next) - { - return next_.compareAndSet(compare, next, false, false); - } -}; + intptr_t ptr = reinterpret_cast(value); + return isMultipleOf(ptr, alignment); +} template struct DeviceMap { @@ -642,6 +269,13 @@ inline uint leastBitSet(T value) leastBitSet32((uint32_t)value); } +static inline bool Is32Bits() { + return LP64_SWITCH(true, false); +} + +static inline bool Is64Bits() { + return LP64_SWITCH(false, true); +} /*@}*/} // namespace amd #endif /*UTIL_HPP_*/