AI/rocm-systems
2
0
Fork 0
Koodi Ongelmat Pull-pyynnöt Actions Paketit Projektit Julkaisut Wiki Toiminta

P4 to Git Change 1082964 by lmoriche@lmoriche_opencl_dev on 2014/09/30 17:09:28

Selaa lähdekoodia 
ECR #304775 - Replace amd::Atomic with std::atomic (cont'd)

	Pre-checkin: http://ocltc.amd.com:8111/viewModification.html?modId=40674&personal=true&buildTypeId=&tab=vcsModificationBuilds&show_all_builds=true

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/thread/atomic.hpp#6 edit
... //depot/stg/opencl/drivers/opencl/runtime/thread/monitor.cpp#6 edit
... //depot/stg/opencl/drivers/opencl/runtime/thread/monitor.hpp#7 edit
... //depot/stg/opencl/drivers/opencl/runtime/utils/concurrent.hpp#7 edit


[ROCm/clr commit: d780c32a1b]
This commit is contained in:
foreman
2014-09-30 17:40:37 -04:00
vanhempi 5f45d44db6
commit 8f6983d69b
4 muutettua tiedostoa jossa 78 lisäystä ja 204 poistoa
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/rocclr/runtime/thread/atomic.hpp
+3 -132
Näytä tiedosto
@@ -23,6 +23,7 @@
# include <xmmintrin.h>
#endif // !_WIN32
#include <atomic>
#include <utility>
namespace amd {
@@ -30,59 +31,6 @@ namespace amd {
/*! \addtogroup Threads
* @{
*
* \defgroup MemOrder Memory ordering
* @{
*/
/*! \brief Memory order access operations.
*/
class MemoryOrder : AllStatic
{
public:
/*! \brief Execute a memory fence.
*
* Perform a serializing operation on loads and stores which guarantees
* that all memory operations dispatched prior to the fence will be
* globally visible before any other memory operation following the fence.
*/
static void fence() {
# if defined(ATI_ARCH_X86)
_mm_mfence();
# else // !ATI_ARCH_X86
__sync_synchronize();
# endif // !ATI_ARCH_X86
}
/*! \brief Execute a loads fence.
*
* Perform a serializing operation on loads which guarantees that all
* load from memory operations dispatched prior to the lfence will be
* globally visible before any other load following the lfence.
*/
static void lfence() {
# if defined(ATI_ARCH_X86)
_mm_lfence();
# else // !ATI_ARCH_X86
fence();
# endif // !ATI_ARCH_X86
}
/*! \brief Execute a stores fence.
*
* Perform a serializing operation on stores which guarantees that all
* store to memory operations dispatched prior to the sfence will be
* globally visible before any other store following the sfence.
*/
static void sfence() {
# if defined(ATI_ARCH_X86)
_mm_sfence();
# else // !ATI_ARCH_X86
fence();
# endif // !ATI_ARCH_X86
}
};
/*! @}
* \addtogroup Atomic Atomic Operations
* @{
*/
@@ -564,7 +512,7 @@ public:
*/
void storeRelease(T value)
{
MemoryOrder::fence();
std::atomic_thread_fence(std::memory_order_release);
value_ = value;
}
@@ -577,7 +525,7 @@ public:
T loadAcquire() const
{
T value = value_;
MemoryOrder::fence();
std::atomic_thread_fence(std::memory_order_acquire);
return value;
}
};
@@ -591,83 +539,6 @@ make_atomic(T& t)
}
template <typename T>
class AtomicMarkableReference
{
private:
static const intptr_t kMarkBitMask = 0x1;
private:
Atomic<T*> reference_;
private:
static intptr_t markMask(bool mark)
{
return mark ? kMarkBitMask : intptr_t(0);
}
public:
AtomicMarkableReference()
: reference_(NULL)
{ }
AtomicMarkableReference(T* ptr, bool mark = false)
: reference_((T*)((intptr_t) ptr | markMask(mark)))
{ }
bool compareAndSet(
T* expectedPtr, T* newPtr,
bool expectedMark, bool newMark)
{
return reference_.compareAndSet(
(T*)((intptr_t) expectedPtr | markMask(expectedMark)),
(T*)((intptr_t) newPtr | markMask(newMark)));
}
std::pair<T*,bool> swap(T* newPtr, bool newMark)
{
T* prev = reference_.swap(
(T*)((intptr_t) newPtr | markMask(newMark)));
return std::make_pair(
(T*) ((intptr_t) prev & ~kMarkBitMask),
((intptr_t) prev & kMarkBitMask) != 0);
}
bool tryMark(T* expectedPtr, bool newMark)
{
T* current = reference_;
if (((intptr_t) current & ~kMarkBitMask) != (intptr_t) expectedPtr) {
return false;
}
bool currentMark = ((intptr_t) current & kMarkBitMask) != 0;
return currentMark == newMark || reference_.compareAndSet(current,
(T*)((intptr_t) expectedPtr | markMask(newMark)));
}
bool isMarked() const
{
return ((intptr_t)(T*) reference_ & kMarkBitMask) != 0;
}
std::pair<T*,bool> get() const
{
T* current = reference_;
return std::make_pair(
(T*) ((intptr_t) current & ~kMarkBitMask),
((intptr_t) current & kMarkBitMask) != 0);
}
T* getReference() const
{
return (T*) ((intptr_t)(T*) reference_ & ~kMarkBitMask);
}
void set(T* ptr, bool mark)
{
reference_ = (T*)((intptr_t) ptr | markMask(mark));
}
};
/*! @}
* @}
*/
projects/clr/rocclr/runtime/thread/monitor.cpp
+54 -46
Näytä tiedosto
@@ -15,7 +15,7 @@
namespace amd {
Monitor::Monitor(const char* name, bool recursive) :
contendersList_(NULL), onDeck_(NULL), waitersList_(NULL),
contendersList_(0), onDeck_(0), waitersList_(NULL),
owner_(NULL), recursive_(recursive)
{
const size_t maxNameLen = sizeof(name_);
@@ -68,19 +68,17 @@ Monitor::finishLock()
/* The lock is contended. Push the thread's semaphore onto
* the contention list.
*/
Semaphore& sem = thread->lockSemaphore();
sem.reset();
Semaphore& semaphore = thread->lockSemaphore();
semaphore.reset();
LinkedNode newHead;
newHead.setItem(&sem);
while (true) {
LinkedNode* head; bool isLocked;
newHead.setItem(&semaphore);
intptr_t head = contendersList_.load(std::memory_order_acquire);
for (;;) {
// The assumption is that lockWord is locked. Make sure we do not
// continue unless the lock bit is set.
std::tie(head, isLocked) = contendersList_.get();
if (!isLocked) {
if ((head & kLockBit) == 0) {
if (tryLock()) {
return;
}
@@ -88,8 +86,10 @@ Monitor::finishLock()
}
// Set the new contention list head if lockWord is unchanged.
newHead.setNext(head);
if (contendersList_.compareAndSet(head, &newHead, kLocked, kLocked)) {
newHead.setNext(reinterpret_cast<LinkedNode*>(head & ~kLockBit));
if (contendersList_.compare_exchange_weak(head,
reinterpret_cast<intptr_t>(&newHead) | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire)) {
break;
}
@@ -99,7 +99,7 @@ Monitor::finishLock()
int32_t spinCount = 0;
// Go to sleep until we become the on-deck thread.
while (onDeck_.getReference() != &sem) {
while ((onDeck_ & ~kLockBit) != reinterpret_cast<intptr_t>(&semaphore)) {
// First, be SMT friendly
if (spinCount < kMaxReadSpinIter) {
Os::spinPause();
@@ -110,7 +110,7 @@ Monitor::finishLock()
}
// now go to sleep
else {
sem.wait();
semaphore.wait();
}
spinCount++;
}
@@ -120,8 +120,9 @@ Monitor::finishLock()
// From now-on, we are the on-deck thread. It will stay that way until
// we successfuly acquire the lock.
//
while (true) {
assert(onDeck_.getReference() == &sem && "just checking");
for (;;) {
assert((onDeck_ & ~kLockBit) == reinterpret_cast<intptr_t>(&semaphore)
&& "just checking");
if (tryLock()) {
break;
}
@@ -138,13 +139,13 @@ Monitor::finishLock()
}
// now go to sleep
else {
sem.wait();
semaphore.wait();
}
spinCount++;
}
assert(newHead.next() == NULL && "Should not be linked");
onDeck_ = NULL;
onDeck_ = 0;
}
void
@@ -154,53 +155,58 @@ Monitor::finishUnlock()
// list waiting to acquire the lock. We need to select a successor and
// place it on-deck.
while (true) {
for (;;) {
// Grab the onDeck_ microlock to protect the next loop (make sure only
// one semaphore is removed from the contention list).
//
if (!onDeck_.compareAndSet(NULL, NULL, kUnlocked, kLocked)) {
intptr_t ptr = 0;
if (!onDeck_.compare_exchange_strong(ptr, ptr | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire)) {
return; // Somebody else has the microlock, let him select onDeck_
}
LinkedNode* head; bool isLocked;
while (true) {
std::tie(head, isLocked) = contendersList_.get();
if (head == NULL) {
intptr_t head = contendersList_.load(std::memory_order_acquire);
for (;;) {
if (head == 0) {
break; // There's nothing else to do.
}
if (isLocked) {
if ((head & kLockBit) != 0) {
// Somebody could have acquired then released the lock
// and failed to grab the onDeck_ microlock.
head = NULL;
head = 0;
break;
}
if (contendersList_.compareAndSet(
head, head->next(), kUnlocked, kUnlocked)) {
if (contendersList_.compare_exchange_weak(
head, reinterpret_cast<intptr_t>(
reinterpret_cast<LinkedNode*>(head)->next()),
std::memory_order_acq_rel, std::memory_order_acquire)) {
#ifdef ASSERT
head->setNext(NULL);
reinterpret_cast<LinkedNode*>(head)->setNext(NULL);
#endif // ASSERT
break;
}
}
Semaphore* sem = (head != NULL) ? head->item() : NULL;
onDeck_ = sem;
MemoryOrder::fence();
Semaphore* semaphore = (head != 0)
? reinterpret_cast<LinkedNode*>(head)->item()
: NULL;
onDeck_.store(reinterpret_cast<intptr_t>(semaphore),
std::memory_order_release);
//
// Release the onDeck_ microlock (end of critical region);
if (sem != NULL) {
sem->post();
if (semaphore != NULL) {
semaphore->post();
return;
}
// We do not have an on-deck thread (sem == NULL). Return if
// We do not have an on-deck thread (semaphore == NULL). Return if
// the contention list is empty or if the lock got acquired again.
std::tie(head, isLocked) = contendersList_.get();
if (isLocked || head == NULL) {
head = contendersList_;
if (head == 0 || (head & kLockBit) != 0) {
return;
}
}
@@ -230,7 +236,7 @@ Monitor::wait()
// Go to sleep until we become the on-deck thread.
int32_t spinCount = 0;
while (onDeck_.getReference() != &suspend) {
while ((onDeck_ & ~kLockBit) != reinterpret_cast<intptr_t>(&suspend)) {
// First, be SMT friendly
if (spinCount < kMaxReadSpinIter) {
Os::spinPause();
@@ -247,8 +253,9 @@ Monitor::wait()
}
spinCount = 0;
while (true) {
assert(onDeck_.getReference() == &suspend && "just checking");
for (;;) {
assert((onDeck_ & ~kLockBit) == reinterpret_cast<intptr_t>(&suspend)
&& "just checking");
if (trySpinLock()) {
break;
@@ -274,8 +281,7 @@ Monitor::wait()
// Restore the lock count (for recursive mutexes)
lockCount_ = lockCount;
onDeck_ = NULL;
MemoryOrder::fence();
onDeck_.store(0, std::memory_order_release);
}
void
@@ -290,11 +296,13 @@ Monitor::notify()
// Dequeue a waiter from the wait list and add it to the contention list.
waitersList_ = waiter->next();
while (true) {
LinkedNode* node = contendersList_.getReference();
waiter->setNext(node);
if (contendersList_.compareAndSet(node, waiter, kLocked, kLocked)) {
intptr_t node = contendersList_.load(std::memory_order_acquire);
for (;;) {
waiter->setNext(reinterpret_cast<LinkedNode*>(node & ~kLockBit));
if (contendersList_.compare_exchange_weak(node,
reinterpret_cast<intptr_t>(waiter) | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire)) {
break;
}
}
projects/clr/rocclr/runtime/thread/monitor.hpp
+20 -25
Näytä tiedosto
@@ -62,8 +62,7 @@ class Monitor : public HeapObject
typedef details::SimplyLinkedNode<Semaphore*,StackObject> LinkedNode;
private:
static const bool kUnlocked = false;
static const bool kLocked = true;
static const intptr_t kLockBit = 0x1;
static const int kMaxSpinIter = 55; //!< Total number of spin iterations.
static const int kMaxReadSpinIter = 50; //!< Read iterations before yielding
@@ -71,12 +70,12 @@ private:
/*! Linked list of semaphores the contending threads are waiting on
* and main lock.
*/
AtomicMarkableReference<LinkedNode> contendersList_;
std::atomic_intptr_t contendersList_;
//! The Mutex's name
char name_[64];
//! Semaphore of the next thread to contend for the lock.
AtomicMarkableReference<Semaphore> onDeck_;
std::atomic_intptr_t onDeck_;
//! Linked list of the suspended threads resume semaphores.
LinkedNode* volatile waitersList_;
@@ -101,7 +100,7 @@ protected:
*
* \note The user is responsible for the memory ordering.
*/
bool isLocked() const { return contendersList_.isMarked(); }
bool isLocked() const { return (contendersList_ & kLockBit) != 0; }
//! Return this monitor's owner thread (NULL if unlocked).
Thread* owner() const { return owner_; }
@@ -179,10 +178,9 @@ Monitor::tryLock()
Thread* thread = Thread::current();
assert(thread != NULL && "cannot lock() from (null)");
LinkedNode* ptr; bool isLocked;
std::tie(ptr, isLocked) = contendersList_.get();
intptr_t ptr = contendersList_.load(std::memory_order_acquire);
if (unlikely(isLocked)) {
if (unlikely((ptr & kLockBit) != 0)) {
if (recursive_ && thread == owner_) {
// Recursive lock: increment the lock count and return.
++lockCount_;
@@ -191,8 +189,8 @@ Monitor::tryLock()
return false; // Already locked!
}
if (unlikely(!contendersList_.compareAndSet(
ptr, ptr, kUnlocked, kLocked))) {
if (unlikely(!contendersList_.compare_exchange_weak(ptr, ptr | kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire))) {
return false; // We failed the CAS from unlocked to locked.
}
@@ -227,23 +225,21 @@ Monitor::unlock()
setOwner(NULL);
while (true) {
LinkedNode* ptr = contendersList_.getReference();
// Clear the lock bit.
if (contendersList_.compareAndSet(ptr, ptr, kLocked, kUnlocked)) {
break; // We succeeded the CAS from locked to unlocked.
}
}
// Clear the lock bit.
intptr_t ptr = contendersList_.load(std::memory_order_acquire);
while (!contendersList_.compare_exchange_weak(ptr, ptr & ~kLockBit,
std::memory_order_acq_rel, std::memory_order_acquire))
;
//
// We succeeded the CAS from locked to unlocked.
// This is the end of the critical region.
// Check if we have an on-deck thread that needs signaling.
Semaphore* onDeck; bool isMarked;
std::tie(onDeck, isMarked) = onDeck_.get();
if (onDeck != NULL) {
if (!isMarked) {
intptr_t onDeck = onDeck_;
if (onDeck != 0) {
if ((onDeck & kLockBit) == 0) {
// Only signal if it is unmarked.
onDeck->post();
reinterpret_cast<Semaphore*>(onDeck)->post();
}
return; // We are done.
}
@@ -253,9 +249,8 @@ Monitor::unlock()
// so return if the list is empty or if the lock got acquired again (it's
// somebody else's problem now!)
LinkedNode* head; bool isLocked;
std::tie(head, isLocked) = contendersList_.get();
if (isLocked || head == NULL) {
intptr_t head = contendersList_;
if (head == 0 || (head & kLockBit) != 0) {
return;
}
projects/clr/rocclr/runtime/utils/concurrent.hpp
+1 -1
Näytä tiedosto
@@ -126,7 +126,7 @@ ConcurrentLinkedQueue<T,N>::ConcurrentLinkedQueue()
// Make sure the instance is fully initialized before it becomes
// globally visible.
MemoryOrder::sfence();
std::atomic_thread_fence(std::memory_order_release);
}
template <typename T, int N>