Refactor copy code.

-Move staging buffer locks inside the staging buffer code.
-Remove dedicated per-device completion_signal + per-device lock -
instead allocated signal from the per-stream pool.   This elimintes
the lock and allows more concurrency.
-remove switch HIP_DISABLE_BIDIR_MEMCPY

src/staging_buffer.cpp

@@ -53,6 +53,8 @@ StagingBuffer::~StagingBuffer()
 //IN: waitFor - hsaSignal to wait for - the copy will begin only when the specified dependency is resolved.  May be NULL indicating no dependency.
 void StagingBuffer::CopyHostToDevicePinInPlace(void* dst, const void* src, size_t sizeBytes, hsa_signal_t *waitFor)
 {
+    std::lock_guard<std::mutex> l (_copy_lock);
+
     const char *srcp = static_cast<const char*> (src);
     char *dstp = static_cast<char*> (dst);
 
@@ -120,6 +122,8 @@ void StagingBuffer::CopyHostToDevicePinInPlace(void* dst, const void* src, size_
 //IN: waitFor - hsaSignal to wait for - the copy will begin only when the specified dependency is resolved.  May be NULL indicating no dependency.
 void StagingBuffer::CopyHostToDevice(void* dst, const void* src, size_t sizeBytes, hsa_signal_t *waitFor)
 {
+    std::lock_guard<std::mutex> l (_copy_lock);
+
     const char *srcp = static_cast<const char*> (src);
     char *dstp = static_cast<char*> (dst);
 
@@ -175,6 +179,8 @@ void StagingBuffer::CopyHostToDevice(void* dst, const void* src, size_t sizeByte
 //IN: waitFor - hsaSignal to wait for - the copy will begin only when the specified dependency is resolved.  May be NULL indicating no dependency.
 void StagingBuffer::CopyDeviceToHost(void* dst, const void* src, size_t sizeBytes, hsa_signal_t *waitFor)
 {
+    std::lock_guard<std::mutex> l (_copy_lock);
+
     const char *srcp0 = static_cast<const char*> (src);
     char *dstp1 = static_cast<char*> (dst);