Refactor copy code.

-Move staging buffer locks inside the staging buffer code.
-Remove dedicated per-device completion_signal + per-device lock -
instead allocated signal from the per-stream pool.   This elimintes
the lock and allows more concurrency.
-remove switch HIP_DISABLE_BIDIR_MEMCPY


[ROCm/clr commit: e64174f47a]

projects/clr/hipamd/include/hcc_detail/staging_buffer.h

@@ -4,6 +4,18 @@
 
 
 //-------------------------------------------------------------------------------------------------
+// An optimized "staging buffer" used to implement Host-To-Device and Device-To-Host copies.
+// Some GPUs may not be able to directly access host memory, and in these cases we need to 
+// stage the copy through a pinned staging buffer.  For example, the CopyHostToDevice
+// uses the CPU to copy to a pinned "staging buffer", and then use the GPU DMA engine to copy
+// from the staging buffer to the final destination.  The copy is broken into buffer-sized chunks
+// to limit the size of the buffer and also to provide better performance by overlapping the CPU copies 
+// with the DMA copies.
+//
+// PinInPlace is another algorithm which pins the host memory "in-place", and copies it with the DMA
+// engine.  This routine is under development.
+//
+// Staging buffer provides thread-safe access via a mutex.
 struct StagingBuffer {
 
     static const int _max_buffers = 4;
@@ -25,4 +37,5 @@ private:
 
     char            *_pinnedStagingBuffer[_max_buffers];
     hsa_signal_t     _completion_signal[_max_buffers];
+    std::mutex       _copy_lock;    // provide thread-safe access 
 };