P2P Update.

- add P2P staging buffer copy.
- If copy device does not have sufficient access permissions, fall back
  to staging buffer.
- improve docs for which copy device is used.

include/hcc_detail/hip_hcc.h

@@ -38,7 +38,7 @@ THE SOFTWARE.
 // Compile peer-to-peer support.
 // >= 2 : use HCC hc:accelerator::get_is_peer
 // >= 3 : use hc::am_memtracker_update_peers(...)
-#define USE_PEER_TO_PEER 0
+#define USE_PEER_TO_PEER 2
 
 // Use new lock API in HCC:
 #define USE_HCC_LOCK 0
@@ -247,11 +247,12 @@ enum ihipCommand_t {
     ihipCommandCopyH2D,  
     ihipCommandCopyD2H,
     ihipCommandCopyD2D,
+    ihipCommandCopyP2P,
     ihipCommandKernel,
 };
 
 static const char* ihipCommandName[] = {
-    "CopyH2H", "CopyH2D", "CopyD2H", "CopyD2D", "Kernel"
+    "CopyH2H", "CopyH2D", "CopyD2H", "CopyD2D", "CopyP2P", "Kernel"
 };
 
 
@@ -451,7 +452,7 @@ private:
 
     // The unsigned return is hipMemcpyKind
     unsigned resolveMemcpyDirection(bool srcInDeviceMem, bool dstInDeviceMem);
-    void setCopyAgents(unsigned kind, ihipCommand_t *commandType, hsa_agent_t *srcAgent, hsa_agent_t *dstAgent);
+    void setAsyncCopyAgents(unsigned kind, ihipCommand_t *commandType, hsa_agent_t *srcAgent, hsa_agent_t *dstAgent);
 
     unsigned                    _device_index;       // index into the g_device array 
 
@@ -527,6 +528,7 @@ public:
     // "Allocate" a stream ID:
     ihipStream_t::SeqNum_t  incStreamId() { return _stream_id++; };
 
+    bool isPeer(const ihipDevice_t *peer); // returns Trus if peer has access to memory physically located on this device.
     bool addPeer(ihipDevice_t *peer);
     bool removePeer(ihipDevice_t *peer);
     void resetPeers(ihipDevice_t *thisDevice);
@@ -540,9 +542,10 @@ private:
     ihipStream_t::SeqNum_t   _stream_id;
 
     // These reflect the currently Enabled set of peers for this GPU:
+    // Enabled peers have permissions to access the memory physically allocated on this device.
     std::list<ihipDevice_t*>  _peers;     // list of enabled peer devices.
     uint32_t                  _peerCnt;     // number of enabled peers
-    hsa_agent_t              *_peerAgents;  // efficient packed array of enabled agents (to use for allocations.)
+    hsa_agent_t              *_peerAgents;  // efficient packed array of enabled agents (to use for allocations.) 
 private:
     void recomputePeerAgents();
 };

include/hcc_detail/hip_runtime_api.h

@@ -798,6 +798,12 @@ hipError_t hipHostFree(void* ptr);
  *  device to host, device to device and host to host
  *  The src and dst must not overlap.
  *
+ *  For hipMemcpy, the copy is always performed by the current device (set by hipSetDevice).
+ *  For multi-gpu or peer-to-peer configurations, it is recommended to set the current device to the device where the src data is physically located.
+ *  For optimal peer-to-peer copies, the copy device must be able to access the src and dst pointers (by calling hipDeviceEnablePeerAccess with copy agent as the 
+ *  current device and src/dest as the peerDevice argument.  if this is not done, the hipMemcpy will still work, but will perform the copy using a staging buffer
+ *  on the host.
+ *
  *  @param[out]  dst Data being copy to
  *  @param[in]  src Data being copy from
  *  @param[in]  sizeBytes Data size in bytes
@@ -830,6 +836,13 @@ hipError_t hipMemcpyToSymbol(const char* symbolName, const void *src, size_t siz
  *  @warning If host or dest are not pinned, the memory copy will be performed synchronously.  For best performance, use hipHostMalloc to
  *  allocate host memory that is transferred asynchronously.
  *
+ *  For hipMemcpy, the copy is always performed by the device associated with the specified stream.
+ *
+ *  For multi-gpu or peer-to-peer configurations, it is recommended to use a stream which is a attached to the device where the src data is physically located.
+ *  For optimal peer-to-peer copies, the copy device must be able to access the src and dst pointers (by calling hipDeviceEnablePeerAccess with copy agent as the 
+ *  current device and src/dest as the peerDevice argument.  if this is not done, the hipMemcpy will still work, but will perform the copy using a staging buffer
+ *  on the host.
+ *
  *  @param[out] dst Data being copy to
  *  @param[in]  src Data being copy from
  *  @param[in]  sizeBytes Data size in bytes
@@ -902,17 +915,15 @@ hipError_t hipMemGetInfo  (size_t * free, size_t * total)   ;
 /**
  * @brief Determine if a device can access a peer's memory.
  *
- * @param [out] canAccessPeer returns true if specified devices are peers.
- * @param [in] device
- * @param [in] peerDevice
+ * @param [out] canAccessPeer Returns the peer access capability (0 or 1)
+ * @param [in] device - device from where memory may be accessed.
+ * @param [in] peerDevice - device where memory is physically located
  *
  * Returns "1" in @p canAccessPeer if the specified @p device is capable
  * of directly accessing memory physically located on peerDevice , or "0" if not.
  *
  * Returns "0" in @p canAccessPeer if deviceId == peerDeviceId, and both are valid devices : a device is not a peer of itself.
  *
- *
- *
  * @returns #hipSuccess, 
  * @returns #hipErrorInvalidDevice if deviceId or peerDeviceId are not valid devices
  */

include/hcc_detail/staging_buffer.h

@@ -50,6 +50,8 @@ struct StagingBuffer {
     void CopyDeviceToHost   (void* dst, const void* src, size_t sizeBytes, hsa_signal_t *waitFor);
     void CopyDeviceToHostPinInPlace(void* dst, const void* src, size_t sizeBytes, hsa_signal_t *waitFor);
 
+    void CopyPeerToPeer( void* dst, hsa_agent_t dstAgent, const void* src, hsa_agent_t srcAgent, size_t sizeBytes, hsa_signal_t *waitFor);
+
 
 private:
     hsa_agent_t     _hsa_agent;
@@ -58,6 +60,7 @@ private:
 
     char            *_pinnedStagingBuffer[_max_buffers];
     hsa_signal_t     _completion_signal[_max_buffers];
+    hsa_signal_t     _completion_signal2[_max_buffers]; // P2P needs another set of signals.
     std::mutex       _copy_lock;    // provide thread-safe access 
 };