Add option to deny peer access.

Also fix test.

Change-Id: I1b247f6c4271442b008e560669bca4daf8eb94c7


[ROCm/hip commit: 1e5515ee9f]

projects/hip/src/hip_hcc.cpp

@@ -76,6 +76,7 @@ int HIP_NUM_KERNELS_INFLIGHT = 128;
 int HIP_WAIT_MODE = 0;
 
 int HIP_FORCE_P2P_HOST = 0;
+int HIP_DENY_PEER_ACCESS = 0;
 
 // Force async copies to actually use the synchronous copy interface. 
 int HIP_FORCE_SYNC_COPY = 0;
@@ -1374,7 +1375,7 @@ void ihipInit()
 
 
     READ_ENV_I(release, HIP_WAIT_MODE, 0, "Force synchronization mode. 1= force yield, 2=force spin, 0=defaults specified in application");
-    READ_ENV_I(release, HIP_FORCE_P2P_HOST, 0, "Force use of host/staging copy for peer-to-peer copiecopies"); 
+    READ_ENV_I(release, HIP_FORCE_P2P_HOST, 0, "Force use of host/staging copy for peer-to-peer copies.1=always use copies, 2=always return false for hipDeviceCanAccessPeer"); 
     READ_ENV_I(release, HIP_FORCE_SYNC_COPY, 0, "Force all copies (even hipMemcpyAsync) to use sync copies"); 
     READ_ENV_I(release, HIP_NUM_KERNELS_INFLIGHT, 128, "Max number of inflight kernels per stream before active synchronization is forced.");
 
@@ -1856,7 +1857,7 @@ void ihipStream_t::resolveHcMemcpyDirection(unsigned hipMemKind,
     *forceUnpinnedCopy = false;
     if (canSeeMemory(*copyDevice, dstPtrInfo, srcPtrInfo)) {
 
-        if (HIP_FORCE_P2P_HOST ) {
+        if (HIP_FORCE_P2P_HOST & 0x1) {
             *forceUnpinnedCopy = true;
             tprintf (DB_COPY, "P2P.  Copy engine (dev:%d) can see src and dst but HIP_FORCE_P2P_HOST=0, forcing copy through staging buffers.\n", (*copyDevice)->getDeviceNum());
         } else {

projects/hip/src/hip_hcc.h

@@ -54,6 +54,7 @@ extern int HIP_DB;
 extern int HIP_STAGING_SIZE;   /* size of staging buffers, in KB */
 extern int HIP_STREAM_SIGNALS;  /* number of signals to allocate at stream creation */
 extern int HIP_VISIBLE_DEVICES; /* Contains a comma-separated sequence of GPU identifiers */
+extern int HIP_FORCE_P2P_HOST;
 
 
 //---

projects/hip/src/hip_peer.cpp

@@ -43,13 +43,18 @@ hipError_t ihipDeviceCanAccessPeer (int* canAccessPeer, hipCtx_t thisCtx, hipCtx
 
 
     if ((thisCtx != NULL) && (peerCtx != NULL)) {
+
         if (thisCtx == peerCtx) {
             *canAccessPeer = 0;
-            tprintf(DB_COPY, "Can't be peer to self. (this=%s, peer=%s)\n", 
+            tprintf(DB_MEM, "Can't be peer to self. (this=%s, peer=%s)\n", 
                     thisCtx->toString().c_str(), peerCtx->toString().c_str()); 
+        } else  if (HIP_FORCE_P2P_HOST & 0x2) {
+            *canAccessPeer = false;
+            tprintf(DB_MEM, "HIP_FORCE_P2P_HOST denies peer access this=%s peer=%s  canAccessPeer=%d\n", 
+                    thisCtx->toString().c_str(), peerCtx->toString().c_str(), *canAccessPeer); 
         } else {
             *canAccessPeer = peerCtx->getDevice()->_acc.get_is_peer(thisCtx->getDevice()->_acc);
-            tprintf(DB_COPY, "deviceCanAccessPeer this=%s peer=%s  canAccessPeer=%d\n", 
+            tprintf(DB_MEM, "deviceCanAccessPeer this=%s peer=%s  canAccessPeer=%d\n", 
                     thisCtx->toString().c_str(), peerCtx->toString().c_str(), *canAccessPeer); 
         }
 
@@ -58,6 +63,7 @@ hipError_t ihipDeviceCanAccessPeer (int* canAccessPeer, hipCtx_t thisCtx, hipCtx
         err = hipErrorInvalidDevice;
     }
 
+
     return err;
 }
 
@@ -93,7 +99,7 @@ hipError_t ihipDisablePeerAccess (hipCtx_t peerCtx)
             LockedAccessor_CtxCrit_t peerCrit(peerCtx->criticalData());
             bool changed = peerCrit->removePeerWatcher(peerCtx, thisCtx);
             if (changed) {
-                tprintf(DB_COPY, "device %s disable access to memory allocated on peer:%s\n", 
+                tprintf(DB_MEM, "device %s disable access to memory allocated on peer:%s\n", 
                                   thisCtx->toString().c_str(), peerCtx->toString().c_str()); 
                 // Update the peers for all memory already saved in the tracker:
                 am_memtracker_update_peers(peerCtx->getDevice()->_acc, peerCrit->peerCnt(), peerCrit->peerAgents());
@@ -127,7 +133,7 @@ hipError_t ihipEnablePeerAccess (hipCtx_t peerCtx, unsigned int flags)
             // Add thisCtx to peerCtx's access list so that new allocations on peer will be made visible to this device:
             bool isNewPeer = peerCrit->addPeerWatcher(peerCtx, thisCtx);
             if (isNewPeer) {
-                tprintf(DB_COPY, "device=%s can now see all memory allocated on peer=%s\n", 
+                tprintf(DB_MEM, "device=%s can now see all memory allocated on peer=%s\n", 
                                   thisCtx->toString().c_str(), peerCtx->toString().c_str()); 
                 am_memtracker_update_peers(peerCtx->getDevice()->_acc, peerCrit->peerCnt(), peerCrit->peerAgents());
             } else {