AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add option to deny peer access.

Browse Source 
Also fix test.

Change-Id: I1b247f6c4271442b008e560669bca4daf8eb94c7


[ROCm/hip commit: 1e5515ee9f]
This commit is contained in:
Ben Sander
2016-11-10 23:10:42 -06:00
parent 99ce555300
commit 1c7e71e1b3
4 changed files with 27 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/hip/src/hip_hcc.cpp
+3 -2
View File
@@ -76,6 +76,7 @@ int HIP_NUM_KERNELS_INFLIGHT = 128;
int HIP_WAIT_MODE = 0;
int HIP_FORCE_P2P_HOST = 0;
int HIP_DENY_PEER_ACCESS = 0;
// Force async copies to actually use the synchronous copy interface.
int HIP_FORCE_SYNC_COPY = 0;
@@ -1374,7 +1375,7 @@ void ihipInit()
READ_ENV_I(release, HIP_WAIT_MODE, 0, "Force synchronization mode. 1= force yield, 2=force spin, 0=defaults specified in application");
READ_ENV_I(release, HIP_FORCE_P2P_HOST, 0, "Force use of host/staging copy for peer-to-peer copiecopies");
READ_ENV_I(release, HIP_FORCE_P2P_HOST, 0, "Force use of host/staging copy for peer-to-peer copies.1=always use copies, 2=always return false for hipDeviceCanAccessPeer");
READ_ENV_I(release, HIP_FORCE_SYNC_COPY, 0, "Force all copies (even hipMemcpyAsync) to use sync copies");
READ_ENV_I(release, HIP_NUM_KERNELS_INFLIGHT, 128, "Max number of inflight kernels per stream before active synchronization is forced.");
@@ -1856,7 +1857,7 @@ void ihipStream_t::resolveHcMemcpyDirection(unsigned hipMemKind,
*forceUnpinnedCopy = false;
if (canSeeMemory(*copyDevice, dstPtrInfo, srcPtrInfo)) {
if (HIP_FORCE_P2P_HOST ) {
if (HIP_FORCE_P2P_HOST & 0x1) {
*forceUnpinnedCopy = true;
tprintf (DB_COPY, "P2P. Copy engine (dev:%d) can see src and dst but HIP_FORCE_P2P_HOST=0, forcing copy through staging buffers.\n", (*copyDevice)->getDeviceNum());
} else {
projects/hip/src/hip_hcc.h
+1
View File
@@ -54,6 +54,7 @@ extern int HIP_DB;
extern int HIP_STAGING_SIZE; /* size of staging buffers, in KB */
extern int HIP_STREAM_SIGNALS; /* number of signals to allocate at stream creation */
extern int HIP_VISIBLE_DEVICES; /* Contains a comma-separated sequence of GPU identifiers */
extern int HIP_FORCE_P2P_HOST;
//---
projects/hip/src/hip_peer.cpp
+10 -4
View File
@@ -43,13 +43,18 @@ hipError_t ihipDeviceCanAccessPeer (int* canAccessPeer, hipCtx_t thisCtx, hipCtx
if ((thisCtx != NULL) && (peerCtx != NULL)) {
if (thisCtx == peerCtx) {
*canAccessPeer = 0;
tprintf(DB_COPY, "Can't be peer to self. (this=%s, peer=%s)\n",
tprintf(DB_MEM, "Can't be peer to self. (this=%s, peer=%s)\n",
thisCtx->toString().c_str(), peerCtx->toString().c_str());
} else if (HIP_FORCE_P2P_HOST & 0x2) {
*canAccessPeer = false;
tprintf(DB_MEM, "HIP_FORCE_P2P_HOST denies peer access this=%s peer=%s canAccessPeer=%d\n",
thisCtx->toString().c_str(), peerCtx->toString().c_str(), *canAccessPeer);
} else {
*canAccessPeer = peerCtx->getDevice()->_acc.get_is_peer(thisCtx->getDevice()->_acc);
tprintf(DB_COPY, "deviceCanAccessPeer this=%s peer=%s canAccessPeer=%d\n",
tprintf(DB_MEM, "deviceCanAccessPeer this=%s peer=%s canAccessPeer=%d\n",
thisCtx->toString().c_str(), peerCtx->toString().c_str(), *canAccessPeer);
}
@@ -58,6 +63,7 @@ hipError_t ihipDeviceCanAccessPeer (int* canAccessPeer, hipCtx_t thisCtx, hipCtx
err = hipErrorInvalidDevice;
}
return err;
}
@@ -93,7 +99,7 @@ hipError_t ihipDisablePeerAccess (hipCtx_t peerCtx)
LockedAccessor_CtxCrit_t peerCrit(peerCtx->criticalData());
bool changed = peerCrit->removePeerWatcher(peerCtx, thisCtx);
if (changed) {
tprintf(DB_COPY, "device %s disable access to memory allocated on peer:%s\n",
tprintf(DB_MEM, "device %s disable access to memory allocated on peer:%s\n",
thisCtx->toString().c_str(), peerCtx->toString().c_str());
// Update the peers for all memory already saved in the tracker:
am_memtracker_update_peers(peerCtx->getDevice()->_acc, peerCrit->peerCnt(), peerCrit->peerAgents());
@@ -127,7 +133,7 @@ hipError_t ihipEnablePeerAccess (hipCtx_t peerCtx, unsigned int flags)
// Add thisCtx to peerCtx's access list so that new allocations on peer will be made visible to this device:
bool isNewPeer = peerCrit->addPeerWatcher(peerCtx, thisCtx);
if (isNewPeer) {
tprintf(DB_COPY, "device=%s can now see all memory allocated on peer=%s\n",
tprintf(DB_MEM, "device=%s can now see all memory allocated on peer=%s\n",
thisCtx->toString().c_str(), peerCtx->toString().c_str());
am_memtracker_update_peers(peerCtx->getDevice()->_acc, peerCrit->peerCnt(), peerCrit->peerAgents());
} else {
projects/hip/tests/src/hipPeerToPeer_simple.cpp
+13 -9
View File
@@ -45,7 +45,7 @@ void help(char *argv[])
{
printf ("usage: %s [OPTIONS]\n", argv[0]);
printf (" --memcpyWithPeer : Perform memcpy with peer.\n");
printf (" --mirrorPeersi : Mirror memory onto both default device and peerdevice. If 0, memory is mapped only on the default device.\n");
printf (" --mirrorPeers : Mirror memory onto both default device and peerdevice. If 0, memory is mapped only on the default device.\n");
printf (" --peerDevice N : Set peer device.\n");
};
@@ -175,6 +175,9 @@ void enablePeerFirst(bool useAsyncCopy)
HIPCHECK (myHipMemcpy(A_d1, A_d0, Nbytes, hipMemcpyDefault, 0/*stream*/, useAsyncCopy)); // This is P2P copy.
// Copy data back to host:
// Have to wait for previous operation to finish, since we are switching to another one:
HIPCHECK(hipDeviceSynchronize());
HIPCHECK (hipSetDevice(g_peerDevice));
HIPCHECK (myHipMemcpy(A_h, A_d1, Nbytes, hipMemcpyDeviceToHost, 0/*stream*/, useAsyncCopy));
HIPCHECK(hipDeviceSynchronize());
@@ -241,12 +244,13 @@ void allocMemoryFirst(bool useAsyncCopy)
HIPCHECK (hipSetDevice(p_memcpyWithPeer ? g_peerDevice : g_currentDevice));
HIPCHECK (myHipMemcpy(A_d1, A_d0, Nbytes, hipMemcpyDefault, 0/*stream*/, useAsyncCopy));
syncBothDevices(); // TODO - remove me, should handle this in implementation.
// Copy data back to host:
HIPCHECK (hipSetDevice(g_peerDevice));
HIPCHECK (myHipMemcpy(A_h, A_d1, Nbytes, hipMemcpyDeviceToHost, 0/*stream*/, useAsyncCopy));
HIPCHECK(hipDeviceSynchronize());
HIPCHECK (hipSetDevice(g_currentDevice));
syncBothDevices(); // TODO - remove me, should handle this in implementation.
//---
@@ -287,15 +291,15 @@ void testPeerHostToDevice(bool useAsyncCopy)
size_t Nbytes = N*sizeof(char);
char *A_d0, *A_d1;
char *A_host_d0, *A_d1;
char *A_h;
A_h = (char*)malloc(Nbytes);
// allocate and initialize memory on device0
HIPCHECK (hipSetDevice(g_currentDevice));
HIPCHECK (hipHostMalloc(&A_d0, Nbytes) );
HIPCHECK (hipMemset(A_d0, memsetval, Nbytes) );
HIPCHECK (hipHostMalloc(&A_host_d0, Nbytes) );
HIPCHECK (hipMemset(A_host_d0, memsetval, Nbytes) );
// allocate and initialize memory on peer device
HIPCHECK (hipSetDevice(g_peerDevice));
@@ -314,15 +318,15 @@ void testPeerHostToDevice(bool useAsyncCopy)
if (p_memcpyWithPeer) {
// p_memcpyWithPeer=1 case is HostToDevice.
// if p_mirrorPeers = 1, this is accelerated copy over PCIe.
// if p_mirrorPeers = 0, this should fall back to host (because peer can't see A_d0)
// if p_mirrorPeers = 0, this should fall back to host (because peer can't see A_host_d0)
HIPCHECK (hipSetDevice(g_peerDevice));
HIPCHECK (myHipMemcpy(A_d1, A_d0, Nbytes, hipMemcpyHostToDevice, 0/*stream*/, firstAsyncCopy)); // This is P2P copy.
HIPCHECK (myHipMemcpy(A_d1, A_host_d0, Nbytes, hipMemcpyHostToDevice, 0/*stream*/, firstAsyncCopy)); // This is P2P copy.
} else {
// p_memcpyWithPeer=0 case is HostToDevice.
// if p_mirrorPeers = 1, this is accelerated copy over PCIe.
// if p_mirrorPeers = 0, this should fall back to host (because device0 can't see A_d1)
HIPCHECK (hipSetDevice(g_currentDevice));
HIPCHECK (myHipMemcpy(A_d1, A_d0, Nbytes, hipMemcpyHostToDevice, 0/*stream*/, firstAsyncCopy)); // This is P2P copy.
HIPCHECK (myHipMemcpy(A_d1, A_host_d0, Nbytes, hipMemcpyHostToDevice, 0/*stream*/, firstAsyncCopy)); // This is P2P copy.
}
syncBothDevices();