AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'amd-master' into amd-master-next

Browse Source 
Change-Id: I3094c15008093f2072bcd38aca4ea90aeae2d97b


[ROCm/hip commit: 2af31479e2]
This commit is contained in:
Maneesh Gupta
2020-04-07 06:57:42 -04:00
parent c34599e77c
commit 4e1f6f2a0e
67 changed files with 2278 additions and 601 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/hip/src/hip_memory.cpp
+68 -33
View File
@@ -309,31 +309,52 @@ void generic_copy(void* __restrict dst, const void* __restrict src, size_t n,
if (di.size == is_cpu_owned) return d2h_copy(dst, src, n, si);
if (si.size == is_cpu_owned) return h2d_copy(dst, src, n, di);
throwing_result_check(hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr,
di.agentBaseAddress),
__FILE__, __func__, __LINE__);
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
hsa_status_t res = hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr, di.agentBaseAddress);
if (res == HSA_STATUS_SUCCESS){
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
}
// If devices do not have access then fallback mechanism will be used
// copy will be slower
throwing_result_check(hsa_memory_copy(dst,src,n), __FILE__, __func__, __LINE__);
}
inline
void memcpy_impl(void* __restrict dst, const void* __restrict src, size_t n,
hipMemcpyKind k) {
auto si{info(src)};
auto di{info(dst)};
if (!is_large_BAR){
// Pointer info takes presidence over hipMemcpyKind
// if there is mismatch b/w Memcpy kind and dst/src pointer
// E.g. dst(host pointer),src(device pointer) and hipMemcpyKind set as hipMemcpyHostToDevice
if (di.size == is_cpu_owned && si.size == is_cpu_owned)
k = hipMemcpyHostToHost;
else if (si.size == is_cpu_owned && di.size != is_cpu_owned)
k = hipMemcpyHostToDevice;
else if (di.size == is_cpu_owned && si.size != is_cpu_owned)
k = hipMemcpyDeviceToHost;
else
k = hipMemcpyDeviceToDevice;
}
switch (k) {
case hipMemcpyHostToHost: std::memcpy(dst, src, n); break;
case hipMemcpyHostToDevice: return h2d_copy(dst, src, n, info(dst));
case hipMemcpyDeviceToHost: return d2h_copy(dst, src, n, info(src));
case hipMemcpyHostToDevice: return h2d_copy(dst, src, n, di);
case hipMemcpyDeviceToHost: return d2h_copy(dst, src, n, si);
case hipMemcpyDeviceToDevice: {
const auto di{info(dst)};
const auto si{info(src)};
throwing_result_check(hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr,
di.agentBaseAddress),
__FILE__, __func__, __LINE__);
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
hsa_status_t res = hsa_amd_agents_allow_access(1u, &si.agentOwner,
nullptr, di.agentBaseAddress);
if (res == HSA_STATUS_SUCCESS){
return do_copy(dst, src, n, di.agentOwner, si.agentOwner);
}
// If devices do not have access then fallback mechanism will be used
// copy will be slower
throwing_result_check(hsa_memory_copy(dst,src,n), __FILE__, __func__, __LINE__);
break;
}
default: return generic_copy(dst, src, n, info(dst), info(src));
default: return generic_copy(dst, src, n, di, si);
}
}
@@ -478,6 +499,10 @@ void* allocAndSharePtr(const char* msg, size_t sizeBytes, ihipCtx_t* ctx, bool s
hipError_t ihipHostMalloc(TlsData *tls, void** ptr, size_t sizeBytes, unsigned int flags) {
hipError_t hip_status = hipSuccess;
if (sizeBytes == 0) {
return hipSuccess;
}
if (HIP_SYNC_HOST_ALLOC) {
hipDeviceSynchronize();
}
@@ -485,10 +510,6 @@ hipError_t ihipHostMalloc(TlsData *tls, void** ptr, size_t sizeBytes, unsigned i
auto ctx = ihipGetTlsDefaultCtx();
if ((ctx == nullptr) || (ptr == nullptr)) {
hip_status = hipErrorInvalidValue;
}
else if (sizeBytes == 0) {
hip_status = hipSuccess;
// TODO - should size of 0 return err or be siliently ignored?
} else {
unsigned trueFlags = flags;
if (flags == hipHostMallocDefault) {
@@ -673,14 +694,15 @@ hipError_t hipMalloc(void** ptr, size_t sizeBytes) {
HIP_SET_DEVICE();
hipError_t hip_status = hipSuccess;
if (sizeBytes == 0) {
if (ptr) *ptr = NULL;
return ihipLogStatus(hipSuccess);
}
auto ctx = ihipGetTlsDefaultCtx();
// return NULL pointer when malloc size is 0
if ( nullptr == ctx || nullptr == ptr) {
hip_status = hipErrorInvalidValue;
}
else if (sizeBytes == 0) {
*ptr = NULL;
hip_status = hipSuccess;
} else {
auto device = ctx->getWriteableDevice();
*ptr = hip_internal::allocAndSharePtr("device_mem", sizeBytes, ctx, false /*shareWithAll*/,
@@ -700,14 +722,15 @@ hipError_t hipExtMallocWithFlags(void** ptr, size_t sizeBytes, unsigned int flag
HIP_SET_DEVICE();
#if (__hcc_workweek__ >= 19115)
if (sizeBytes == 0) {
if (ptr) *ptr = NULL;
return ihipLogStatus(hipSuccess);
}
hipError_t hip_status = hipSuccess;
auto ctx = ihipGetTlsDefaultCtx();
// return NULL pointer when malloc size is 0
if (sizeBytes == 0) {
*ptr = NULL;
hip_status = hipSuccess;
} else if ((ctx == nullptr) || (ptr == nullptr)) {
if ((ctx == nullptr) || (ptr == nullptr)) {
hip_status = hipErrorInvalidValue;
} else {
unsigned amFlags = 0;
@@ -736,6 +759,9 @@ hipError_t hipExtMallocWithFlags(void** ptr, size_t sizeBytes, unsigned int flag
hipError_t hipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags) {
HIP_INIT_SPECIAL_API(hipHostMalloc, (TRACE_MEM), ptr, sizeBytes, flags);
HIP_SET_DEVICE();
if (sizeBytes == 0) {
return ihipLogStatus(hipSuccess);
}
hipError_t hip_status = hipSuccess;
hip_status = hip_internal::ihipHostMalloc(tls, ptr, sizeBytes, flags);
return ihipLogStatus(hip_status);
@@ -744,6 +770,9 @@ hipError_t hipHostMalloc(void** ptr, size_t sizeBytes, unsigned int flags) {
hipError_t hipMallocManaged(void** devPtr, size_t size, unsigned int flags) {
HIP_INIT_SPECIAL_API(hipMallocManaged, (TRACE_MEM), devPtr, size, flags);
HIP_SET_DEVICE();
if (size == 0) {
return ihipLogStatus(hipSuccess);
}
hipError_t hip_status = hipSuccess;
if(flags != hipMemAttachGlobal)
hip_status = hipErrorInvalidValue;
@@ -1224,6 +1253,7 @@ hipError_t hipMemcpyToSymbol(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, dst);
if (count == 0) return ihipLogStatus(hipSuccess);
if (dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1246,6 +1276,7 @@ hipError_t hipMemcpyFromSymbol(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, dst);
if (count == 0) return ihipLogStatus(hipSuccess);
if (src == nullptr || dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1269,6 +1300,7 @@ hipError_t hipMemcpyToSymbolAsync(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, dst);
if (count == 0) return ihipLogStatus(hipSuccess);
if (dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1301,6 +1333,7 @@ hipError_t hipMemcpyFromSymbolAsync(void* dst, const void* src, size_t count,
tprintf(DB_MEM, " symbol '%s' resolved to address:%p\n", symbol_name, src);
if (count == 0) return ihipLogStatus(hipSuccess);
if (src == nullptr || dst == nullptr) {
return ihipLogStatus(hipErrorInvalidSymbol);
}
@@ -1592,6 +1625,7 @@ hipError_t ihipMemcpy3D(const struct hipMemcpy3DParms* p, hipStream_t stream, bo
srcXoffset = p->srcPos.x;
srcYoffset = p->srcPos.y;
srcZoffset = p->srcPos.z;
if (copyWidth == 0) return hipSuccess;
if (p->dstArray != nullptr) {
if ((p->dstArray->isDrv == true) ||( p->dstPtr.ptr!= nullptr)){
return hipErrorInvalidValue;
@@ -1933,6 +1967,7 @@ hipError_t getLockedPointer(void *hostPtr, size_t dataLen, void **devicePtrPtr)
// TODO - review and optimize
hipError_t ihipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
size_t height, hipMemcpyKind kind) {
if (height == 0 || width == 0) return hipSuccess;
if (dst == nullptr || src == nullptr || width > dpitch || width > spitch) return hipErrorInvalidValue;
hipStream_t stream = ihipSyncAndResolveStream(hipStreamNull);
@@ -1989,6 +2024,7 @@ hipError_t hipMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch,
hipError_t ihipMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width,
size_t height, hipMemcpyKind kind, hipStream_t stream) {
if (height == 0 || width == 0) return hipSuccess;
if (dst == nullptr || src == nullptr || width > dpitch || width > spitch) return hipErrorInvalidValue;
hipError_t e = hipSuccess;
int isLockedOrD2D = 0;
@@ -2043,6 +2079,7 @@ hipError_t ihip2dOffsetMemcpy(void* dst, size_t dpitch, const void* src, size_t
size_t height, size_t srcXOffsetInBytes, size_t srcYOffset,
size_t dstXOffsetInBytes, size_t dstYOffset,hipMemcpyKind kind,
hipStream_t stream, bool isAsync) {
if (height == 0 || width == 0) return hipSuccess;
if((spitch < width + srcXOffsetInBytes) || (srcYOffset >= height)){
return hipErrorInvalidValue;
} else if((dpitch < width + dstXOffsetInBytes) || (dstYOffset >= height)){
@@ -2061,6 +2098,7 @@ hipError_t ihipMemcpyParam2D(const hip_Memcpy2D* pCopy, hipStream_t stream, bool
if (pCopy == nullptr) {
return hipErrorInvalidValue;
}
if (pCopy->Height == 0 || pCopy->WidthInBytes == 0) return hipSuccess;
void* dst; const void* src;
size_t spitch = pCopy->srcPitch;
size_t dpitch = pCopy->dstPitch;
@@ -2140,6 +2178,7 @@ hipError_t hipMemcpy2DFromArray( void* dst, size_t dpitch, hipArray_const_t src,
hipError_t hipMemcpy2DFromArrayAsync( void* dst, size_t dpitch, hipArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, hipMemcpyKind kind, hipStream_t stream ){
HIP_INIT_SPECIAL_API(hipMemcpy2DFromArrayAsync, (TRACE_MCMD), dst, dpitch, src, wOffset, hOffset, width, height, kind, stream);
size_t byteSize;
if (height == 0 || width == 0) return ihipLogStatus(hipSuccess);
if(src) {
switch (src->desc.f) {
case hipChannelFormatKindSigned:
@@ -2239,8 +2278,6 @@ hipError_t hipMemGetInfo(size_t* free, size_t* total) {
auto device = ctx->getWriteableDevice();
if (total) {
*total = device->_props.totalGlobalMem;
} else {
e = hipErrorInvalidValue;
}
if (free) {
@@ -2263,8 +2300,6 @@ hipError_t hipMemGetInfo(size_t* free, size_t* total) {
} else {
return ihipLogStatus(hipErrorInvalidValue);
}
} else {
e = hipErrorInvalidValue;
}
} else {