AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-477324 - Capture Memcpy1D pinned H2D D2H

Browse Source 
Change-Id: I1f4744f20a9caeed005ec68da44e5fde737e09f7


[ROCm/clr commit: 742b0210d3]
This commit is contained in:
Anusha GodavarthySurya
2024-08-27 13:46:07 +00:00
committed by Anusha Godavarthy Surya
parent 887b11894b
commit c0ceb1cf12
6 changed files with 102 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/hipamd/src/hip_memory.cpp
+67 -39
View File
@@ -463,6 +463,35 @@ hipError_t ihipMemcpy_validate(void* dst, const void* src, size_t sizeBytes,
return hipSuccess;
}
hip::MemcpyType ihipGetMemcpyType(const void* src, void* dst, hipMemcpyKind kind) {
size_t sOffset = 0;
amd::Memory* srcMemory = getMemoryObject(src, sOffset);
size_t dOffset = 0;
amd::Memory* dstMemory = getMemoryObject(dst, dOffset);
hip::MemcpyType type;
if(srcMemory == nullptr && dstMemory == nullptr) {
type = hipHostToHost;
} else if ((srcMemory == nullptr) && (dstMemory != nullptr)) {
type = hipWriteBuffer;
} else if ((srcMemory != nullptr) && (dstMemory == nullptr)) {
type = hipReadBuffer;
} else if ((srcMemory != nullptr) && (dstMemory != nullptr)) {
// Check if the queue device doesn't match the device on any memory object.
// And any of them are not host allocation.
// Hence it's a P2P transfer, because the app has requested access to another GPU
if ((srcMemory->GetDeviceById() != dstMemory->GetDeviceById()) &&
((srcMemory->getContext().devices().size() == 1) &&
(dstMemory->getContext().devices().size() == 1))) {
type = hipCopyBufferP2P;
} else if( kind == hipMemcpyDeviceToDeviceNoCU) {
type = hipCopyBufferSDMA;
} else {
type = hipCopyBuffer;
}
}
return type;
}
// ================================================================================================
hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src, size_t sizeBytes,
hipMemcpyKind kind, hip::Stream& stream, bool isAsync) {
@@ -473,37 +502,37 @@ hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src,
amd::Memory* dstMemory = getMemoryObject(dst, dOffset);
amd::Device* queueDevice = &stream.device();
amd::CopyMetadata copyMetadata(isAsync, amd::CopyMetadata::CopyEnginePreference::NONE);
if ((srcMemory == nullptr) && (dstMemory != nullptr)) {
hip::Stream* pStream = &stream;
if (queueDevice != dstMemory->GetDeviceById()) {
pStream = hip::getNullStream(dstMemory->GetDeviceById()->context());
amd::Command* cmd = stream.getLastQueuedCommand(true);
if (cmd != nullptr) {
waitList.push_back(cmd);
hip::MemcpyType type = ihipGetMemcpyType(src, dst, kind);
hip::Stream* pStream = &stream;
switch (type) {
case hipWriteBuffer:
if (queueDevice != dstMemory->GetDeviceById()) {
pStream = hip::getNullStream(dstMemory->GetDeviceById()->context());
amd::Command* cmd = stream.getLastQueuedCommand(true);
if (cmd != nullptr) {
waitList.push_back(cmd);
}
}
}
command = new amd::WriteMemoryCommand(*pStream, CL_COMMAND_WRITE_BUFFER, waitList,
*dstMemory->asBuffer(), dOffset, sizeBytes, src, 0, 0, copyMetadata);
} else if ((srcMemory != nullptr) && (dstMemory == nullptr)) {
hip::Stream* pStream = &stream;
if (queueDevice != srcMemory->GetDeviceById()) {
pStream = hip::getNullStream(srcMemory->GetDeviceById()->context());
amd::Command* cmd = stream.getLastQueuedCommand(true);
if (cmd != nullptr) {
waitList.push_back(cmd);
command = new amd::WriteMemoryCommand(*pStream, CL_COMMAND_WRITE_BUFFER, waitList,
*dstMemory->asBuffer(), dOffset, sizeBytes, src, 0, 0,
copyMetadata);
break;
case hipReadBuffer:
if (queueDevice != srcMemory->GetDeviceById()) {
pStream = hip::getNullStream(srcMemory->GetDeviceById()->context());
amd::Command* cmd = stream.getLastQueuedCommand(true);
if (cmd != nullptr) {
waitList.push_back(cmd);
}
}
}
command = new amd::ReadMemoryCommand(*pStream, CL_COMMAND_READ_BUFFER, waitList,
*srcMemory->asBuffer(), sOffset, sizeBytes, dst, 0, 0, copyMetadata);
} else if ((srcMemory != nullptr) && (dstMemory != nullptr)) {
// Check if the queue device doesn't match the device on any memory object.
// And any of them are not host allocation.
// Hence it's a P2P transfer, because the app has requested access to another GPU
if ((srcMemory->GetDeviceById() != dstMemory->GetDeviceById()) &&
((srcMemory->getContext().devices().size() == 1) &&
(dstMemory->getContext().devices().size() == 1))) {
command = new amd::ReadMemoryCommand(*pStream, CL_COMMAND_READ_BUFFER, waitList,
*srcMemory->asBuffer(), sOffset, sizeBytes, dst, 0, 0,
copyMetadata);
break;
case hipCopyBufferP2P:
command = new amd::CopyMemoryP2PCommand(stream, CL_COMMAND_COPY_BUFFER, waitList,
*srcMemory->asBuffer(), *dstMemory->asBuffer(), sOffset, dOffset, sizeBytes);
*srcMemory->asBuffer(), *dstMemory->asBuffer(),
sOffset, dOffset, sizeBytes);
if (command == nullptr) {
return hipErrorOutOfMemory;
}
@@ -513,10 +542,12 @@ hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src,
delete command;
return hipErrorInvalidValue;
}
} else {
hip::Stream* pStream = &stream;
break;
case hipCopyBufferSDMA:
copyMetadata.copyEnginePreference_ = amd::CopyMetadata::CopyEnginePreference::SDMA;
case hipCopyBuffer:
if ((srcMemory->GetDeviceById() == dstMemory->GetDeviceById()) &&
(queueDevice != srcMemory->GetDeviceById())) {
queueDevice != srcMemory->GetDeviceById()) {
pStream = hip::getNullStream(srcMemory->GetDeviceById()->context());
amd::Command* cmd = stream.getLastQueuedCommand(true);
if (cmd != nullptr) {
@@ -531,7 +562,7 @@ hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src,
if (cmd != nullptr) {
waitList.push_back(cmd);
}
// Scenarios such as HtoD where src is pinned memory
// Scenarios such as HtoD where src is pinned memory
} else if ((queueDevice != dstMemory->GetDeviceById()) &&
(srcMemory->getContext().devices().size() != 1)) {
pStream = hip::getNullStream(dstMemory->GetDeviceById()->context());
@@ -541,14 +572,10 @@ hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src,
}
}
}
copyMetadata.copyEnginePreference_ = (kind == hipMemcpyDeviceToDeviceNoCU) ?
amd::CopyMetadata::CopyEnginePreference::SDMA :
amd::CopyMetadata::CopyEnginePreference::NONE;
command = new amd::CopyMemoryCommand(*pStream, CL_COMMAND_COPY_BUFFER, waitList,
*srcMemory->asBuffer(), *dstMemory->asBuffer(), sOffset, dOffset, sizeBytes,
copyMetadata);
}
*srcMemory->asBuffer(), *dstMemory->asBuffer(), sOffset,
dOffset, sizeBytes, copyMetadata);
break;
}
if (command == nullptr) {
return hipErrorOutOfMemory;
@@ -558,6 +585,7 @@ hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src,
}
return hipSuccess;
}
bool IsHtoHMemcpy(void* dst, const void* src) {
size_t sOffset = 0;
amd::Memory* srcMemory = getMemoryObject(src, sOffset);