AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-283515 - Fix crashing in kernel launch on MGPUS

Browse Source 
Fix wrong mixing of current device and stream device in
ihipModuleLaunchKernel() and hipLaunchCooperativeKernel().

Fix missing hipSetDevice() in hipMemcpyWithStream* tests.

Change-Id: I09333bb40d239bb42c832df5ea16d17eeaeff5e7
This commit is contained in:
Tao Sang
2021-04-27 19:03:22 -04:00
committed by Tao Sang
parent 6b14050424
commit a95ff95bf7
7 changed files with 25 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rocclr/hip_internal.hpp
+3
View File
@@ -158,6 +158,8 @@ namespace hip {
void Finish() const;
/// Get device ID associated with the current stream;
int DeviceId() const;
/// Get device ID associated with a stream;
static int DeviceId(const hipStream_t hStream);
/// Returns if stream is null stream
bool Null() const { return null_; }
/// Returns the lock object for the current stream
@@ -263,6 +265,7 @@ extern void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null
extern std::vector<hip::Device*> g_devices;
extern hipError_t ihipDeviceGetCount(int* count);
extern int ihipGetDevice();
extern hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags);
extern amd::Memory* getMemoryObject(const void* ptr, size_t& offset);
extern amd::Memory* getMemoryObjectWithOffset(const void* ptr, const size_t size);
rocclr/hip_module.cpp
+7 -5
View File
@@ -213,7 +213,7 @@ inline hipError_t ihipLaunchKernel_validate(hipFunction_t f, uint32_t globalWork
uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
uint32_t blockDimX, uint32_t blockDimY,
uint32_t blockDimZ, uint32_t sharedMemBytes,
void** kernelParams, void** extra,
void** kernelParams, void** extra, int deviceId,
uint32_t params = 0) {
if (f == nullptr) {
LogPrintfError("%s", "Function passed is null");
@@ -235,7 +235,8 @@ inline hipError_t ihipLaunchKernel_validate(hipFunction_t f, uint32_t globalWork
return hipErrorNotInitialized;
}
}
const amd::Device* device = hip::getCurrentDevice()->devices()[0];
const amd::Device* device = g_devices[deviceId]->devices()[0];
// Make sure dispatch doesn't exceed max workgroup size limit
if (blockDimX * blockDimY * blockDimZ > device->info().maxWorkGroupSize_) {
return hipErrorInvalidConfiguration;
@@ -354,7 +355,8 @@ hipError_t ihipModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra,
startEvent, stopEvent, flags, params);
HIP_RETURN_ONFAIL(PlatformState::instance().initStatManagedVarDevicePtr(ihipGetDevice()));
int deviceId = hip::Stream::DeviceId(hStream);
HIP_RETURN_ONFAIL(PlatformState::instance().initStatManagedVarDevicePtr(deviceId));
if (f == nullptr) {
LogPrintfError("%s", "Function passed is null");
return hipErrorInvalidImage;
@@ -365,7 +367,7 @@ hipError_t ihipModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
hipError_t status =
ihipLaunchKernel_validate(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX,
blockDimY, blockDimZ, sharedMemBytes, kernelParams, extra, params);
blockDimY, blockDimZ, sharedMemBytes, kernelParams, extra, deviceId, params);
if (status != hipSuccess) {
return status;
}
@@ -505,8 +507,8 @@ hipError_t hipLaunchCooperativeKernel(const void* f,
HIP_INIT_API(hipLaunchCooperativeKernel, f, gridDim, blockDim,
sharedMemBytes, hStream);
int deviceId = ihipGetDevice();
hipFunction_t func = nullptr;
int deviceId = hip::Stream::DeviceId(hStream);
HIP_RETURN_ONFAIL(PlatformState::instance().getStatFunc(&func, f, deviceId));
size_t globalWorkSizeX = static_cast<size_t>(gridDim.x) * blockDim.x;
size_t globalWorkSizeY = static_cast<size_t>(gridDim.y) * blockDim.y;
rocclr/hip_platform.cpp
+1 -7
View File
@@ -621,14 +621,8 @@ hipError_t ihipLaunchKernel(const void* hostFunction,
hipEvent_t stopEvent,
int flags)
{
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
int deviceId = (s != nullptr)? s->DeviceId() : ihipGetDevice();
if (deviceId == -1) {
LogPrintfError("Wrong Device Id: %d \n", deviceId);
HIP_RETURN(hipErrorNoDevice);
}
hipFunction_t func = nullptr;
int deviceId = hip::Stream::DeviceId(stream);
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, hostFunction, deviceId);
if ((hip_error != hipSuccess) || (func == nullptr)) {
HIP_RETURN(hipErrorInvalidDeviceFunction);
rocclr/hip_stream.cpp
+7
View File
@@ -114,6 +114,13 @@ int Stream::DeviceId() const {
return device_->deviceId();
}
int Stream::DeviceId(const hipStream_t hStream) {
hip::Stream* s = reinterpret_cast<hip::Stream*>(hStream);
int deviceId = (s != nullptr)? s->DeviceId() : ihipGetDevice();
assert(deviceId >= 0 && deviceId < static_cast<int>(g_devices.size()));
return deviceId;
}
void Stream::syncNonBlockingStreams() {
amd::ScopedLock lock(streamSetLock);
for (auto& it : streamSet) {