SWDEV-531711 - Report correct error code based on device failure. (#286)

[ROCm/clr commit: f5b8db33f1]

projects/clr/hipamd/src/hip_internal.hpp

@@ -140,19 +140,23 @@ const char* ihipGetErrorName(hipError_t hip_error);
 
 // This macro should be called at the beginning of every HIP API.
 #define HIP_INIT_API(cid, ...)                                                                     \
+  if (amd::Device::IsGPUInError()) {                                                              \
+    HIP_RETURN(ConvertCLErrorIntoHIPError(amd::Device::GetGPUError()));                            \
+  }                                                                                                \
   HIP_INIT_API_INTERNAL(0, cid, __VA_ARGS__)                                                       \
   if (hip::g_devices.size() == 0) {                                                                \
     HIP_RETURN(hipErrorNoDevice);                                                                  \
-  }
+  }                                                                                                \
 
 #define HIP_INIT_API_NO_RETURN(cid, ...)                                                           \
   HIP_INIT_API_INTERNAL(1, cid, __VA_ARGS__)
 
 #define HIP_RETURN_DURATION(ret, ...)                                                              \
   hip::tls.last_command_error_ = ret;                                                              \
-  if (amd::Device::IsDeviceNotUsable()) {                                                          \
-    hip::tls.last_error_ = hipErrorNoDevice;                                                       \
-    hip::tls.last_command_error_ = hipErrorNoDevice;                                               \
+  if (amd::Device::IsGPUInError()) {                                                               \
+    hipError_t hip_error = ConvertCLErrorIntoHIPError(amd::Device::GetGPUError());                 \
+    hip::tls.last_error_ = hip_error;                                                              \
+    hip::tls.last_command_error_ = hip_error;                                                      \
   } else if (DEBUG_HIP_7_PREVIEW & amd::CHANGE_HIP_GET_LAST_ERROR) {                               \
     if (hip::tls.last_command_error_ != hipSuccess &&                                              \
            hip::tls.last_command_error_ != hipErrorNotReady) {                                     \
@@ -168,9 +172,10 @@ const char* ihipGetErrorName(hipError_t hip_error);
 
 #define HIP_RETURN(ret, ...)                                                                       \
   hip::tls.last_command_error_ = ret;                                                              \
-  if (amd::Device::IsDeviceNotUsable()) {                                                          \
-    hip::tls.last_error_ = hipErrorNoDevice;                                                       \
-    hip::tls.last_command_error_ = hipErrorNoDevice;                                               \
+  if (amd::Device::IsGPUInError()) {                                                               \
+    hipError_t hip_error = ConvertCLErrorIntoHIPError(amd::Device::GetGPUError());                 \
+    hip::tls.last_error_ = hip_error;                                                              \
+    hip::tls.last_command_error_ = hip_error;                                                      \
   } else if (DEBUG_HIP_7_PREVIEW & amd::CHANGE_HIP_GET_LAST_ERROR) {                               \
     if (hip::tls.last_command_error_ != hipSuccess &&                                              \
            hip::tls.last_command_error_ != hipErrorNotReady) {                                     \

projects/clr/hipamd/src/hip_platform.cpp

@@ -24,6 +24,7 @@
 #include "hip_internal.hpp"
 #include "platform/program.hpp"
 #include "platform/runtime.hpp"
+#include "utils/flags.hpp"
 
 #include <unordered_map>
 #include <mutex>
@@ -185,13 +186,16 @@ void __hipRegisterTexture(
 
 void __hipUnregisterFatBinary(hip::FatBinaryInfo** modules) {
   static std::once_flag unregister_device_sync;
-  std::call_once(unregister_device_sync, [](){
-    for (auto& hipDevice : g_devices) {
-      // By synchronizing devices ensure that all HSA signal handlers
-      // complete before removeFatBinary
-      hipDevice->SyncAllStreams(true);
-    }
-  });
+  // If SKIP ABORT is set and GPU is in error, dont need to sync streams.
+  if (!HIP_SKIP_ABORT_ON_GPU_ERROR || !amd::Device::IsGPUInError()) {
+    std::call_once(unregister_device_sync, [](){
+      for (auto& hipDevice : g_devices) {
+        // By synchronizing devices ensure that all HSA signal handlers
+        // complete before removeFatBinary
+        hipDevice->SyncAllStreams(true);
+      }
+    });
+  }
   hipError_t err = PlatformState::instance().removeFatBinary(modules);
   guarantee((err == hipSuccess), "Cannot Unregister Fat Binary, error:%d", err);
 }

projects/clr/hipamd/src/trace_helper.h

@@ -245,3 +245,35 @@ inline std::string ToString(T first, Args... args) {
     return ToString(first) + ", " + ToString(args...);
 }
 
+
+inline hipError_t ConvertCLErrorIntoHIPError(cl_int cl_error) {
+  hipError_t hip_error = hipSuccess;
+  switch (cl_error) {
+    case CL_INVALID_OPERATION :
+      hip_error = hipErrorLaunchFailure;
+      break;
+    case CL_MEM_OBJECT_ALLOCATION_FAILURE :
+      hip_error = hipErrorIllegalAddress;
+      break;
+    case CL_INVALID_PROGRAM :
+      hip_error = hipErrorInvalidSource;
+      break;
+    case CL_INVALID_ARG_VALUE :
+      hip_error = hipErrorInvalidValue;
+      break;
+    case CL_INVALID_KERNEL :
+      hip_error = hipErrorInvalidKernelFile;
+      break;
+    case CL_BUILD_PROGRAM_FAILURE :
+      hip_error = hipErrorLaunchFailure;
+      break;
+    case CL_INVALID_MEM_OBJECT :
+      hip_error = hipErrorIllegalAddress;
+      break;
+    case CL_DEVICE_NOT_AVAILABLE:
+    default:
+      hip_error = hipErrorUnknown;
+      break;
+  }
+  return hip_error;
+}