SWDEV-299127 - Merge 'develop' into 'amd-staging'

Change-Id: I084d0d56ef56f7a466a601223fdc872a33024599

docs/markdown/hip_faq.md

@@ -105,14 +105,15 @@ However, we can provide a rough summary of the features included in each CUDA SD
     - Cooperative Launch, Surface Object Management, Version Management
 
 ### What libraries does HIP support?
-HIP includes growing support for the four key math libraries using hcBlas, hcFft, hcrng and hcsparse, as well as MIOpen for machine intelligence applications.
+HIP includes growing support for the four key math libraries using hipBlas, hipFFt, hipRAND and hipSPARSE, as well as MIOpen for machine intelligence applications.
 These offer pointer-based memory interfaces (as opposed to opaque buffers) and can be easily interfaced with other HIP applications.
 The hip interfaces support both ROCm and CUDA paths, with familiar library interfaces.
 
 - [hipBlas](https://github.com/ROCmSoftwarePlatform/hipBLAS), which utilizes [rocBlas](https://github.com/ROCmSoftwarePlatform/rocBLAS).
-- [hipfft](https://github.com/ROCmSoftwarePlatform/hcFFT)
-- [hipsparse](https://github.com/ROCmSoftwarePlatform/hcSPARSE)
-- [hiprng](https://github.com/ROCmSoftwarePlatform/hcrng)
+- [hipFFt](https://github.com/ROCmSoftwarePlatform/hipfft)
+- [hipsSPARSE](https://github.com/ROCmSoftwarePlatform/hipsparse)
+- [hipRAND](https://github.com/ROCmSoftwarePlatform/hipRAND)
+- [MIOpen](https://github.com/ROCmSoftwarePlatform/MIOpen)
 
 Additionally, some of the cublas routines are automatically converted to hipblas equivalents by the HIPIFY tools. These APIs use cublas or hcblas depending on the platform and replace the need to use conditional compilation.
 

tests/catch/hipTestMain/config/config_amd_windows_MI2xx.json

@@ -90,6 +90,8 @@
 		"Unit_hipStreamValue_Wait64_Blocking_NoMask_And",
 		"Unit_hipStreamValue_Wait64_Blocking_NoMask_Nor",
 		"Unit_hipDeviceGetPCIBusId_Negative_PartialFill",
-		"Unit_hipInit_Negative"
+		"Unit_hipInit_Negative",
+		"Unit_hipGraphAddEventRecordNode_Functional_ElapsedTime",
+		"Unit_hipStreamBeginCapture_captureComplexGraph"
 	]
 }

tests/catch/hipTestMain/config/config_amd_windows_common.json

@@ -85,10 +85,12 @@
 	   "Unit_hipGraphNodeGetDependentNodes_Functional",
 	   "Unit_hipGraphNodeGetDependencies_Functional",
 	   "Unit_hipGraphExecChildGraphNodeSetParams_ChildTopology",
+	   "Unit_hipGraphAddEventRecordNode_MultipleRun",
+	   "Unit_hipGraphAddEventRecordNode_Functional_ElapsedTime",
+	   "Unit_hipStreamBeginCapture_captureComplexGraph",
 	   "Note: needs to be enabled when streamPerThread issues are fixed",
 	   "Unit_hipStreamSynchronize_NullStreamAndStreamPerThread",
 	   "Note: intermittent Seg fault failure ",
-	   "Unit_hipGraphAddEventRecordNode_Functional_WithoutFlags",
-	   "Unit_hipGraphAddEventRecordNode_MultipleRun"
+	   "Unit_hipGraphAddEventRecordNode_Functional_WithoutFlags"
 	]
 }

tests/catch/hipTestMain/hip_test_context.cc

@@ -4,6 +4,7 @@
 #include <fstream>
 #include <sstream>
 #include <regex>
+#include "hip_test_context.hh"
 #include "hip_test_filesystem.hh"
 
 void TestContext::detectOS() {
@@ -98,11 +99,11 @@ void TestContext::getConfigFiles() {
     abort();
   }
 
-  const char* env_config = std::getenv("HIP_CATCH_EXCLUDE_FILE");
+  std::string env_config = TestContext::getEnvVar("HIP_CATCH_EXCLUDE_FILE");
   LogPrintf("Env Config file: %s",
-            (env_config != nullptr) ? env_config : "Not found, using common config");
+            (!env_config.empty()) ? env_config.c_str() : "Not found, using common config");
   // HIP_CATCH_EXCLUDE_FILE is set for custom file path
-  if (env_config != nullptr) {
+  if (!env_config.empty()) {
     if(fs::exists(env_config)) {
       config_.json_files.push_back(env_config);
     }

tests/catch/include/hip_test_context.hh

@@ -32,6 +32,7 @@ THE SOFTWARE.
 #include <set>
 #include <unordered_map>
 
+// OS Check
 #if defined(_WIN32)
 #define HT_WIN 1
 #define HT_LINUX 0
@@ -42,6 +43,7 @@ THE SOFTWARE.
 #error "OS not recognized"
 #endif
 
+// Platform check
 #if defined(__HIP_PLATFORM_HCC__) || defined(__HIP_PLATFORM_AMD__)
 #define HT_AMD 1
 #define HT_NVIDIA 0
@@ -52,16 +54,6 @@ THE SOFTWARE.
 #error "Platform not recognized"
 #endif
 
-static int _log_enable = (std::getenv("HT_LOG_ENABLE") ? 1 : 0);
-
-#define LogPrintf(format, ...)                                                                     \
-  {                                                                                                \
-    if (_log_enable) {                                                                             \
-      printf(format, __VA_ARGS__);                                                                 \
-      printf("%c", '\n');                                                                          \
-    }                                                                                              \
-  }
-
 typedef struct Config_ {
   std::vector<std::string> json_files;  // Json files
   std::string platform;   // amd/nvidia
@@ -124,6 +116,26 @@ class TestContext {
     return instance;
   }
 
+  static std::string getEnvVar(std::string var) {
+    #if defined(_WIN32)
+    rsize_t MAX_LEN = 4096;
+    char dstBuf[MAX_LEN];
+    size_t dstSize;
+    if (!::getenv_s(&dstSize, dstBuf, MAX_LEN, var.c_str())) {
+      return std::string(dstBuf);
+    }
+    #elif defined(__linux__)
+    char* val = std::getenv(var.c_str());
+    if (val != NULL) {
+      return std::string(val);
+    }
+    #else
+    #error "OS not recognized"
+    #endif
+    return std::string("");
+  }
+
+
   bool isWindows() const;
   bool isLinux() const;
   bool isNvidia() const;
@@ -171,3 +183,14 @@ class TestContext {
 
   ~TestContext();
 };
+
+static bool _log_enable = (!TestContext::getEnvVar("HT_LOG_ENABLE").empty() ? true : false);
+
+// printing logs
+#define LogPrintf(format, ...)                                                                   \
+{                                                                                                \
+  if(_log_enable) {                                                                              \
+    printf(format, __VA_ARGS__);                                                                 \
+    printf("%c", '\n');                                                                          \
+  }                                                                                              \
+}

tests/catch/unit/deviceLib/CMakeLists.txt

@@ -51,10 +51,10 @@ set(AMD_ARCH_SPEC_TEST_SRC
 # Note to pass arch use format like -DOFFLOAD_ARCH_STR="--offload-arch=gfx900  --offload-arch=gfx906"
 # having space at the start/end of OFFLOAD_ARCH_STR can cause build failures
 if(UNIX)
-    add_custom_target(kerDevAllocMultCO.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevAllocMultCO.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevAllocMultCO.code -I${CMAKE_CURRENT_SOURCE_DIR}/../../../../include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
-    add_custom_target(kerDevWriteMultCO.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevWriteMultCO.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevWriteMultCO.code -I${CMAKE_CURRENT_SOURCE_DIR}/../../../../include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
-    add_custom_target(kerDevFreeMultCO.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevFreeMultCO.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevFreeMultCO.code -I${CMAKE_CURRENT_SOURCE_DIR}/../../../../include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
-    add_custom_target(kerDevAllocSingleKer.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevAllocSingleKer.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevAllocSingleKer.code -I${CMAKE_CURRENT_SOURCE_DIR}/../../../../include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
+    add_custom_target(kerDevAllocMultCO.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevAllocMultCO.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevAllocMultCO.code -I${HIP_PATH}/include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
+    add_custom_target(kerDevWriteMultCO.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevWriteMultCO.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevWriteMultCO.code -I${HIP_PATH}/include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
+    add_custom_target(kerDevFreeMultCO.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevFreeMultCO.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevFreeMultCO.code -I${HIP_PATH}/include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
+    add_custom_target(kerDevAllocSingleKer.code COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} ${CMAKE_CURRENT_SOURCE_DIR}/kerDevAllocSingleKer.cc -o ${CMAKE_CURRENT_BINARY_DIR}/../../unit/deviceLib/kerDevAllocSingleKer.code -I${HIP_PATH}/include/ -I${CMAKE_CURRENT_SOURCE_DIR}/../../include)
 endif()
 
 if(HIP_PLATFORM MATCHES "amd")

tests/catch/unit/event/CMakeLists.txt

@@ -9,6 +9,7 @@ set(TEST_SRC
     hipEventCreate.cc
     hipEventCreateWithFlags.cc
     hipEventSynchronize.cc
+    Unit_hipEventMGpuMThreads.cc
 )
 
 # The test used wait mechanism and doesnt play well with all arch of nvidia

tests/catch/unit/event/Unit_hipEventMGpuMThreads.cc

@@ -0,0 +1,215 @@
+/*
+ Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved.
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+ */
+#define HIP_TEMPLATE_KERNEL_LAUNCH
+#include <hip_test_common.hh>
+#include <stdio.h>
+#include <ratio>
+#include <chrono>
+
+int64_t timeNanos() {
+  using namespace std::chrono;
+  static auto t0 = steady_clock::now();
+  auto timeSpan = duration_cast<std::chrono::nanoseconds>(steady_clock::now() - t0);
+  return timeSpan.count();
+}
+
+#define WIDTH 1024
+
+#define NUM (WIDTH * WIDTH)
+
+#define THREADS_PER_BLOCK_X 4
+#define THREADS_PER_BLOCK_Y 4
+#define THREADS_PER_BLOCK_Z 1
+
+__global__ void matrixTranspose(float *out, float *in, const int width) {
+  int x = blockDim.x * blockIdx.x + threadIdx.x;
+  int y = blockDim.y * blockIdx.y + threadIdx.y;
+
+  out[y * width + x] = in[x * width + y];
+}
+
+// CPU implementation of matrix transpose
+void matrixTransposeCPUReference(float *output, float *input,
+                                 const unsigned int width) {
+  for (unsigned int j = 0; j < width; j++) {
+    for (unsigned int i = 0; i < width; i++) {
+      output[i * width + j] = input[j * width + i];
+    }
+  }
+}
+
+void thread_run(const int iThread) {
+  int i = 0;
+  int errors = 0;
+  float eventMs = 1.0f;
+  float *matrix = nullptr;
+  float *transposeMatrix = nullptr;
+  float *cpuTransposeMatrix = nullptr;
+  float *gpuMatrix = nullptr;
+  float *gpuTransposeMatrix = nullptr;
+  hipDeviceProp_t devProp;
+  memset(&devProp, 0, sizeof(devProp));
+  HIP_CHECK(hipGetDeviceProperties(&devProp, iThread));
+  fprintf(stderr, "[%d] device name = %s\n", iThread, devProp.name);
+
+  HIP_CHECK(hipSetDevice(iThread));
+  hipEvent_t start, stop;
+
+  auto time = timeNanos();
+  HIP_CHECK(hipEventCreate(&start));
+  fprintf(stderr, "[%d] hipEventCreate(&start) cost cpu time %6.3fms\n", iThread,
+          (timeNanos() - time)/1000000.0);
+
+  HIP_CHECK(hipEventCreate(&stop));
+
+  matrix = (float*) malloc(NUM * sizeof(float));
+  transposeMatrix = (float*) malloc(NUM * sizeof(float));
+  cpuTransposeMatrix = (float*) malloc(NUM * sizeof(float));
+
+  // initialize the input data
+  for (i = 0; i < NUM; i++) {
+    matrix[i] = (float) i * 10.0f;
+  }
+
+  // allocate the memory on the device side
+  HIP_CHECK(hipMalloc((void**) &gpuMatrix, NUM * sizeof(float)));
+  HIP_CHECK(hipMalloc((void**) &gpuTransposeMatrix, NUM * sizeof(float)));
+
+  time = timeNanos();
+  // Record the start event
+  // The first call of hipEventRecord will trigger VirtualDevice creation that will trigger building
+  // of BlitLinearSourceCode, which will cost 200+ ms.
+  HIP_CHECK(hipEventRecord(start));
+  fprintf(stderr, "[%d] hipEventRecord(&start) cost cpu time %6.3fms\n", iThread,
+          (timeNanos() - time)/1000000.0);
+
+  time = timeNanos();
+  // Memory transfer from host to device
+  HIP_CHECK(hipMemcpy(gpuMatrix, matrix, NUM * sizeof(float), hipMemcpyHostToDevice));
+
+  // Record the stop event
+  HIP_CHECK(hipEventRecord(stop));
+  HIP_CHECK(hipEventSynchronize(stop));
+
+  HIP_CHECK(hipEventElapsedTime(&eventMs, start, stop));
+
+  fprintf(stderr, "[%d] hipMemcpyHostToDevice cost gpu time %6.3fms, cpu time %6.3fms\n",
+          iThread, eventMs, (timeNanos() - time)/1000000.0);
+
+  // Record the start event
+  HIP_CHECK(hipEventRecord(start));
+
+  time = timeNanos();
+  // Lauching kernel from host
+  hipLaunchKernelGGL(
+      matrixTranspose,
+      dim3(WIDTH / THREADS_PER_BLOCK_X, WIDTH / THREADS_PER_BLOCK_Y),
+      dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y), 0, 0, gpuTransposeMatrix,
+      gpuMatrix, WIDTH);
+  // Record the stop event
+  HIP_CHECK(hipEventRecord(stop));
+
+  fprintf(stderr, "[%d] hipLaunchKernelGGL() cost cpu time %6.3fms\n", iThread,
+          (timeNanos() - time)/1000000.0);
+
+  HIP_CHECK(hipEventSynchronize(stop));
+  HIP_CHECK(hipEventElapsedTime(&eventMs, start, stop));
+
+  fprintf(stderr, "[%d] kernel Execution cost gpu time %6.3fms, cpu time = %6.3fms\n",
+          iThread, eventMs, (timeNanos() - time)/1000000.0);
+
+  // Record the start event
+  HIP_CHECK(hipEventRecord(start));
+
+  // Memory transfer from device to host
+  HIP_CHECK(hipMemcpy(transposeMatrix, gpuTransposeMatrix, NUM * sizeof(float),
+            hipMemcpyDeviceToHost));
+
+  // Record the stop event
+  HIP_CHECK(hipEventRecord(stop));
+  HIP_CHECK(hipEventSynchronize(stop));
+
+  HIP_CHECK(hipEventElapsedTime(&eventMs, start, stop));
+
+  fprintf(stderr, "[%d] hipMemcpyDeviceToHost cost gpu time %6.3fms\n", iThread, eventMs);
+
+  // CPU MatrixTranspose computation
+  matrixTransposeCPUReference(cpuTransposeMatrix, matrix, WIDTH);
+
+  // verify the results
+  double eps = 1.0E-6;
+  for (i = 0; i < NUM; i++) {
+    if (std::abs(transposeMatrix[i] - cpuTransposeMatrix[i]) > eps) {
+      errors++;
+    }
+  }
+  if (errors != 0) {
+    fprintf(stderr, "[%d] FAILED: %d errors\n", iThread, errors);
+  } else {
+    fprintf(stderr, "[%d] PASSED\n", iThread);
+  }
+  // free the resources on device side
+  HIP_CHECK(hipFree(gpuMatrix));
+  HIP_CHECK(hipFree(gpuTransposeMatrix));
+
+  // free the resources on host side
+  free(matrix);
+  free(transposeMatrix);
+  free(cpuTransposeMatrix);
+  REQUIRE(errors == 0);
+}
+
+void testEventMGpuMThreads(int nThreads = 1) {
+  int iThread = 0;
+  std::thread *threads = new std::thread[nThreads];
+  for (iThread = 0; iThread < nThreads; iThread++) {
+    threads[iThread] = std::thread(thread_run, iThread);
+  }
+  for (iThread = 0; iThread < nThreads; iThread++) {
+    threads[iThread].join();
+  }
+  delete []threads;
+}
+
+TEST_CASE("Unit_hipEventMGpuMThreads_1") {
+  testEventMGpuMThreads(1);
+}
+
+TEST_CASE("Unit_hipEventMGpuMThreads_2") {
+  int numDevices = 0;
+  HIP_CHECK(hipGetDeviceCount(&numDevices));
+  if (numDevices > 1) {
+    testEventMGpuMThreads(numDevices);
+  } else {
+    SUCCEED("skipped the testcase as number of devices is less than 2");
+  }
+}
+
+TEST_CASE("Unit_hipEventMGpuMThreads_3") {
+  int numDevices = 0;
+  HIP_CHECK(hipGetDeviceCount(&numDevices));
+  if (numDevices > 1) {
+    fprintf(stderr, "First round\n");
+    testEventMGpuMThreads(numDevices);
+    fprintf(stderr, "Second round\n");
+    testEventMGpuMThreads(numDevices);
+  } else {
+    SUCCEED("skipped the testcase as number of devices is less than 2");
+  }
+}

tests/catch/unit/graph/CMakeLists.txt

@@ -80,6 +80,7 @@ set(TEST_SRC
   hipGraphHostNodeGetParams.cc
   hipGraphExecChildGraphNodeSetParams.cc
   hipStreamGetCaptureInfo_v2.cc
+  hipUserObjectCreate.cc
 )
 
 hip_add_exe_to_target(NAME GraphsTest

tests/catch/unit/graph/hipUserObjectCreate.cc

@@ -0,0 +1,650 @@
+/*
+Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <hip_test_common.hh>
+#include <hip_test_checkers.hh>
+#include <hip_test_kernels.hh>
+
+
+/**
+ * Functional Test for API - hipUserObjectCreate
+1) Call hipUserObjectCreate once and release it by calling hipUserObjectRelease
+2) Call hipUserObjectCreate refCount as X and release it by calling
+   hipUserObjectRelease with same refCount.
+3) Call hipUserObjectCreate, retain it by calling hipUserObjectRetain
+   and release it by calling hipUserObjectRelease twice.
+4) Call hipUserObjectCreate with refCount as X, retain it by calling
+   hipUserObjectRetain with count as Y and release it by calling
+   hipUserObjectRelease with count as X+Y.
+ */
+
+struct BoxStruct {
+  int count;
+  BoxStruct() {
+    INFO("Constructor called for Struct!\n");
+  }
+  ~BoxStruct() {
+    INFO("Destructor called for Struct!\n");
+  }
+};
+
+class BoxClass {
+ public:
+  BoxClass() {
+    INFO("Constructor called for Class!\n");
+  }
+  ~BoxClass() {
+    INFO("Destructor called for Class!\n");
+  }
+};
+
+static void destroyStructObj(void *ptr) {
+  BoxStruct *ptr1 = reinterpret_cast<BoxStruct *>(ptr);
+  delete ptr1;
+}
+
+static void destroyClassObj(void *ptr) {
+  BoxClass *ptr2 = reinterpret_cast<BoxClass *>(ptr);
+  delete ptr2;
+}
+
+static void destroyIntObj(void *ptr) {
+  int *ptr2 = reinterpret_cast<int *>(ptr);
+  delete ptr2;
+}
+
+static void destroyFloatObj(void *ptr) {
+  float *ptr2 = reinterpret_cast<float *>(ptr);
+  delete ptr2;
+}
+
+/* 1) Call hipUserObjectCreate once and release it by
+      calling hipUserObjectRelease */
+static void hipUserObjectCreate_Functional_1(void *object,
+                                             void destroyObj(void *)) {
+  hipUserObject_t hObject;
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyObj, 1,
+                                  hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  HIP_CHECK(hipUserObjectRelease(hObject));
+}
+
+TEST_CASE("Unit_hipUserObjectCreate_Functional_1") {
+  SECTION("Called with int Object") {
+    int *object = new int();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_1(object, destroyIntObj);
+  }
+  SECTION("Called with float Object") {
+    float *object = new float();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_1(object, destroyFloatObj);
+  }
+  SECTION("Called with Class Object") {
+    BoxClass *object = new BoxClass();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_1(object, destroyClassObj);
+  }
+  SECTION("Called with Struct Object") {
+    BoxStruct *object = new BoxStruct();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_1(object, destroyStructObj);
+  }
+}
+
+/* 2) Call hipUserObjectCreate refCount as X and release it by
+      calling hipUserObjectRelease with same refCount. */
+static void hipUserObjectCreate_Functional_2(void *object,
+                                             void destroyObj(void *)) {
+  int refCount = 5;
+  hipUserObject_t hObject;
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyObj,
+                                  refCount, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  HIP_CHECK(hipUserObjectRelease(hObject, refCount));
+}
+
+TEST_CASE("Unit_hipUserObjectCreate_Functional_2") {
+  SECTION("Called with int Object") {
+    int *object = new int();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_2(object, destroyIntObj);
+  }
+  SECTION("Called with float Object") {
+    float *object = new float();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_2(object, destroyFloatObj);
+  }
+  SECTION("Called with Class Object") {
+    BoxClass *object = new BoxClass();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_2(object, destroyClassObj);
+  }
+  SECTION("Called with Struct Object") {
+    BoxStruct *object = new BoxStruct();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_2(object, destroyStructObj);
+  }
+}
+
+/* 3) Call hipUserObjectCreate, retain it by calling hipUserObjectRetain
+      and release it by calling hipUserObjectRelease twice. */
+static void hipUserObjectCreate_Functional_3(void *object,
+                                             void destroyObj(void *)) {
+  hipUserObject_t hObject;
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyObj,
+                                  1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  HIP_CHECK(hipUserObjectRetain(hObject));
+  HIP_CHECK(hipUserObjectRelease(hObject));
+  HIP_CHECK(hipUserObjectRelease(hObject));
+}
+
+TEST_CASE("Unit_hipUserObjectCreate_Functional_3") {
+  SECTION("Called with int Object") {
+    int *object = new int();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_3(object, destroyIntObj);
+  }
+  SECTION("Called with float Object") {
+    float *object = new float();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_3(object, destroyFloatObj);
+  }
+  SECTION("Called with Class Object") {
+    BoxClass *object = new BoxClass();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_3(object, destroyClassObj);
+  }
+  SECTION("Called with Struct Object") {
+    BoxStruct *object = new BoxStruct();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_3(object, destroyStructObj);
+  }
+}
+
+/* 4) Call hipUserObjectCreate with refCount as X, retain it by calling
+      hipUserObjectRetain with count as Y and release it by calling
+      hipUserObjectRelease with count as X+Y. */
+static void hipUserObjectCreate_Functional_4(void *object,
+                                             void destroyObj(void *)) {
+  int refCount = 5;
+  int refCountRetain = 8;
+  hipUserObject_t hObject;
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyObj,
+                                  refCount, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  HIP_CHECK(hipUserObjectRetain(hObject, refCountRetain));
+  HIP_CHECK(hipUserObjectRelease(hObject, refCount+refCountRetain));
+}
+
+TEST_CASE("Unit_hipUserObjectCreate_Functional_4") {
+  SECTION("Called with int Object") {
+    int *object = new int();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_4(object, destroyIntObj);
+  }
+  SECTION("Called with float Object") {
+    float *object = new float();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_4(object, destroyFloatObj);
+  }
+  SECTION("Called with Class Object") {
+    BoxClass *object = new BoxClass();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_4(object, destroyClassObj);
+  }
+  SECTION("Called with Struct Object") {
+    BoxStruct *object = new BoxStruct();
+    REQUIRE(object != nullptr);
+    hipUserObjectCreate_Functional_4(object, destroyStructObj);
+  }
+}
+
+
+/**
+ * Negative Test for API - hipUserObjectCreate
+ 1) Pass User Object as nullptr
+ 2) Pass object as nullptr
+ 3) Pass Callback function as nullptr
+ 4) Pass initialRefcount as 0
+ 5) Pass initialRefcount as INT_MAX
+ 6) Pass flag other than hipUserObjectNoDestructorSync
+ */
+
+TEST_CASE("Unit_hipUserObjectCreate_Negative") {
+  hipError_t ret;
+  int *object = new int();
+  REQUIRE(object != nullptr);
+
+  hipUserObject_t hObject;
+  SECTION("Pass User Object as nullptr") {
+    ret = hipUserObjectCreate(nullptr, object, destroyIntObj,
+                               1, hipUserObjectNoDestructorSync);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass object as nullptr") {
+    ret = hipUserObjectCreate(&hObject, nullptr, destroyIntObj,
+                               1, hipUserObjectNoDestructorSync);
+    REQUIRE(hipSuccess == ret);
+  }
+  SECTION("Pass Callback function as nullptr") {
+    ret = hipUserObjectCreate(&hObject, object, nullptr,
+                               1, hipUserObjectNoDestructorSync);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as 0") {
+    ret = hipUserObjectCreate(&hObject, object, destroyIntObj,
+                               0, hipUserObjectNoDestructorSync);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as INT_MAX") {
+    ret = hipUserObjectCreate(&hObject, object, destroyIntObj,
+                               INT_MAX, hipUserObjectNoDestructorSync);
+    REQUIRE(hipSuccess == ret);
+  }
+  SECTION("Pass flag other than hipUserObjectNoDestructorSync") {
+    ret = hipUserObjectCreate(&hObject, object, destroyIntObj,
+                               1, hipUserObjectFlags(9));
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+}
+
+/**
+ * Negative Test for API - hipUserObjectRelease
+ 1) Pass User Object as nullptr
+ 2) Pass initialRefcount as 0
+ 3) Pass initialRefcount as INT_MAX
+ */
+
+TEST_CASE("Unit_hipUserObjectRelease_Negative") {
+  hipError_t ret;
+  int *object = new int();
+  REQUIRE(object != nullptr);
+
+  hipUserObject_t hObject;
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyIntObj,
+                                  1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+
+  SECTION("Pass User Object as nullptr") {
+    ret = hipUserObjectRelease(nullptr, 1);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as 0") {
+    ret = hipUserObjectRelease(hObject, 0);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as INT_MAX") {
+    ret = hipUserObjectRelease(hObject, INT_MAX);
+    REQUIRE(hipSuccess == ret);
+  }
+}
+
+/**
+ * Negative Test for API - hipUserObjectRetain
+ 1) Pass User Object as nullptr
+ 2) Pass initialRefcount as 0
+ 3) Pass initialRefcount as INT_MAX
+ */
+
+TEST_CASE("Unit_hipUserObjectRetain_Negative") {
+  hipError_t ret;
+  int *object = new int();
+  REQUIRE(object != nullptr);
+
+  hipUserObject_t hObject;
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyIntObj,
+                                  1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+
+  SECTION("Pass User Object as nullptr") {
+    ret = hipUserObjectRetain(nullptr, 1);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as 0") {
+    ret = hipUserObjectRetain(hObject, 0);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as INT_MAX") {
+    ret = hipUserObjectRetain(hObject, INT_MAX);
+    REQUIRE(hipSuccess == ret);
+  }
+}
+
+TEST_CASE("Unit_hipUserObj_Negative_Test") {
+  hipError_t ret;
+  int *object = new int();
+  REQUIRE(object != nullptr);
+
+  hipUserObject_t hObject;
+  // Create a new hObject with 2 reference
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyIntObj,
+                                  2, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+
+  // Release more than created.
+  ret = hipUserObjectRelease(hObject, 4);
+  REQUIRE(hipSuccess == ret);
+
+  // Retain reference to a removed user object
+  ret = hipUserObjectRetain(hObject, 1);
+  REQUIRE(hipSuccess == ret);
+}
+
+/**
+ * Functional Test for API - hipGraphRetainUserObject
+ */
+
+/* 1) Create GraphUserObject and retain it by calling hipGraphRetainUserObject
+      and release it by calling hipGraphReleaseUserObject. */
+
+static void hipGraphRetainUserObject_Functional_1(void *object,
+                                                  void destroyObj(void *)) {
+  hipGraph_t graph;
+  HIP_CHECK(hipGraphCreate(&graph, 0));
+
+  hipUserObject_t hObject;
+
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyObj,
+                                  1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  HIP_CHECK(hipGraphRetainUserObject(graph, hObject, 1,
+                                       hipGraphUserObjectMove));
+
+  HIP_CHECK(hipGraphReleaseUserObject(graph, hObject));
+  HIP_CHECK(hipUserObjectRelease(hObject));
+  HIP_CHECK(hipGraphDestroy(graph));
+}
+
+TEST_CASE("Unit_hipGraphRetainUserObject_Functional_1") {
+  SECTION("Called with int Object") {
+    int *object = new int();
+    REQUIRE(object != nullptr);
+    hipGraphRetainUserObject_Functional_1(object, destroyIntObj);
+  }
+  SECTION("Called with float Object") {
+    float *object = new float();
+    REQUIRE(object != nullptr);
+    hipGraphRetainUserObject_Functional_1(object, destroyFloatObj);
+  }
+  SECTION("Called with Class Object") {
+    BoxClass *object = new BoxClass();
+    REQUIRE(object != nullptr);
+    hipGraphRetainUserObject_Functional_1(object, destroyClassObj);
+  }
+  SECTION("Called with Struct Object") {
+    BoxStruct *object = new BoxStruct();
+    REQUIRE(object != nullptr);
+    hipGraphRetainUserObject_Functional_1(object, destroyStructObj);
+  }
+}
+
+/* 2) Create UserObject and GraphUserObject and retain using custom reference
+      count and release it by calling hipGraphReleaseUserObject with count. */
+
+TEST_CASE("Unit_hipGraphRetainUserObject_Functional_2") {
+  constexpr size_t N = 1024;
+  constexpr size_t Nbytes = N * sizeof(int);
+  constexpr auto blocksPerCU = 6;  // to hide latency
+  constexpr auto threadsPerBlock = 256;
+  hipGraph_t graph;
+  hipGraphNode_t memcpyNode, kNode;
+  hipKernelNodeParams kNodeParams{};
+  hipStream_t streamForGraph;
+  int *A_d, *B_d, *C_d;
+  int *A_h, *B_h, *C_h;
+  std::vector<hipGraphNode_t> dependencies;
+  hipGraphExec_t graphExec;
+  size_t NElem{N};
+
+  HIP_CHECK(hipStreamCreate(&streamForGraph));
+  HipTest::initArrays(&A_d, &B_d, &C_d, &A_h, &B_h, &C_h, N, false);
+  unsigned blocks = HipTest::setNumBlocks(blocksPerCU, threadsPerBlock, N);
+
+  HIP_CHECK(hipGraphCreate(&graph, 0));
+  HIP_CHECK(hipGraphAddMemcpyNode1D(&memcpyNode, graph, nullptr, 0, A_d, A_h,
+                                   Nbytes, hipMemcpyHostToDevice));
+  dependencies.push_back(memcpyNode);
+  HIP_CHECK(hipGraphAddMemcpyNode1D(&memcpyNode, graph, nullptr, 0, B_d, B_h,
+                                   Nbytes, hipMemcpyHostToDevice));
+  dependencies.push_back(memcpyNode);
+
+  void* kernelArgs[] = {&A_d, &B_d, &C_d, reinterpret_cast<void *>(&NElem)};
+  kNodeParams.func = reinterpret_cast<void *>(HipTest::vectorADD<int>);
+  kNodeParams.gridDim = dim3(blocks);
+  kNodeParams.blockDim = dim3(threadsPerBlock);
+  kNodeParams.sharedMemBytes = 0;
+  kNodeParams.kernelParams = reinterpret_cast<void**>(kernelArgs);
+  kNodeParams.extra = nullptr;
+  HIP_CHECK(hipGraphAddKernelNode(&kNode, graph, dependencies.data(),
+                                  dependencies.size(), &kNodeParams));
+
+  dependencies.clear();
+  dependencies.push_back(kNode);
+  HIP_CHECK(hipGraphAddMemcpyNode1D(&memcpyNode, graph, dependencies.data(),
+                                    dependencies.size(), C_h, C_d,
+                                    Nbytes, hipMemcpyDeviceToHost));
+
+  int refCount = 2;
+  int refCountRetain = 3;
+
+  float *object = new float();
+  REQUIRE(object != nullptr);
+  hipUserObject_t hObject;
+
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyFloatObj,
+                                  refCount, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  HIP_CHECK(hipUserObjectRetain(hObject, refCountRetain));
+  HIP_CHECK(hipGraphRetainUserObject(graph, hObject, refCountRetain,
+                                       hipGraphUserObjectMove));
+
+  // Instantiate and launch the graph
+  HIP_CHECK(hipGraphInstantiate(&graphExec, graph, NULL, NULL, 0));
+  HIP_CHECK(hipGraphLaunch(graphExec, streamForGraph));
+  HIP_CHECK(hipStreamSynchronize(streamForGraph));
+
+  // Verify result
+  HipTest::checkVectorADD<int>(A_h, B_h, C_h, N);
+
+  HIP_CHECK(hipUserObjectRelease(hObject, refCount + refCountRetain));
+  HIP_CHECK(hipGraphReleaseUserObject(graph, hObject, refCountRetain));
+
+  HipTest::freeArrays(A_d, B_d, C_d, A_h, B_h, C_h, false);
+  HIP_CHECK(hipGraphExecDestroy(graphExec));
+  HIP_CHECK(hipGraphDestroy(graph));
+  HIP_CHECK(hipStreamDestroy(streamForGraph));
+}
+
+
+/**
+ * Negative Test for API - hipGraphRetainUserObject
+ 1) Pass graph as nullptr
+ 2) Pass User Object as nullptr
+ 3) Pass initialRefcount as 0
+ 4) Pass initialRefcount as INT_MAX
+ 5) Pass flag as 0
+ 6) Pass flag as INT_MAX
+ */
+
+TEST_CASE("Unit_hipGraphRetainUserObject_Negative") {
+  hipError_t ret;
+  hipGraph_t graph;
+  HIP_CHECK(hipGraphCreate(&graph, 0));
+
+  float *object = new float();
+  REQUIRE(object != nullptr);
+  hipUserObject_t hObject;
+
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyFloatObj,
+                                  1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+
+  SECTION("Pass graph as nullptr") {
+    ret = hipGraphRetainUserObject(nullptr, hObject, 1,
+                                    hipGraphUserObjectMove);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass User Object as nullptr") {
+    ret = hipGraphRetainUserObject(graph, nullptr, 1,
+                                    hipGraphUserObjectMove);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as 0") {
+    ret = hipGraphRetainUserObject(graph, hObject, 0,
+                                    hipGraphUserObjectMove);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as INT_MAX") {
+    ret = hipGraphRetainUserObject(graph, hObject, INT_MAX,
+                                    hipGraphUserObjectMove);
+    REQUIRE(hipSuccess == ret);
+  }
+  SECTION("Pass flag as 0") {
+    ret = hipGraphRetainUserObject(graph, hObject, 1, 0);
+    REQUIRE(hipSuccess == ret);
+  }
+  SECTION("Pass flag as INT_MAX") {
+    ret = hipGraphRetainUserObject(graph, hObject, 1, INT_MAX);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+
+  HIP_CHECK(hipUserObjectRelease(hObject, 1));
+  HIP_CHECK(hipGraphDestroy(graph));
+}
+
+/**
+ * Negative Test for API - hipGraphReleaseUserObject
+ 1) Pass graph as nullptr
+ 2) Pass User Object as nullptr
+ 3) Pass initialRefcount as 0
+ 4) Pass initialRefcount as INT_MAX
+ */
+
+TEST_CASE("Unit_hipGraphReleaseUserObject_Negative") {
+  hipError_t ret;
+  hipGraph_t graph;
+  HIP_CHECK(hipGraphCreate(&graph, 0));
+
+  float *object = new float();
+  REQUIRE(object != nullptr);
+  hipUserObject_t hObject;
+
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                  destroyFloatObj,
+                                  1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  HIP_CHECK(hipGraphRetainUserObject(graph, hObject, 1,
+                                       hipGraphUserObjectMove));
+
+  SECTION("Pass graph as nullptr") {
+    ret = hipGraphReleaseUserObject(nullptr, hObject, 1);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass User Object as nullptr") {
+    ret = hipGraphReleaseUserObject(graph, nullptr, 1);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as 0") {
+    ret = hipGraphReleaseUserObject(graph, hObject, 0);
+    REQUIRE(hipErrorInvalidValue == ret);
+  }
+  SECTION("Pass initialRefcount as INT_MAX") {
+    ret = hipGraphReleaseUserObject(graph, hObject, INT_MAX);
+    REQUIRE(hipSuccess == ret);
+  }
+  HIP_CHECK(hipUserObjectRelease(hObject, 1));
+  HIP_CHECK(hipGraphDestroy(graph));
+}
+
+TEST_CASE("Unit_hipGraphRetainUserObject_Negative_Basic") {
+  hipError_t ret;
+  hipGraph_t graph;
+  HIP_CHECK(hipGraphCreate(&graph, 0));
+
+  float *object = new float();
+  REQUIRE(object != nullptr);
+  hipUserObject_t hObject;
+
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                destroyFloatObj,
+                                1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  // Retain graph object with reference count 2
+  HIP_CHECK(hipGraphRetainUserObject(graph, hObject, 2,
+                                     hipGraphUserObjectMove));
+
+  // Release graph object with reference count more than 2
+  ret = hipGraphReleaseUserObject(graph, hObject, 4);
+  REQUIRE(hipSuccess == ret);
+
+  // Again Retain graph object with reference count 8
+  ret = hipGraphRetainUserObject(graph, hObject, 8,
+                                 hipGraphUserObjectMove);
+  REQUIRE(hipSuccess == ret);
+  // Release graph object with reference count 1
+  ret = hipGraphReleaseUserObject(graph, hObject, 1);
+  REQUIRE(hipSuccess == ret);
+
+  HIP_CHECK(hipUserObjectRelease(hObject, 1));
+  HIP_CHECK(hipGraphDestroy(graph));
+}
+
+TEST_CASE("Unit_hipGraphRetainUserObject_Negative_Null_Object") {
+  hipError_t ret;
+  hipGraph_t graph;
+  HIP_CHECK(hipGraphCreate(&graph, 0));
+
+  float *object = nullptr;  // this is used for Null_Object test
+  hipUserObject_t hObject;
+
+  HIP_CHECK(hipUserObjectCreate(&hObject, object,
+                                destroyFloatObj,
+                                1, hipUserObjectNoDestructorSync));
+  REQUIRE(hObject != nullptr);
+  // Retain graph object with reference count 2
+  HIP_CHECK(hipGraphRetainUserObject(graph, hObject, 2,
+                                     hipGraphUserObjectMove));
+
+  // Release graph object with reference count more than 2
+  ret = hipGraphReleaseUserObject(graph, hObject, 4);
+  REQUIRE(hipSuccess == ret);
+
+  // Again Retain graph object with reference count 8
+  ret = hipGraphRetainUserObject(graph, hObject, 8, 0);
+  REQUIRE(hipSuccess == ret);
+  // Release graph object with reference count 1
+  ret = hipGraphReleaseUserObject(graph, hObject, 1);
+  REQUIRE(hipSuccess == ret);
+
+  HIP_CHECK(hipUserObjectRelease(hObject, 1));
+  HIP_CHECK(hipGraphDestroy(graph));
+}
+

tests/catch/unit/memory/hipMallocManaged.cc

@@ -20,8 +20,10 @@
 /* Test Case Description:
    1) This testcase verifies the hipMallocManaged basic scenario - supported on
      all devices
-   2) This testcase verifies the hipMallocManaged basic scenario - supported
-       only on HMM enabled devices
+   2) This testcase verifies the hipMallocManaged advanced scenario - supported
+     only on HMM enabled devices
+   3) This testcase verifies that hipMallocManaged returns an OutOfMemory error
+     for allocations much larger than the available memory - supported on all devices
 */
 
 #include "hipMallocManagedCommon.hh"
@@ -138,3 +140,19 @@ TEST_CASE("Unit_hipMallocManaged_Advanced") {
   HIP_CHECK(hipFree(B));
   REQUIRE(maxError != 0.0f);
 }
+
+/*
+   This testcase verifies that hipMallocManaged returns an OutOfMemory error for allocations much
+   larger than the available memory - supported on all devices
+ */
+TEST_CASE("Unit_hipMallocManaged_Large") {
+  auto managed = HmmAttrPrint();
+  if (managed != 1) {
+    WARN(
+        "GPU doesn't support hipDeviceAttributeManagedMemory attribute so defaulting to system "
+        "memory.");
+  }
+
+  float* A;
+  HIP_CHECK_ERROR(hipMallocManaged(&A, std::numeric_limits<size_t>::max()), hipErrorOutOfMemory);
+}

tests/catch/unit/stream/CMakeLists.txt

@@ -18,6 +18,7 @@ set(TEST_SRC
     hipStreamQuery.cc
     hipStreamWaitEvent.cc
     hipDeviceGetStreamPriorityRange.cc
+    hipStreamACb_StrmSyncTiming.cc
     )
 else()
 set(TEST_SRC
@@ -38,6 +39,7 @@ set(TEST_SRC
     hipStreamSynchronize.cc
     hipStreamQuery.cc
     hipDeviceGetStreamPriorityRange.cc
+    hipStreamACb_StrmSyncTiming.cc
     )
 
     # set_source_files_properties(hipStreamAttachMemAsync.cc PROPERTIES COMPILE_FLAGS -std=c++17)

tests/catch/unit/stream/hipStreamACb_StrmSyncTiming.cc

@@ -0,0 +1,138 @@
+/*
+Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR
+IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+Testcase Scenario :
+Validate behaviour of HIP when multiple hipStreaAddCallback() are called over
+multiple Threads.
+*/
+
+#include <stdio.h>
+#include <atomic>
+#include <chrono>
+#include <hip_test_common.hh>
+#include <hip_test_kernels.hh>
+
+#ifdef __HIP_PLATFORM_AMD__
+#define HIPRT_CB
+#endif
+
+#define SECONDS_TO_WAIT 2
+#define TO_MICROSECONDS 1000000
+
+hipStream_t mystream;
+size_t N_elmts = 4096;
+bool cbDone = false;
+std::atomic<int> Data_mismatch{0};
+
+__global__ void vector_square(float* C_d, float* A_d, size_t N_elmts) {
+  size_t offset = (blockIdx.x * blockDim.x + threadIdx.x);
+  size_t stride = blockDim.x * gridDim.x;
+
+  for (size_t i = offset; i < N_elmts; i += stride) {
+    C_d[i] = A_d[i] * A_d[i];
+  }
+
+  // Delay the thread 1
+  if (offset == 1) {
+    unsigned long long int wait_t = 3200000000, start = clock64(), cur;
+    do {
+      cur = clock64() - start;
+    } while (cur < wait_t);
+  }
+}
+
+float *A_h, *C_h;
+
+static void HIPRT_CB Callback1(hipStream_t stream, hipError_t status, void* userData) {
+  (void)stream;
+  (void)status;
+  (void)userData;
+  // Validate the data
+  for (size_t i = 0; i < N_elmts; i++) {
+    if (C_h[i] != A_h[i] * A_h[i]) {
+      Data_mismatch++;
+    }
+  }
+
+  // Delay the callback completion
+  std::this_thread::sleep_for(std::chrono::seconds(SECONDS_TO_WAIT));
+  cbDone = true;
+}
+
+/**
+ Test multiple hipStreamAddCallback() called over
+ multiple Threads.
+ */
+TEST_CASE("Unit_hipStreamAddCallback_StrmSyncTiming") {
+  float *A_d, *C_d;
+  size_t Nbytes = N_elmts * sizeof(float);
+
+  A_h = (float*)malloc(Nbytes);
+  HIPCHECK(A_h == 0 ? hipErrorOutOfMemory : hipSuccess);
+  C_h = (float*)malloc(Nbytes);
+  HIPCHECK(C_h == 0 ? hipErrorOutOfMemory : hipSuccess);
+
+  // Fill with Phi + i
+  for (size_t i = 0; i < N_elmts; i++) {
+    A_h[i] = 1.618f + i;
+  }
+
+  HIPCHECK(hipMalloc(&A_d, Nbytes));
+  HIPCHECK(hipMalloc(&C_d, Nbytes));
+
+  HIPCHECK(hipStreamCreateWithFlags(&mystream, hipStreamNonBlocking));
+
+  HIPCHECK(hipMemcpyAsync(A_d, A_h, Nbytes, hipMemcpyHostToDevice, mystream));
+
+  const unsigned threadsPerBlock = 256;
+  const unsigned blocks = (N_elmts + 255) / threadsPerBlock;
+
+  hipLaunchKernelGGL((vector_square), dim3(blocks), dim3(threadsPerBlock), 0, mystream, C_d, A_d,
+                     N_elmts);
+  HIPCHECK(hipMemcpyAsync(C_h, C_d, Nbytes, hipMemcpyDeviceToHost, mystream));
+  HIPCHECK(hipStreamAddCallback(mystream, Callback1, NULL, 0));
+
+  // Wait untill Callback() function changes the cbDone value to true
+  while (!cbDone) {
+    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  }
+
+  // Since the callback is supposed to be called only after an implicit stream
+  // synchronization, and the runtime cannot continue until the callback is done
+  // hipStreamSynchronize call should not take much time.
+  auto start = std::chrono::high_resolution_clock::now();
+  HIPCHECK(hipStreamSynchronize(mystream));
+  auto stop = std::chrono::high_resolution_clock::now();
+  auto duration = std::chrono::duration_cast<std::chrono::microseconds>(stop - start);
+
+  HIPCHECK(hipStreamDestroy(mystream));
+  HIPCHECK(hipFree(A_d));
+  HIPCHECK(hipFree(C_d));
+  free(A_h);
+  free(C_h);
+
+  REQUIRE(Data_mismatch.load() == 0);
+  // HIP runtime cannot proceed further in the queue until callback completes
+  // Stream synchronize should not have much task to do after callback
+  // It should just be an extra empty marker wait
+  // Therefore the hipStreamSynchronize() in the
+  // main thread should hardly take any time to complete.
+  REQUIRE(duration.count() < 100);
+}

tests/catch/unit/stream/hipStreamGetCUMask.cc

@@ -37,7 +37,7 @@ TEST_CASE("Unit_hipExtStreamGetCUMask_verifyDefaultAndCustomMask") {
   std::vector<uint32_t> cuMask(maxNum);
   hipDeviceProp_t props;
   std::stringstream ss;
-  char* gCUMask{nullptr};
+  std::string gCUMask;
   std::string globalCUMask("");
   std::vector<uint32_t> defaultCUMask;
 
@@ -54,8 +54,8 @@ TEST_CASE("Unit_hipExtStreamGetCUMask_verifyDefaultAndCustomMask") {
          props.name << " with " << props.multiProcessorCount << " CUs");
 
   // Get global CU Mask if exists
-  gCUMask = getenv("ROC_GLOBAL_CU_MASK");
-  if (gCUMask != nullptr && gCUMask[0] != '\0') {
+  gCUMask = TestContext::getEnvVar("ROC_GLOBAL_CU_MASK");
+  if (!gCUMask.empty()) {
     globalCUMask.assign(gCUMask);
 
     for_each(globalCUMask.begin(), globalCUMask.end(), [](char & c) {
@@ -155,7 +155,7 @@ TEST_CASE("Unit_hipExtStreamGetCUMask_verifyDefaultAndCustomMask") {
     INFO("info: reading back CU mask 0x" << ss.str() <<
                                                 " for stream " << stream);
 
-    if (!gCUMask) {
+    if (!gCUMask.empty()) {
       for (size_t i = 0; i < customMask.size(); i++) {
         if (customMask[i] != cuMask[i]) {
           INFO("Error! expected CU mask:" << customMask[i]

tests/catch/unit/streamperthread/hipStreamPerThrdTsts.cc

@@ -47,9 +47,7 @@ THE SOFTWARE.
 #endif
 
 #include <hip_test_common.hh>
-#ifdef HT_AMD
-  #include "hip/hip_cooperative_groups.h"
-#endif
+#include "hip/hip_cooperative_groups.h"
 using namespace std::chrono;
 using namespace cooperative_groups;
 #if HT_AMD
@@ -69,8 +67,12 @@ __global__ void StreamPerThrd(int *Ad, int *Ad1, size_t n, int Pk_Clk,
     int64_t GpuFrq = (Pk_Clk * 1000);
     int64_t StrtTck = clock64();
     if (index == 0) {
-      // The following while loop checks the value in ptr for around 4 seconds
-      while ((clock64() - StrtTck) <= (6 * GpuFrq)) {
+      // The following while loop holds the execution for ~2 seconds.
+      // Busy sleep on nvidia
+      while ((clock64() - StrtTck) <= (2 * GpuFrq)) {
+        #if HT_AMD
+          __builtin_amdgcn_s_sleep(10);
+        #endif
       }
       if (WaitEvnt == 1) {
         *Ad1 = 1;
@@ -83,8 +85,12 @@ __global__ void StreamPerThrd(int *Ad, int *Ad1, size_t n, int Pk_Clk,
 __global__ void StreamPerThrd1(int *A, int Pk_Clk) {
   int64_t GpuFrq = (Pk_Clk * 1000);
   int64_t StrtTck = clock64();
-  // The following while loop checks the value in ptr for around 3-4 seconds
-  while ((clock64() - StrtTck) <= (3 * GpuFrq)) {
+  // The following while loop holds the execution for ~1 second
+  // Busy sleep on nvidia
+  while ((clock64() - StrtTck) <= (GpuFrq)) {
+    #if HT_AMD
+      __builtin_amdgcn_s_sleep(10);
+    #endif
   }
   *A = 1;
 }