SWDEV-1 - Merge github PRs to amd-staging

Change-Id: I2944a63ddc2eec8dc1403d9790ffffbaec343385

catch/unit/module/CMakeLists.txt

@@ -29,31 +29,34 @@ set(TEST_SRC
     hipModuleLaunchKernel.cc
     hipModuleGetGlobal.cc
     hipModuleGetTexRef.cc
+    hipModuleLaunchCooperativeKernel.cc
+    hipModuleLaunchCooperativeKernelMultiDevice.cc
+    hipFuncGetAttribute.cc
 )
 
 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/get_function_module.code
-                   COMMAND ${CMAKE_CXX_COMPILER} --genco --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/get_function_module.cc
+                   COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/get_function_module.cc
                    -o get_function_module.code
                    -I${HIP_PATH}/include/ --rocm-path=${ROCM_PATH}
                    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/get_function_module.cc)
 add_custom_target(get_function_module ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/get_function_module.code)
 
 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/launch_kernel_module.code
-                   COMMAND ${CMAKE_CXX_COMPILER} --genco --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/launch_kernel_module.cc
+                   COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/launch_kernel_module.cc
                    -o launch_kernel_module.code
                    -I${HIP_PATH}/include/ --rocm-path=${ROCM_PATH}
                    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/launch_kernel_module.cc)
 add_custom_target(launch_kernel_module ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/launch_kernel_module.code)
 
 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/get_global_test_module.code
-                   COMMAND ${CMAKE_CXX_COMPILER} --genco --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/get_global_test_module.cc
+                   COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/get_global_test_module.cc
                    -o get_global_test_module.code
                    -I${HIP_PATH}/include/ --rocm-path=${ROCM_PATH}
                    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/get_global_test_module.cc)
 add_custom_target(get_global_test_module ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/get_global_test_module.code)
 
 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/get_tex_ref_module.code
-                   COMMAND ${CMAKE_CXX_COMPILER} --genco --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/get_tex_ref_module.cc
+                   COMMAND ${CMAKE_CXX_COMPILER} --genco ${OFFLOAD_ARCH_STR} --std=c++17 ${CMAKE_CURRENT_SOURCE_DIR}/get_tex_ref_module.cc
                    -o get_tex_ref_module.code
                    -I${HIP_PATH}/include/ --rocm-path=${ROCM_PATH}
                    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/get_tex_ref_module.cc)

catch/unit/module/hipExtModuleLaunchKernel.cc

@@ -44,7 +44,7 @@ THE SOFTWARE.
  */
 
 #include <hip_test_common.hh>
-#include <hip_test_defgroups.hh>
+ 
 #include <iostream>
 #include <fstream>
 #include "hip/hip_ext.h"

catch/unit/module/hipFuncGetAttribute.cc

@@ -0,0 +1,96 @@
+/*
+Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "hip_module_common.hh"
+
+#include <hip_test_common.hh>
+#include <hip/hip_runtime_api.h>
+#include <utils.hh>
+
+static hipModule_t GetModule() {
+  HIP_CHECK(hipFree(nullptr));
+  static const auto mg = ModuleGuard::LoadModule("get_function_module.code");
+  return mg.module();
+}
+
+TEST_CASE("Unit_hipFuncGetAttribute_Positive_Basic") {
+  hipFunction_t kernel = GetKernel(GetModule(), "GlobalKernel");
+
+  int value;
+
+  SECTION("binaryVersion") {
+    HIP_CHECK(hipFuncGetAttribute(&value, HIP_FUNC_ATTRIBUTE_BINARY_VERSION, kernel));
+#if HT_NVIDIA
+    const auto major = GetDeviceAttribute(hipDeviceAttributeComputeCapabilityMajor, 0);
+    const auto minor = GetDeviceAttribute(hipDeviceAttributeComputeCapabilityMinor, 0);
+    REQUIRE(value == major * 10 + minor);
+#elif HT_AMD
+    REQUIRE(value > 0);
+#endif
+  }
+
+  SECTION("cacheModeCA") {
+    HIP_CHECK(hipFuncGetAttribute(&value, HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA, kernel));
+    REQUIRE((value == 0 || value == 1));
+  }
+
+  SECTION("maxThreadsPerBlock") {
+    HIP_CHECK(hipFuncGetAttribute(&value, HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, kernel));
+    REQUIRE(value == GetDeviceAttribute(hipDeviceAttributeMaxThreadsPerBlock, 0));
+  }
+
+  SECTION("numRegs") {
+    HIP_CHECK(hipFuncGetAttribute(&value, HIP_FUNC_ATTRIBUTE_NUM_REGS, kernel));
+    REQUIRE(value >= 0);
+  }
+
+  SECTION("ptxVersion") {
+    HIP_CHECK(hipFuncGetAttribute(&value, HIP_FUNC_ATTRIBUTE_PTX_VERSION, kernel));
+    REQUIRE(value > 0);
+  }
+
+  SECTION("sharedSizeBytes") {
+    HIP_CHECK(hipFuncGetAttribute(&value, HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, kernel));
+    REQUIRE(value <= GetDeviceAttribute(hipDeviceAttributeMaxSharedMemoryPerBlock, 0));
+  }
+}
+
+TEST_CASE("Unit_hipFuncGetAttribute_Negative_Parameters") {
+  hipFunction_t kernel = GetKernel(GetModule(), "GlobalKernel");
+
+  int value;
+
+  SECTION("value == nullptr") {
+    HIP_CHECK_ERROR(hipFuncGetAttribute(nullptr, HIP_FUNC_ATTRIBUTE_BINARY_VERSION, kernel),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("invalid attribute") {
+    HIP_CHECK_ERROR(hipFuncGetAttribute(&value, static_cast<hipFunction_attribute>(-1), kernel),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("hfunc == nullptr") {
+    HIP_CHECK_ERROR(hipFuncGetAttribute(&value, HIP_FUNC_ATTRIBUTE_BINARY_VERSION, nullptr),
+                    hipErrorInvalidResourceHandle);
+  }
+}

catch/unit/module/hipModuleLaunchCooperativeKernel.cc

@@ -0,0 +1,211 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * @addtogroup hipModuleLaunchCooperativeKernel hipModuleLaunchCooperativeKernel
+ * @{
+ * @ingroup ModuleTest
+ * `hipModuleLaunchCooperativeKernel(hipFunction_t f, unsigned int gridDimX, unsigned int gridDimY,
+ * unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ,
+ * unsigned int sharedMemBytes, hipStream_t stream, void ** kernelParams)` -
+ * Launches kernel f with launch parameters and shared memory on stream with arguments passed to
+ * kernelParams, where thread blocks can cooperate and synchronize as they execute.
+ */
+
+#include <hip_test_common.hh>
+#include <resource_guards.hh>
+#include <utils.hh>
+
+#include "hip_module_launch_kernel_common.hh"
+
+/**
+ * Test Description
+ * ------------------------
+ *  - Tests `hipModuleLaunchCooperativeKernel` for a cooperative kernel with no parameters, and for
+ * a normal kernel with parameters.
+ * Test source
+ * ------------------------
+ *  - unit/module/hipModuleLaunchCooperativeKernel.cc
+ * Test requirements
+ * ------------------------
+ *  - HIP_VERSION >= 5.5
+ */
+TEST_CASE("Unit_hipModuleLaunchCooperativeKernel_Positive_Basic") {
+  if (!DeviceAttributesSupport(0, hipDeviceAttributeCooperativeLaunch)) {
+    HipTest::HIP_SKIP_TEST("CooperativeLaunch not supported");
+    return;
+  }
+
+  SECTION("Cooperative kernel with no arguments") {
+    hipFunction_t f = GetKernel(mg.module(), "CoopKernel");
+    HIP_CHECK(hipModuleLaunchCooperativeKernel(f, 2, 2, 1, 1, 1, 1, 0, nullptr, nullptr));
+    HIP_CHECK(hipDeviceSynchronize());
+  }
+
+  SECTION("Kernel with arguments using kernelParams") {
+    hipFunction_t f = GetKernel(mg.module(), "Kernel42");
+
+    LinearAllocGuard<int> result_dev(LinearAllocs::hipMalloc, sizeof(int));
+    HIP_CHECK(hipMemset(result_dev.ptr(), 0, sizeof(*result_dev.ptr())));
+
+    int* result_ptr = result_dev.ptr();
+    void* kernel_args[1] = {&result_ptr};
+    HIP_CHECK(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 1, 1, 1, 0, nullptr, kernel_args));
+
+    int result = 0;
+    HIP_CHECK(hipMemcpy(&result, result_dev.ptr(), sizeof(result), hipMemcpyDefault));
+    REQUIRE(result == 42);
+  }
+}
+
+/**
+ * Test Description
+ * ------------------------
+ *  - Positive parameters test for `hipModuleLaunchCooperativeKernel`.
+ * Test source
+ * ------------------------
+ *  - unit/module/hipModuleLaunchCooperativeKernel.cc
+ * Test requirements
+ * ------------------------
+ *  - HIP_VERSION >= 5.5
+ */
+TEST_CASE("Unit_hipModuleLaunchCooperativeKernel_Positive_Parameters") {
+  if (!DeviceAttributesSupport(0, hipDeviceAttributeCooperativeLaunch)) {
+    HipTest::HIP_SKIP_TEST("CooperativeLaunch not supported");
+    return;
+  }
+
+  hipFunction_t f = GetKernel(mg.module(), "NOPKernel");
+
+  SECTION("blockDim.x == maxBlockDimX") {
+    const unsigned int x = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimX, 0);
+    HIP_CHECK(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, x, 1, 1, 0, nullptr, nullptr));
+  }
+
+  SECTION("blockDim.y == maxBlockDimY") {
+    const unsigned int y = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimY, 0);
+    HIP_CHECK(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, y, 1, 1, 0, nullptr, nullptr));
+  }
+
+  SECTION("blockDim.z == maxBlockDimZ") {
+    const unsigned int z = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimZ, 0);
+    HIP_CHECK(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, z, 1, 1, 0, nullptr, nullptr));
+  }
+}
+
+/**
+ * Test Description
+ * ------------------------
+ *  - Negative parameters test for `hipModuleLaunchCooperativeKernel`.
+ * Test source
+ * ------------------------
+ *  - unit/module/hipModuleLaunchCooperativeKernel.cc
+ * Test requirements
+ * ------------------------
+ *  - HIP_VERSION >= 5.5
+ */
+TEST_CASE("Unit_hipModuleLaunchCooperativeKernel_Negative_Parameters") {
+  if (!DeviceAttributesSupport(0, hipDeviceAttributeCooperativeLaunch)) {
+    HipTest::HIP_SKIP_TEST("CooperativeLaunch not supported");
+    return;
+  }
+
+  hipFunction_t f = GetKernel(mg.module(), "NOPKernel");
+
+  SECTION("f == nullptr") {
+    HIP_CHECK_ERROR(
+        hipModuleLaunchCooperativeKernel(nullptr, 1, 1, 1, 1, 1, 1, 0, nullptr, nullptr),
+        hipErrorInvalidResourceHandle);
+  }
+
+  SECTION("gridDim.x == 0") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 0, 1, 1, 1, 1, 1, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("gridDim.y == 0") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 0, 1, 1, 1, 1, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("gridDim.z == 0") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 0, 1, 1, 1, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("blockDim.x == 0") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 0, 1, 1, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("blockDim.y == 0") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 1, 0, 1, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("blockDim.z == 0") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 1, 1, 0, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("blockDim.x > maxBlockDimX") {
+    const unsigned int x = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimX, 0) + 1u;
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, x, 1, 1, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("blockDim.y > maxBlockDimY") {
+    const unsigned int y = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimY, 0) + 1u;
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 1, y, 1, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("blockDim.z > maxBlockDimZ") {
+    const unsigned int z = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimZ, 0) + 1u;
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 1, 1, z, 0, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("blockDim.x * blockDim.y * blockDim.z > maxThreadsPerBlock") {
+    const unsigned int max = GetDeviceAttribute(hipDeviceAttributeMaxThreadsPerBlock, 0);
+    const unsigned int dim = std::ceil(std::cbrt(max));
+    HIP_CHECK_ERROR(
+        hipModuleLaunchCooperativeKernel(f, 1, 1, 1, dim, dim, dim, 0, nullptr, nullptr),
+        hipErrorInvalidValue);
+  }
+
+#if HT_AMD  // Disabled due to defect EXSWHTEC-351
+  SECTION("sharedMemBytes > maxSharedMemoryPerBlock") {
+    const unsigned int max = GetDeviceAttribute(hipDeviceAttributeMaxSharedMemoryPerBlock, 0) + 1u;
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 1, 1, 1, max, nullptr, nullptr),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("Invalid stream") {
+    hipStream_t stream = nullptr;
+    HIP_CHECK(hipStreamCreate(&stream));
+    HIP_CHECK(hipStreamDestroy(stream));
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernel(f, 1, 1, 1, 1, 1, 1, 0, stream, nullptr),
+                    hipErrorInvalidValue);
+  }
+#endif
+}

catch/unit/module/hipModuleLaunchCooperativeKernelMultiDevice.cc

@@ -0,0 +1,227 @@
+/*
+Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * @addtogroup hipModuleLaunchCooperativeKernelMultiDevice
+ * hipModuleLaunchCooperativeKernelMultiDevice
+ * @{
+ * @ingroup ModuleTest
+ * `hipModuleLaunchCooperativeKernelMultiDevice(hipFunctionLaunchParams* launchParamsList, unsigned
+ * int numDevices, unsigned int flags)` -
+ * Launches kernels on multiple devices where thread blocks can cooperate and synchronize as they
+ * execute.
+ */
+
+#include <hip_test_common.hh>
+#include <resource_guards.hh>
+#include <utils.hh>
+
+#include "hip_module_launch_kernel_common.hh"
+
+/**
+ * Test Description
+ * ------------------------
+ *  - Tests `hipModuleLaunchCooperativeKernel` for a cooperative kernel with no parameters.
+ * Test source
+ * ------------------------
+ *  - unit/module/hipModuleLaunchCooperativeKernelMultiDevice.cc
+ * Test requirements
+ * ------------------------
+ *  - HIP_VERSION >= 5.5
+ */
+TEST_CASE("Unit_hipModuleLaunchCooperativeKernelMultiDevice_Positive_Basic") {
+  if (!DeviceAttributesSupport(0, hipDeviceAttributeCooperativeLaunch)) {
+    HipTest::HIP_SKIP_TEST("CooperativeLaunch not supported");
+    return;
+  }
+
+  hipFunction_t f = GetKernel(mg.module(), "CoopKernel");
+
+  const auto device_count = HipTest::getDeviceCount();
+
+  std::vector<hipFunctionLaunchParams> params_list(device_count);
+
+  int device = 0;
+  for (auto& params : params_list) {
+    params.function = f;
+    params.gridDimX = 1;
+    params.gridDimY = 1;
+    params.gridDimZ = 1;
+    params.blockDimX = 1;
+    params.blockDimY = 1;
+    params.blockDimZ = 1;
+    params.kernelParams = nullptr;
+    params.sharedMemBytes = 0;
+    HIP_CHECK(hipSetDevice(device++));
+    HIP_CHECK(hipStreamCreate(&params.hStream));
+  }
+
+  HIP_CHECK(hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), device_count, 0u));
+
+  for (const auto params : params_list) {
+    HIP_CHECK(hipStreamSynchronize(params.hStream));
+  }
+
+  for (const auto params : params_list) {
+    HIP_CHECK(hipStreamDestroy(params.hStream));
+  }
+}
+
+/**
+ * Test Description
+ * ------------------------
+ *  - Negative parameters test for `hipModuleLaunchCooperativeKernelMultiDevice`.
+ * Test source
+ * ------------------------
+ *  - unit/module/hipModuleLaunchCooperativeKernelMultiDevice.cc
+ * Test requirements
+ * ------------------------
+ *  - HIP_VERSION >= 5.5
+ */
+TEST_CASE("Unit_hipModuleLaunchCooperativeKernelMultiDevice_Negative_Parameters") {
+  if (!DeviceAttributesSupport(0, hipDeviceAttributeCooperativeLaunch)) {
+    HipTest::HIP_SKIP_TEST("CooperativeLaunch not supported");
+    return;
+  }
+
+  hipFunction_t f = GetKernel(mg.module(), "CoopKernel");
+
+  const auto device_count = HipTest::getDeviceCount();
+
+  std::vector<hipFunctionLaunchParams> params_list(device_count);
+
+  int device = 0;
+  for (auto& params : params_list) {
+    params.function = f;
+    params.gridDimX = 1;
+    params.gridDimY = 1;
+    params.gridDimZ = 1;
+    params.blockDimX = 1;
+    params.blockDimY = 1;
+    params.blockDimZ = 1;
+    params.kernelParams = nullptr;
+    params.sharedMemBytes = 0;
+    HIP_CHECK(hipSetDevice(device++));
+    HIP_CHECK(hipStreamCreate(&params.hStream));
+  }
+
+  SECTION("launchParamsList == nullptr") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernelMultiDevice(nullptr, device_count, 0u),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("numDevices == 0") {
+    HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), 0, 0u),
+                    hipErrorInvalidValue);
+  }
+
+  SECTION("numDevices > device count") {
+    HIP_CHECK_ERROR(
+        hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), device_count + 1, 0u),
+        hipErrorInvalidValue);
+  }
+
+  SECTION("invalid flags") {
+    HIP_CHECK_ERROR(
+        hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), device_count, 999),
+        hipErrorInvalidValue);
+  }
+
+  if (device_count > 1) {
+    SECTION("launchParamsList.func doesn't match across all devices") {
+      params_list[1].function = GetKernel(mg.module(), "NOPKernel");
+      HIP_CHECK_ERROR(
+          hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), device_count, 0u),
+          hipErrorInvalidValue);
+    }
+
+    SECTION("launchParamsList.gridDim doesn't match across all kernels") {
+      params_list[1].gridDimX = 2;
+      HIP_CHECK_ERROR(
+          hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), device_count, 0u),
+          hipErrorInvalidValue);
+    }
+
+    SECTION("launchParamsList.blockDim doesn't match across all kernels") {
+      params_list[1].blockDimX = 2;
+      HIP_CHECK_ERROR(
+          hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), device_count, 0u),
+          hipErrorInvalidValue);
+    }
+
+    SECTION("launchParamsList.sharedMem doesn't match across all kernels") {
+      params_list[1].sharedMemBytes = 1024;
+      HIP_CHECK_ERROR(
+          hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), device_count, 0u),
+          hipErrorInvalidValue);
+    }
+  }
+
+  for (const auto params : params_list) {
+    HIP_CHECK(hipStreamDestroy(params.hStream));
+  }
+}
+
+/**
+ * Test Description
+ * ------------------------
+ *  - Tries running `hipModuleLaunchCooperativeKernelMultiDevice` with multiple kernels on the same
+ * device.
+ * Test source
+ * ------------------------
+ *  - unit/module/hipModuleLaunchCooperativeKernelMultiDevice.cc
+ * Test requirements
+ * ------------------------
+ *  - HIP_VERSION >= 5.5
+ */
+TEST_CASE("Unit_hipModuleLaunchCooperativeKernelMultiDevice_Negative_MultiKernelSameDevice") {
+  if (!DeviceAttributesSupport(0, hipDeviceAttributeCooperativeLaunch)) {
+    HipTest::HIP_SKIP_TEST("CooperativeLaunch not supported");
+    return;
+  }
+
+  hipFunction_t f = GetKernel(mg.module(), "CoopKernel");
+
+  HIP_CHECK(hipSetDevice(0));
+
+  std::vector<hipFunctionLaunchParams> params_list(2);
+
+  for (auto& params : params_list) {
+    params.function = f;
+    params.gridDimX = 1;
+    params.gridDimY = 1;
+    params.gridDimZ = 1;
+    params.blockDimX = 1;
+    params.blockDimY = 1;
+    params.blockDimZ = 1;
+    params.kernelParams = nullptr;
+    params.sharedMemBytes = 0;
+    HIP_CHECK(hipStreamCreate(&params.hStream));
+  }
+
+  HIP_CHECK_ERROR(hipModuleLaunchCooperativeKernelMultiDevice(params_list.data(), 2, 0u),
+                  hipErrorInvalidValue);
+
+  for (const auto params : params_list) {
+    HIP_CHECK(hipStreamDestroy(params.hStream));
+  }
+}

catch/unit/module/hip_module_launch_kernel_common.hh

@@ -88,32 +88,32 @@ template <ExtModuleLaunchKernelSig* func> void ModuleLaunchKernelPositiveParamet
   };
 
   SECTION("gridDimX == maxGridDimX") {
-    const unsigned int x = GetDeviceAttribute(0, hipDeviceAttributeMaxGridDimX);
+    const unsigned int x = GetDeviceAttribute(hipDeviceAttributeMaxGridDimX, 0);
     LaunchNOPKernel(x, 1, 1, 1, 1, 1);
   }
 
   SECTION("gridDimY == maxGridDimY") {
-    const unsigned int y = GetDeviceAttribute(0, hipDeviceAttributeMaxGridDimY);
+    const unsigned int y = GetDeviceAttribute(hipDeviceAttributeMaxGridDimY, 0);
     LaunchNOPKernel(1, y, 1, 1, 1, 1);
   }
 
   SECTION("gridDimZ == maxGridDimZ") {
-    const unsigned int z = GetDeviceAttribute(0, hipDeviceAttributeMaxGridDimZ);
+    const unsigned int z = GetDeviceAttribute(hipDeviceAttributeMaxGridDimZ, 0);
     LaunchNOPKernel(1, 1, z, 1, 1, 1);
   }
 
   SECTION("blockDimX == maxBlockDimX") {
-    const unsigned int x = GetDeviceAttribute(0, hipDeviceAttributeMaxBlockDimX);
+    const unsigned int x = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimX, 0);
     LaunchNOPKernel(1, 1, 1, x, 1, 1);
   }
 
   SECTION("blockDimY == maxBlockDimY") {
-    const unsigned int y = GetDeviceAttribute(0, hipDeviceAttributeMaxBlockDimY);
+    const unsigned int y = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimY, 0);
     LaunchNOPKernel(1, 1, 1, 1, y, 1);
   }
 
   SECTION("blockDimZ == maxBlockDimZ") {
-    const unsigned int z = GetDeviceAttribute(0, hipDeviceAttributeMaxBlockDimZ);
+    const unsigned int z = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimZ, 0);
     LaunchNOPKernel(1, 1, 1, 1, 1, z);
   }
 }
@@ -163,19 +163,19 @@ template <ExtModuleLaunchKernelSig* func> void ModuleLaunchKernelNegativeParamet
 // Disabled on AMD due to defect - EXSWHTEC-158
 #if HT_NVIDIA
   SECTION("gridDimX > maxGridDimX") {
-    const unsigned int x = GetDeviceAttribute(0, hipDeviceAttributeMaxGridDimX) + 1u;
+    const unsigned int x = GetDeviceAttribute(hipDeviceAttributeMaxGridDimX, 0) + 1u;
     HIP_CHECK_ERROR(func(f, x, 1, 1, 1, 1, 1, 0, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
                     hipErrorInvalidValue);
   }
 
   SECTION("gridDimY > maxGridDimY") {
-    const unsigned int y = GetDeviceAttribute(0, hipDeviceAttributeMaxGridDimY) + 1u;
+    const unsigned int y = GetDeviceAttribute(hipDeviceAttributeMaxGridDimY, 0) + 1u;
     HIP_CHECK_ERROR(func(f, 1, y, 1, 1, 1, 1, 0, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
                     hipErrorInvalidValue);
   }
 
   SECTION("gridDimZ > maxGridDimZ") {
-    const unsigned int z = GetDeviceAttribute(0, hipDeviceAttributeMaxGridDimZ) + 1u;
+    const unsigned int z = GetDeviceAttribute(hipDeviceAttributeMaxGridDimZ, 0) + 1u;
     HIP_CHECK_ERROR(func(f, 1, 1, z, 1, 1, 1, 0, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
                     hipErrorInvalidValue);
   }
@@ -184,19 +184,19 @@ template <ExtModuleLaunchKernelSig* func> void ModuleLaunchKernelNegativeParamet
 // Disabled on AMD due to defect - EXSWHTEC-156
 #if HT_NVIDIA
   SECTION("blockDimX > maxBlockDimX") {
-    const unsigned int x = GetDeviceAttribute(0, hipDeviceAttributeMaxBlockDimX) + 1u;
+    const unsigned int x = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimX, 0) + 1u;
     HIP_CHECK_ERROR(func(f, 1, 1, 1, x, 1, 1, 0, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
                     hipErrorInvalidValue);
   }
 
   SECTION("blockDimY > maxBlockDimY") {
-    const unsigned int y = GetDeviceAttribute(0, hipDeviceAttributeMaxBlockDimY) + 1u;
+    const unsigned int y = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimY, 0) + 1u;
     HIP_CHECK_ERROR(func(f, 1, 1, 1, 1, y, 1, 0, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
                     hipErrorInvalidValue);
   }
 
   SECTION("blockDimZ > maxBlockDimZ") {
-    const unsigned int z = GetDeviceAttribute(0, hipDeviceAttributeMaxBlockDimZ) + 1u;
+    const unsigned int z = GetDeviceAttribute(hipDeviceAttributeMaxBlockDimZ, 0) + 1u;
     HIP_CHECK_ERROR(func(f, 1, 1, 1, 1, 1, z, 0, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
                     hipErrorInvalidValue);
   }
@@ -205,7 +205,7 @@ template <ExtModuleLaunchKernelSig* func> void ModuleLaunchKernelNegativeParamet
 // Disabled on AMD due to defect - EXSWHTEC-162
 #if HT_NVIDIA
   SECTION("blockDimX * blockDimY * blockDimZ > MaxThreadsPerBlock") {
-    const unsigned int max = GetDeviceAttribute(0, hipDeviceAttributeMaxThreadsPerBlock);
+    const unsigned int max = GetDeviceAttribute(hipDeviceAttributeMaxThreadsPerBlock, 0);
     const unsigned int dim = std::ceil(std::cbrt(max)) + 1;
     HIP_CHECK_ERROR(
         func(f, 1, 1, 1, dim, dim, dim, 0, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
@@ -216,7 +216,7 @@ template <ExtModuleLaunchKernelSig* func> void ModuleLaunchKernelNegativeParamet
 // Disabled on AMD due to defect - EXSWHTEC-159
 #if HT_NVIDIA
   SECTION("sharedMemBytes > max shared memory per block") {
-    const unsigned int max = GetDeviceAttribute(0, hipDeviceAttributeMaxSharedMemoryPerBlock) + 1u;
+    const unsigned int max = GetDeviceAttribute(hipDeviceAttributeMaxSharedMemoryPerBlock, 0) + 1u;
     HIP_CHECK_ERROR(func(f, 1, 1, 1, 1, 1, 1, max, nullptr, nullptr, nullptr, nullptr, nullptr, 0u),
                     hipErrorInvalidValue);
   }
@@ -241,8 +241,8 @@ template <ExtModuleLaunchKernelSig* func> void ModuleLaunchKernelNegativeParamet
     void* kernel_args[1] = {&result_ptr};
     // clang-format off
     void *extra[] = {
-        HIP_LAUNCH_PARAM_BUFFER_POINTER, &result_ptr, 
-        HIP_LAUNCH_PARAM_BUFFER_SIZE, &size, 
+        HIP_LAUNCH_PARAM_BUFFER_POINTER, &result_ptr,
+        HIP_LAUNCH_PARAM_BUFFER_SIZE, &size,
         HIP_LAUNCH_PARAM_END
     };
     // clang-format on

catch/unit/module/launch_kernel_module.cc

@@ -20,6 +20,7 @@ THE SOFTWARE.
 */
 
 #include <hip/hip_runtime.h>
+#include <hip/hip_cooperative_groups.h>
 
 extern "C" {
 __global__ void NOPKernel() {}
@@ -34,4 +35,9 @@ __global__ void Delay(uint32_t interval, const uint32_t ticks_per_ms) {
     }
   }
 }
+
+__global__ void CoopKernel() {
+  cooperative_groups::grid_group grid = cooperative_groups::this_grid();
+  grid.sync();
+}
 }