SWDEV-238517 - [dtest] Catch2 multiprocess tests for device files.

Changes contain Catch2 device multiprocess tests which were covered under "hip/tests/src/runtimeApi/device" and some additional tests. These test files makes use of fork call or sets visible devices env variable and validates various device aspects.

Change-Id: Iaaca37598d386104da9e5f37f92ab176bc8a2845


[ROCm/hip commit: 6a8fd2b762]

projects/hip/tests/catch/CMakeLists.txt

@@ -70,7 +70,7 @@ add_subdirectory(ABM)
 add_subdirectory(hipTestMain)
 add_subdirectory(stress)
 
-if(UNIX AND HIP_PLATFORM MATCHES "amd")
+if(UNIX)
     add_subdirectory(multiproc)
 endif()
 

projects/hip/tests/catch/hipTestMain/CMakeLists.txt

@@ -37,9 +37,15 @@ catch_discover_tests(ABMTests PROPERTIES  SKIP_REGULAR_EXPRESSION "HIP_SKIP_THIS
 add_dependencies(build_tests UnitTests ABMTests)
 
 # Add Multiproc tests as seperate binary
-if(UNIX AND HIP_PLATFORM MATCHES "amd")
+if(UNIX)
     add_executable(MultiProcTests EXCLUDE_FROM_ALL main.cc hip_test_context.cc)
-    set_property(TARGET MultiProcTests PROPERTY CXX_STANDARD 17)
+
+    if(HIP_PLATFORM MATCHES "amd")
+        set_property(TARGET MultiProcTests PROPERTY CXX_STANDARD 17)
+    else()
+        target_compile_options(MultiProcTests PUBLIC -std=c++17)
+    endif()
+
     target_link_libraries(MultiProcTests PRIVATE MultiProc
                                           stdc++fs)
     catch_discover_tests(MultiProcTests PROPERTIES  SKIP_REGULAR_EXPRESSION "HIP_SKIP_THIS_TEST")

projects/hip/tests/catch/multiproc/CMakeLists.txt

@@ -2,6 +2,13 @@
 set(LINUX_TEST_SRC
     hipMallocConcurrency.cc
     childMalloc.cc
+    hipDeviceComputeCapabilityMproc.cc
+    hipDeviceGetPCIBusIdMproc.cc
+    hipDeviceTotalMemMproc.cc
+    hipGetDeviceAttributeMproc.cc
+    hipGetDeviceCountMproc.cc
+    hipGetDevicePropertiesMproc.cc
+    hipSetGetDeviceMproc.cc
 )
 
 if(UNIX)

projects/hip/tests/catch/multiproc/hipDeviceComputeCapabilityMproc.cc

@@ -0,0 +1,159 @@
+/*
+Copyright (c) 2021-2022 Advanced Micro Devices, Inc. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR
+IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * hipDeviceComputeCapability tests
+ * Scenario: Validate behavior of hipDeviceComputeCapability for masked devices
+ */
+
+#include <hip_test_common.hh>
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/wait.h>
+
+
+#define MAX_SIZE 30
+#define VISIBLE_DEVICE 0
+
+/**
+ * Fetches Gpu device count
+ */
+static void getDeviceCount(int *pdevCnt) {
+  int fd[2], val = 0;
+  pid_t childpid;
+
+  // create pipe descriptors
+  pipe(fd);
+
+  // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+  unsetenv("CUDA_VISIBLE_DEVICES");
+#else
+  unsetenv("ROCR_VISIBLE_DEVICES");
+  unsetenv("HIP_VISIBLE_DEVICES");
+#endif
+
+  childpid = fork();
+
+  if (childpid > 0) {  // Parent
+    close(fd[1]);
+    // parent will wait to read the device cnt
+    read(fd[0], &val, sizeof(val));
+
+    // close the read-descriptor
+    close(fd[0]);
+
+    // wait for child exit
+    wait(NULL);
+
+    *pdevCnt = val;
+  } else if (!childpid) {  // Child
+    int devCnt = 1;
+    // writing only, no need for read-descriptor
+    close(fd[0]);
+
+    HIP_CHECK(hipGetDeviceCount(&devCnt));
+    // send the value on the write-descriptor:
+    write(fd[1], &devCnt, sizeof(devCnt));
+
+    // close the write descriptor:
+    close(fd[1]);
+    exit(0);
+  } else {  // failure
+    *pdevCnt = 0;
+    return;
+  }
+}
+
+/**
+ * Runs test on masked devices
+ */
+bool runMaskedDeviceTest(int actualNumGPUs) {
+  bool testResult = true;
+  int fd[2];
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    hipError_t err;
+    char visibleDeviceString[MAX_SIZE] = {};
+    snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE);
+
+    // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+    unsetenv("CUDA_VISIBLE_DEVICES");
+    setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+    HIP_CHECK(hipInit(0));
+#else
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+    setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+#endif
+
+    for (int count = 1;
+        count < actualNumGPUs; count++) {
+      int major, minor;
+      err = hipDeviceComputeCapability(&major, &minor, count);
+      if (err == hipSuccess) {
+        testResult = false;
+      } else {
+        printf("hipDeviceComputeCapability: Error Code Returned: '%s'(%d)\n",
+              hipGetErrorString(err), err);
+      }
+    }
+    close(fd[0]);
+    printf("testResult = %d \n", testResult);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+
+  } else if (cPid > 0) {  // parent
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  return testResult;
+}
+
+/**
+ * Validate behavior of hipDeviceComputeCapability for masked devices.
+ */
+TEST_CASE("Unit_hipDeviceGet_MaskedDevices") {
+  int count = -1;
+  constexpr int ReqGPUs = 2;
+  bool ret;
+
+  getDeviceCount(&count);
+
+  if (count >= ReqGPUs) {
+    ret = runMaskedDeviceTest(count);
+    REQUIRE(ret == true);
+  } else {
+    SUCCEED("Not enough GPUs to run the masked GPU tests");
+  }
+}
+
+#endif  // __linux__

projects/hip/tests/catch/multiproc/hipDeviceGetPCIBusIdMproc.cc

@@ -0,0 +1,258 @@
+/*
+ * Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved.
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * Tests to
+ * 1. Compare {pciDomainID, pciBusID, pciDeviceID} values
+ *    hipDeviceGetPCIBusId vs lspci
+ * 2. Validate behavior of hipDeviceGetPCIBusId for masked devices
+ */
+
+#include <hip_test_common.hh>
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/wait.h>
+
+#define MAX_DEVICE_LENGTH 20
+#define MAX_SIZE 30
+#define VISIBLE_DEVICE 0
+
+namespace hipDeviceGetPCIBusIdTests {
+
+/**
+ * Fetches Gpu device count
+ */
+void getDeviceCount(int *pdevCnt) {
+  int fd[2], val = 0;
+  pid_t childpid;
+
+  // create pipe descriptors
+  pipe(fd);
+
+  // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+  unsetenv("CUDA_VISIBLE_DEVICES");
+#else
+  unsetenv("ROCR_VISIBLE_DEVICES");
+  unsetenv("HIP_VISIBLE_DEVICES");
+#endif
+
+  childpid = fork();
+
+  if (childpid > 0) {  // Parent
+    close(fd[1]);
+    // parent will wait to read the device cnt
+    read(fd[0], &val, sizeof(val));
+
+    // close the read-descriptor
+    close(fd[0]);
+
+    // wait for child exit
+    wait(NULL);
+
+    *pdevCnt = val;
+  } else if (!childpid) {  // Child
+    int devCnt = 1;
+    // writing only, no need for read-descriptor
+    close(fd[0]);
+
+    HIP_CHECK(hipGetDeviceCount(&devCnt));
+    // send the value on the write-descriptor:
+    write(fd[1], &devCnt, sizeof(devCnt));
+
+    // close the write descriptor:
+    close(fd[1]);
+    exit(0);
+  } else {  // failure
+    *pdevCnt = 0;
+    return;
+  }
+}
+
+/**
+ * Runs test on masked devices
+ */
+bool testWithMaskedDevices(int actualNumGPUs) {
+  bool testResult = true;
+  int fd[2];
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    hipError_t err;
+    char pciBusId[MAX_DEVICE_LENGTH];
+    char visibleDeviceString[MAX_SIZE] = {};
+    snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE);
+
+    // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+    unsetenv("CUDA_VISIBLE_DEVICES");
+    setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+    HIP_CHECK(hipInit(0));
+#else
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+    setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+#endif
+
+    for (int count = 1;
+        count < actualNumGPUs; count++) {
+      err = hipDeviceGetPCIBusId(pciBusId, MAX_DEVICE_LENGTH, count);
+      if (err == hipSuccess) {
+        testResult &= false;
+      } else {
+        printf("hipGetDeviceProperties: Error Code Returned: '%s'(%d)\n",
+              hipGetErrorString(err), err);
+      }
+    }
+    close(fd[0]);
+    printf("testResult = %d \n", testResult);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+
+  } else if (cPid > 0) {  // parent
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  return testResult;
+}
+
+
+bool getPciBusId(int deviceCount,
+                 char **hipDeviceList) {
+  for (int i = 0; i < deviceCount; i++) {
+    HIP_CHECK(hipDeviceGetPCIBusId(hipDeviceList[i], MAX_DEVICE_LENGTH, i));
+  }
+  return true;
+}
+}  // namespace hipDeviceGetPCIBusIdTests
+
+
+/**
+ * Scenario: Validate behavior of hipDeviceGetPCIBusId for masked devices.
+ */
+TEST_CASE("Unit_hipDeviceGetPCIBusId_MaskedDevices") {
+  int count = -1;
+  constexpr int ReqGPUs = 2;
+  bool ret;
+
+  hipDeviceGetPCIBusIdTests::getDeviceCount(&count);
+
+  if (count >= ReqGPUs) {
+    ret = hipDeviceGetPCIBusIdTests::testWithMaskedDevices(count);
+    REQUIRE(ret == true);
+  } else {
+    SUCCEED("Not enough GPUs to run the masked GPU tests");
+  }
+}
+
+/* Compare {pciDomainID, pciBusID, pciDeviceID} values
+ * hipDeviceGetPCIBusId vs lspci
+ */
+TEST_CASE("Unit_hipDeviceGetPCIBusId_CheckPciBusIDWithLspci") {
+  FILE *fpipe;
+  {
+    // Check if lspci is installed, if not, don't proceed
+    char const *cmd = "lspci --version";
+    char *lspciCheck{nullptr};
+    constexpr auto MaxLen = 50;
+    char temp[MaxLen]{};
+
+    fpipe = popen(cmd, "r");
+    REQUIRE_FALSE(fpipe == nullptr);
+
+    lspciCheck = fgets(temp, MaxLen, fpipe);
+    pclose(fpipe);
+
+    if (lspciCheck == nullptr) {
+      WARN("Skipping test as lspci is not found in system");
+      return;
+    }
+  }
+
+  int deviceCount = 0;
+  HIP_CHECK(hipGetDeviceCount(&deviceCount));
+  REQUIRE_FALSE(deviceCount == 0);
+  // Allocate an array of pointer to characters
+  char **hipDeviceList = new char*[deviceCount];
+  REQUIRE_FALSE(hipDeviceList == nullptr);
+  char **pciDeviceList = new char*[deviceCount];
+  REQUIRE_FALSE(pciDeviceList == nullptr);
+  for (int i = 0; i < deviceCount; i++) {
+    hipDeviceList[i] = new char[MAX_DEVICE_LENGTH];
+    REQUIRE_FALSE(hipDeviceList[i] == nullptr);
+    pciDeviceList[i] = new char[MAX_DEVICE_LENGTH];
+    REQUIRE_FALSE(pciDeviceList[i] == nullptr);
+  }
+
+  hipDeviceGetPCIBusIdTests::getPciBusId(deviceCount, hipDeviceList);
+  char const *command = nullptr;
+  // Get lspci device list and compare with hip device list
+  if ((TestContext::get()).isNvidia()) {
+    command = "lspci -D | grep controller | grep NVIDIA | "
+              "cut -d ' ' -f 1";
+  } else {
+    command = "lspci -D | grep controller | grep AMD/ATI | "
+              "cut -d ' ' -f 1";
+  }
+  fpipe = popen(command, "r");
+  REQUIRE_FALSE(fpipe == nullptr);
+
+  int index = 0;
+  int deviceMatchCount = 0;
+  constexpr auto cmpLen = 10;
+  while (fgets(pciDeviceList[index], MAX_DEVICE_LENGTH, fpipe)) {
+    bool bMatchFound = false;
+    for (int deviceNo = 0; deviceNo < deviceCount; deviceNo++) {
+      if (!strncasecmp(pciDeviceList[index], hipDeviceList[deviceNo],
+                                                             cmpLen)) {
+        deviceMatchCount++;
+        bMatchFound = true;
+      }
+    }
+    if (bMatchFound == false) {
+      printf("PCI device: %s is not reported by HIP\n",
+                                   pciDeviceList[index]);
+    }
+    index++;
+    if (index >= deviceCount) break;
+  }
+  // Deallocate
+  for (int i = 0; i < deviceCount; i++) {
+    delete hipDeviceList[i];
+  }
+  delete[] hipDeviceList;
+  for (int i = 0; i < deviceCount; i++) {
+    delete pciDeviceList[i];
+  }
+  delete[] pciDeviceList;
+  pclose(fpipe);
+
+  REQUIRE(deviceMatchCount == deviceCount);
+}
+#endif

projects/hip/tests/catch/multiproc/hipDeviceTotalMemMproc.cc

@@ -0,0 +1,161 @@
+/*
+Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR
+IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * hipDeviceTotalMem tests
+ * Scenario: Validate behavior of hipDeviceTotalMem for masked devices.
+ */
+
+#include <hip_test_common.hh>
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/wait.h>
+
+#define MAX_SIZE 30
+#define VISIBLE_DEVICE 0
+
+/**
+ * Fetches Gpu device count
+ */
+static void getDeviceCount(int *pdevCnt) {
+  int fd[2], val = 0;
+  pid_t childpid;
+
+  // create pipe descriptors
+  pipe(fd);
+
+  // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+  unsetenv("CUDA_VISIBLE_DEVICES");
+#else
+  unsetenv("ROCR_VISIBLE_DEVICES");
+  unsetenv("HIP_VISIBLE_DEVICES");
+#endif
+
+  childpid = fork();
+
+  if (childpid > 0) {  // Parent
+    close(fd[1]);
+    // parent will wait to read the device cnt
+    read(fd[0], &val, sizeof(val));
+
+    // close the read-descriptor
+    close(fd[0]);
+
+    // wait for child exit
+    wait(NULL);
+
+    *pdevCnt = val;
+  } else if (!childpid) {  // Child
+    int devCnt = 1;
+    // writing only, no need for read-descriptor
+    close(fd[0]);
+
+    HIP_CHECK(hipGetDeviceCount(&devCnt));
+    // send the value on the write-descriptor:
+    write(fd[1], &devCnt, sizeof(devCnt));
+
+    // close the write descriptor:
+    close(fd[1]);
+    exit(0);
+  } else {  // failure
+    *pdevCnt = 0;
+    return;
+  }
+}
+
+
+/**
+ * Func tries to fetch total memory of masked devices and returns pass/fail.
+ */
+static bool getTotalMemoryOfMaskedDevices(int actualNumGPUs) {
+  bool testResult = true;
+  int fd[2];
+
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    hipError_t err;
+    char visibleDeviceString[MAX_SIZE] = {};
+    snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE);
+
+    // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+    unsetenv("CUDA_VISIBLE_DEVICES");
+    setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+    HIP_CHECK(hipInit(0));
+#else
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+    setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+#endif
+
+    for (int count = 1;
+        count < actualNumGPUs; count++) {
+      size_t totMem;
+      err = hipDeviceTotalMem(&totMem, count);
+      if (err == hipSuccess) {
+        testResult &= false;
+      } else {
+        printf("hipDeviceTotalMem: Error Code Returned: '%s'(%d)\n",
+              hipGetErrorString(err), err);
+      }
+    }
+    close(fd[0]);
+    printf("testResult = %d \n", testResult);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+
+  } else if (cPid > 0) {  // parent
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  return testResult;
+}
+
+
+/**
+ * Scenario: Validate behavior of hipDeviceTotalMem for masked devices.
+ */
+TEST_CASE("Unit_hipDeviceTotalMem_MaskedDevices") {
+  int count = -1;
+  constexpr int ReqGPUs = 2;
+  bool ret;
+
+  getDeviceCount(&count);
+
+  if (count >= ReqGPUs) {
+    ret = getTotalMemoryOfMaskedDevices(count);
+    REQUIRE(ret == true);
+  } else {
+    SUCCEED("Not enough GPUs to run the masked GPU tests");
+  }
+}
+
+#endif

projects/hip/tests/catch/multiproc/hipGetDeviceAttributeMproc.cc

@@ -0,0 +1,164 @@
+/*
+Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * hipGetDeviceAttribute tests
+ * Scenario: Validate behavior of hipGetDeviceAttribute for masked devices.
+ */
+
+#include <hip_test_common.hh>
+#include <iostream>
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/wait.h>
+
+#define MAX_SIZE 30
+#define VISIBLE_DEVICE 0
+
+/**
+ * Fetches Gpu device count
+ */
+static void getDeviceCount(int *pdevCnt) {
+  int fd[2], val = 0;
+  pid_t childpid;
+
+  // create pipe descriptors
+  pipe(fd);
+
+  // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+  unsetenv("CUDA_VISIBLE_DEVICES");
+#else
+  unsetenv("ROCR_VISIBLE_DEVICES");
+  unsetenv("HIP_VISIBLE_DEVICES");
+#endif
+
+  childpid = fork();
+
+  if (childpid > 0) {  // Parent
+    close(fd[1]);
+    // parent will wait to read the device cnt
+    read(fd[0], &val, sizeof(val));
+
+    // close the read-descriptor
+    close(fd[0]);
+
+    // wait for child exit
+    wait(NULL);
+
+    *pdevCnt = val;
+  } else if (!childpid) {  // Child
+    int devCnt = 1;
+    // writing only, no need for read-descriptor
+    close(fd[0]);
+
+    HIP_CHECK(hipGetDeviceCount(&devCnt));
+    // send the value on the write-descriptor:
+    write(fd[1], &devCnt, sizeof(devCnt));
+
+    // close the write descriptor:
+    close(fd[1]);
+    exit(0);
+  } else {  // failure
+    *pdevCnt = 0;
+    return;
+  }
+}
+
+
+/**
+ * Tries to fetch device attribute of masked devices and returns pass/fail.
+ */
+static bool validateGetAttributeOfMaskedDevices(int actualNumGPUs) {
+  bool testResult = true;
+  int fd[2];
+
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    hipError_t err;
+    char visibleDeviceString[MAX_SIZE] = {};
+    snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE);
+
+    // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+    unsetenv("CUDA_VISIBLE_DEVICES");
+    setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+    HIP_CHECK(hipInit(0));
+#else
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+    setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+#endif
+
+    for (int count = 1;
+        count < actualNumGPUs; count++) {
+      int pi = -1;
+      err = hipDeviceGetAttribute(&pi, hipDeviceAttributePciBusId, count);
+      if (err == hipSuccess) {
+        testResult &= false;
+      } else {
+        printf("hipDeviceGetAttribute: Error Code Returned: '%s'(%d)\n",
+              hipGetErrorString(err), err);
+      }
+    }
+    close(fd[0]);
+    printf("testResult = %d \n", testResult);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+
+  } else if (cPid > 0) {  // parent
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  return testResult;
+}
+
+
+/**
+ * Scenario: Validate behavior of hipDeviceGetAttribute for masked devices.
+ */
+TEST_CASE("Unit_hipDeviceGetAttribute_MaskedDevices") {
+  int count = -1;
+  constexpr int ReqGPUs = 2;
+  bool ret;
+
+  getDeviceCount(&count);
+
+  if (count >= ReqGPUs) {
+    ret = validateGetAttributeOfMaskedDevices(count);
+    REQUIRE(ret == true);
+  } else {
+    SUCCEED("Not enough GPUs to run the masked GPU tests");
+  }
+}
+#endif

projects/hip/tests/catch/multiproc/hipGetDeviceCountMproc.cc

@@ -0,0 +1,54 @@
+/*
+Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR
+IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * hipGetDeviceCount tests
+ * Scenario: Validates the value of numDevices when devices are hidden.
+ */
+
+#include <hip_test_common.hh>
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/wait.h>
+
+#define MAX_SIZE 30
+#define VISIBLE_DEVICE 0
+
+/**
+ * Validate behavior of hipGetDeviceCount for masked devices.
+ */
+TEST_CASE("Unit_hipGetDeviceCount_MaskedDevices") {
+  int numDevices = 0;
+  char visibleDeviceString[MAX_SIZE] = {};
+  snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE);
+
+#ifdef __HIP_PLATFORM_NVCC__
+  unsetenv("CUDA_VISIBLE_DEVICES");
+  setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+#else
+  unsetenv("ROCR_VISIBLE_DEVICES");
+  unsetenv("HIP_VISIBLE_DEVICES");
+  setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+  setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+#endif
+
+  HIP_CHECK(hipGetDeviceCount(&numDevices));
+  REQUIRE(numDevices == 1);
+}
+#endif

projects/hip/tests/catch/multiproc/hipGetDevicePropertiesMproc.cc

@@ -0,0 +1,165 @@
+/*
+Copyright (c) 2020-Present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/**
+ * Scenario: Validate behavior of hipGetDeviceProperties for masked devices.
+ */
+
+#include <hip_test_common.hh>
+#ifdef __linux__
+#include <unistd.h>
+#include <sys/wait.h>
+
+
+#define MAX_SIZE 30
+#define VISIBLE_DEVICE 0
+
+/**
+ * Fetches Gpu device count
+ */
+static void getDeviceCount(int *pdevCnt) {
+  int fd[2], val = 0;
+  pid_t childpid;
+
+  // create pipe descriptors
+  pipe(fd);
+
+  // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+  unsetenv("CUDA_VISIBLE_DEVICES");
+#else
+  unsetenv("ROCR_VISIBLE_DEVICES");
+  unsetenv("HIP_VISIBLE_DEVICES");
+#endif
+
+  childpid = fork();
+
+  if (childpid > 0) {  // Parent
+    close(fd[1]);
+    // parent will wait to read the device cnt
+    read(fd[0], &val, sizeof(val));
+
+    // close the read-descriptor
+    close(fd[0]);
+
+    // wait for child exit
+    wait(NULL);
+
+    *pdevCnt = val;
+  } else if (!childpid) {  // Child
+    int devCnt = 1;
+    // writing only, no need for read-descriptor
+    close(fd[0]);
+
+    HIP_CHECK(hipGetDeviceCount(&devCnt));
+    // send the value on the write-descriptor:
+    write(fd[1], &devCnt, sizeof(devCnt));
+
+    // close the write descriptor:
+    close(fd[1]);
+    exit(0);
+  } else {  // failure
+    *pdevCnt = 0;
+    return;
+  }
+}
+
+
+
+/**
+ * Tries to fetch device properties of masked devices and returns pass/fail.
+ */
+static bool validateGetPropsOfMaskedDevices(int actualNumGPUs) {
+  bool testResult = true;
+  int fd[2];
+
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    hipError_t err;
+    char visibleDeviceString[MAX_SIZE] = {};
+    snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE);
+
+    // disable visible_devices env from shell
+#ifdef __HIP_PLATFORM_NVCC__
+    unsetenv("CUDA_VISIBLE_DEVICES");
+    setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+    HIP_CHECK(hipInit(0));
+#else
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+    setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+#endif
+
+    for (int count = 1;
+        count < actualNumGPUs; count++) {
+      hipDeviceProp_t prop;
+      err = hipGetDeviceProperties(&prop, count);
+      if (err == hipSuccess) {
+        testResult &= false;
+      } else {
+        printf("hipGetDeviceProperties: Error Code Returned: '%s'(%d)\n",
+              hipGetErrorString(err), err);
+      }
+    }
+    close(fd[0]);
+    printf("testResult = %d \n", testResult);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+
+  } else if (cPid > 0) {  // parent
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  return testResult;
+}
+
+
+
+/**
+ * Scenario: Validate behavior of hipGetDeviceProperties for masked devices.
+ */
+TEST_CASE("Unit_hipGetDeviceProperties_MaskedDevices") {
+  int count = -1;
+  constexpr int ReqGPUs = 2;
+  bool ret;
+
+  getDeviceCount(&count);
+
+  if (count >= ReqGPUs) {
+    ret = validateGetPropsOfMaskedDevices(count);
+    REQUIRE(ret == true);
+  } else {
+    SUCCEED("Not enough GPUs to run the masked GPU tests");
+  }
+}
+#endif

projects/hip/tests/catch/multiproc/hipSetGetDeviceMproc.cc

@@ -0,0 +1,570 @@
+/*
+ * Copyright (c) 2020-present Advanced Micro Devices, Inc. All rights reserved.
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * Test designed to run on Linux based platforms
+ * Verifies functionality of
+ * -- hipSetDevice and hipGetDevice with different ROCR_VISIBLE_DEVICES and
+ *    HIP_VISIBLE_DEVICES values set
+ */
+
+#include <hip_test_common.hh>
+
+#ifdef __linux__
+#include <sys/wait.h>
+#include <unistd.h>
+
+
+#define MAX_SIZE 30
+
+/**
+ * Fetches Gpu device count
+ */
+static void getDeviceCount(int *pdevCnt) {
+  int fd[2], val = 0;
+  pid_t childpid;
+
+  // create pipe descriptors
+  pipe(fd);
+
+  // disable visible_devices env from shell
+  unsetenv("ROCR_VISIBLE_DEVICES");
+  unsetenv("HIP_VISIBLE_DEVICES");
+
+
+  childpid = fork();
+
+  if (childpid > 0) {  // Parent
+    close(fd[1]);
+    // parent will wait to read the device cnt
+    read(fd[0], &val, sizeof(val));
+
+    // close the read-descriptor
+    close(fd[0]);
+
+    // wait for child exit
+    wait(NULL);
+
+    *pdevCnt = val;
+  } else if (!childpid) {  // Child
+    int devCnt = 1;
+    // writing only, no need for read-descriptor
+    close(fd[0]);
+
+    HIP_CHECK(hipGetDeviceCount(&devCnt));
+
+    // send the value on the write-descriptor:
+    write(fd[1], &devCnt, sizeof(devCnt));
+
+    // close the write descriptor:
+    close(fd[1]);
+    exit(0);
+  } else {  // failure
+    *pdevCnt = 0;
+  }
+}
+
+
+// Pass either -1 in deviceNumber or invalid device number
+static void testInvalidDevice(int numDevices, bool useRocrEnv,
+                                                   int deviceNumber) {
+  bool testResult = true;
+  int device;
+  int tempCount = 0;
+  int setDeviceErrorCheck = 0;
+  int getDeviceErrorCheck = 0;
+  int getDeviceCountErrorCheck = 0;
+  int fd[2];
+  pipe(fd);
+
+  pid_t cPid;
+  cPid = fork();
+
+  char visibleDeviceString[MAX_SIZE] = {};
+  snprintf(visibleDeviceString, MAX_SIZE, "%d", deviceNumber);
+
+  if (cPid == 0) {  // child
+    hipError_t err;
+#ifdef __HIP_PLATFORM_NVCC__
+    setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+#else
+    if (true == useRocrEnv) {
+      setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+    } else {
+      setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+    }
+#endif
+    err = hipGetDeviceCount(&tempCount);
+    if (err != hipSuccess) {
+      getDeviceCountErrorCheck = 1;
+    }
+    for (int i = 0; i < numDevices; i++) {
+      err = hipSetDevice(i);
+      if (err != hipSuccess) {
+        setDeviceErrorCheck+= 1;
+      }
+
+      err = hipGetDevice(&device);
+      if (err != hipSuccess) {
+        getDeviceErrorCheck+= 1;
+      }
+    }
+
+    if ((getDeviceCountErrorCheck == 1) && (setDeviceErrorCheck == numDevices)
+        && (getDeviceErrorCheck == numDevices)) {
+      testResult = true;
+
+    } else {
+      printf("Test failed for invalid device, getDeviceCountErrorCheck %d,"
+             "setDeviceErrorCheck %d, getDeviceErrorCheck %d\n",
+             getDeviceCountErrorCheck, setDeviceErrorCheck,
+             getDeviceErrorCheck);
+
+      testResult = false;
+    }
+
+    close(fd[0]);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+
+  } else if (cPid > 0) {  // parent
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  REQUIRE(testResult == true);
+}
+
+
+static void testValidDevices(int numDevices, bool useRocrEnv, int *deviceList,
+    int deviceListLength) {
+  bool testResult = true;
+  int tempCount = 0;
+  int device;
+  int setDeviceErrorCheck = 0;
+  int getDeviceErrorCheck = 0;
+  int getDeviceCountErrorCheck = 0;
+  int *deviceListPtr = deviceList;
+  char visibleDeviceString[MAX_SIZE] = {};
+
+  if ((NULL == deviceList) || ((deviceListLength < 1) ||
+        deviceListLength > numDevices)) {
+    INFO("Invalid argument for number of devices. Skipping current test");
+    REQUIRE(false);
+  }
+
+  for (int i = 0; i < deviceListLength; i++) {
+    snprintf(visibleDeviceString + strlen(visibleDeviceString), MAX_SIZE, "%d,",
+        *deviceListPtr++);
+  }
+
+  visibleDeviceString[strlen(visibleDeviceString)-1] = 0;
+
+  int fd[2];
+  pipe(fd);
+
+  pid_t cPid;
+  cPid = fork();
+
+  if (cPid == 0) {
+#ifdef __HIP_PLATFORM_NVCC__
+    unsetenv("CUDA_VISIBLE_DEVICES");
+    setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1);
+#else
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    if (true == useRocrEnv) {
+      setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1);
+    } else {
+      setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1);
+    }
+#endif
+
+
+    hipError_t err;
+    err = hipGetDeviceCount(&tempCount);
+
+    if (tempCount == deviceListLength) {
+      getDeviceCountErrorCheck = 1;
+    } else {
+      printf("hipGetDeviceCount failed. return value: %u\n", hipError_t(err));
+    }
+
+    for (int i = 0; i < numDevices; i++) {
+      err = hipSetDevice(i);
+      if (err != hipSuccess) {
+        setDeviceErrorCheck+= 1;
+      }
+
+      err = hipGetDevice(&device);
+      if (err != hipSuccess) {
+        getDeviceErrorCheck+= 1;
+      }
+    }
+
+    if ((getDeviceCountErrorCheck == 1) && (setDeviceErrorCheck ==
+          (numDevices-deviceListLength)) && (getDeviceErrorCheck == 0)) {
+      testResult = true;
+
+    } else {
+      printf("Test failed for device count %d\n", deviceListLength);
+      testResult = false;
+    }
+
+    close(fd[0]);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+
+  } else if (cPid > 0) {
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  REQUIRE(testResult == true);
+}
+
+
+static void Initialize(int *deviceList, int numDevices, int count,
+    char min_visibleDeviceString[], char max_visibleDeviceString[]) {
+  int *deviceListPtr = deviceList;
+  for (int i =0; i < count; i++) {
+    if (i == count-1) {
+      snprintf(min_visibleDeviceString + strlen(min_visibleDeviceString),
+              MAX_SIZE, "%d", *deviceListPtr++);
+    } else {
+      snprintf(min_visibleDeviceString + strlen(min_visibleDeviceString),
+              MAX_SIZE, "%d,", *deviceListPtr++);
+    }
+  }
+  for (int i =0; i < numDevices; i++) {
+    if (i == numDevices-1) {
+      snprintf(max_visibleDeviceString + strlen(max_visibleDeviceString),
+               MAX_SIZE, "%d", i);
+    } else {
+      snprintf(max_visibleDeviceString + strlen(max_visibleDeviceString),
+               MAX_SIZE, "%d,", i);
+    }
+  }
+}
+
+static void testMaxRvdMinHvd(int numDevices, int *deviceList, int count) {
+  bool testResult = true;
+  int device;
+  int validateCount = 0;
+  char min_visibleDeviceString[MAX_SIZE] = {0};
+  char max_visibleDeviceString[MAX_SIZE] = {0};
+  int fd[2];
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    Initialize(deviceList, numDevices,
+        count, min_visibleDeviceString, max_visibleDeviceString);
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", max_visibleDeviceString, 1);
+    setenv("HIP_VISIBLE_DEVICES", min_visibleDeviceString, 1);
+    HIP_CHECK(hipGetDeviceCount(&numDevices));
+    for (int i = 0; i < numDevices; i++) {
+      HIP_CHECK(hipSetDevice(i));
+      HIP_CHECK(hipGetDevice(&device));
+      if (device == i) {
+         validateCount+= 1;
+      }
+    }
+    if (count != validateCount) {
+      testResult = false;
+    }
+  } else if (cPid > 0) {
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  REQUIRE(testResult == true);
+}
+
+static void testRvdCvd(int numDevices, int *deviceList, int count) {
+  bool testResult = true;
+  int device;
+  int validateCount = 0;
+  char min_visibleDeviceString[MAX_SIZE] = {0};
+  char max_visibleDeviceString[MAX_SIZE] = {0};
+  int fd[2];
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    Initialize(deviceList, numDevices, count,
+             min_visibleDeviceString, max_visibleDeviceString);
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", max_visibleDeviceString, 1);
+    setenv("CUDA_VISIBLE_DEVICES", min_visibleDeviceString, 1);
+    HIP_CHECK(hipGetDeviceCount(&numDevices));
+    for (int i = 0; i < numDevices; i++) {
+      HIP_CHECK(hipSetDevice(i));
+      HIP_CHECK(hipGetDevice(&device));
+      if (device == i) {
+        validateCount+= 1;
+      }
+    }
+    if (count != validateCount) {
+      testResult = false;
+    }
+  } else if (cPid > 0) {
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  REQUIRE(testResult == true);
+}
+
+static void testMinRvdMaxHvd(int numDevices, int *deviceList, int count) {
+  bool testResult = true;
+  int device;
+  int validateCount = 0;
+  char min_visibleDeviceString[MAX_SIZE] = {0};
+  char max_visibleDeviceString[MAX_SIZE] = {0};
+  int fd[2];
+  pipe(fd);
+  pid_t cPid;
+  cPid = fork();
+  if (cPid == 0) {  // child
+    Initialize(deviceList, numDevices, count,
+              min_visibleDeviceString, max_visibleDeviceString);
+    unsetenv("ROCR_VISIBLE_DEVICES");
+    unsetenv("HIP_VISIBLE_DEVICES");
+    setenv("ROCR_VISIBLE_DEVICES", min_visibleDeviceString, 1);
+    setenv("HIP_VISIBLE_DEVICES", max_visibleDeviceString, 1);
+    HIP_CHECK(hipGetDeviceCount(&numDevices));
+    for (int i = 0; i < numDevices; i++) {
+      HIP_CHECK(hipSetDevice(i));
+      HIP_CHECK(hipGetDevice(&device));
+      if (device == i) {
+         validateCount+= 1;
+      }
+    }
+    if (count != validateCount) {
+      testResult = false;
+    }
+    close(fd[0]);
+    write(fd[1], &testResult, sizeof(testResult));
+    close(fd[1]);
+    exit(0);
+  } else if (cPid > 0) {
+    close(fd[1]);
+    read(fd[0], &testResult, sizeof(testResult));
+    close(fd[0]);
+    wait(NULL);
+  } else {
+    printf("fork() failed\n");
+    HIP_ASSERT(false);
+  }
+
+  REQUIRE(testResult == true);
+}
+
+/**
+ * Scenario sets Invalid visible device list and checks behavior.
+ */
+TEST_CASE("Unit_hipSetDevice_InvalidVisibleDeviceList") {
+  int numDevices = 0;
+
+  getDeviceCount(&numDevices);
+  REQUIRE(numDevices != 0);
+
+  SECTION("Test setting -1 to HIP_VISIBLE_DEVICES") {
+    testInvalidDevice(numDevices, false, -1);
+  }
+
+  SECTION("Test setting invalid device to HIP_VISIBLE_DEVICES") {
+    testInvalidDevice(numDevices, false, numDevices);
+  }
+#ifndef __HIP_PLATFORM_NVCC__
+  SECTION("Test setting -1 to ROCR_VISIBLE_DEVICES") {
+    testInvalidDevice(numDevices, true, -1);
+  }
+
+  SECTION("Test setting invalid device to ROCR_VISIBLE_DEVICES") {
+    testInvalidDevice(numDevices, true, numDevices);
+  }
+#endif
+}
+
+/**
+ * Scenario sets valid visible device list and checks behavior.
+ */
+TEST_CASE("Unit_hipSetDevice_ValidVisibleDeviceList") {
+  int numDevices = 0;
+  int deviceList[MAX_SIZE];
+
+  getDeviceCount(&numDevices);
+  REQUIRE(numDevices != 0);
+
+  // Test for all available devices
+  for (int i = 0; i < numDevices; i++) {
+    deviceList[i] = i;
+  }
+
+  SECTION("Test setting valid hip visible device list") {
+    testValidDevices(numDevices, false, deviceList, numDevices);
+  }
+#ifndef __HIP_PLATFORM_NVCC__
+  SECTION("Test setting valid rocr visible device list") {
+    testValidDevices(numDevices, true, deviceList, numDevices);
+  }
+#endif
+}
+
+/**
+ * Scenario sets subset of available devices and checks behavior.
+ */
+TEST_CASE("Unit_hipSetDevice_SubsetOfAvailableDevices") {
+  int numDevices = 0;
+  int deviceList[MAX_SIZE];
+  int deviceListLength = 1;
+
+  getDeviceCount(&numDevices);
+  REQUIRE(numDevices != 0);
+
+  // Test for subset of available gpus
+  for (int i=0; i < deviceListLength; i++) {
+    deviceList[i] = deviceListLength-1-i;
+  }
+
+#ifndef __HIP_PLATFORM_NVCC__
+  testValidDevices(numDevices, true, deviceList,
+        deviceListLength);
+#endif
+  testValidDevices(numDevices, false, deviceList,
+        deviceListLength);
+}
+
+#ifndef __HIP_PLATFORM_NVCC__
+/* Following tests apply only for AMD Platforms */
+
+/**
+ * Scenario tests getDevice behavior with Minimal Len of RVD
+ * and Maximal Len of HVD
+ */
+TEST_CASE("Unit_hipSetDevice_MinRvdMaxHvdDevicesList") {
+  int numDevices = 0;
+  int deviceList[MAX_SIZE];
+  int count = 0;
+
+  getDeviceCount(&numDevices);
+
+  REQUIRE(numDevices != 0);
+
+  if (numDevices == 1) {
+    deviceList[0] = 0;
+    count = 1;
+  } else {
+    for (int i=0; i < numDevices; i++) {
+      if (i%2 == 0) {
+        deviceList[count] = i;
+        count++;
+      }
+    }
+  }
+
+  testMinRvdMaxHvd(numDevices, deviceList, count);
+}
+
+/**
+ * Scenario tests getDevice behavior with Maximal Len of RVD
+ * and Minimal Len of HVD
+ */
+TEST_CASE("Unit_hipSetDevice_MaxRvdMinHvdDevicesList") {
+  int numDevices = 0;
+  int deviceList[MAX_SIZE];
+  int count = 0;
+
+  getDeviceCount(&numDevices);
+
+  REQUIRE(numDevices != 0);
+
+  if (numDevices == 1) {
+    deviceList[0] = 0;
+    count = 1;
+  } else {
+    for (int i=0; i < numDevices; i++) {
+      if (i%2 == 0) {
+        deviceList[count] = i;
+        count++;
+      }
+    }
+  }
+
+  testMaxRvdMinHvd(numDevices, deviceList, count);
+}
+
+/**
+ * Scenario tests getDevice behavior with combination of RVD and CVD
+ */
+TEST_CASE("Unit_hipSetDevice_RvdCvdDevicesList") {
+  int numDevices = 0;
+  int deviceList[MAX_SIZE];
+  int count = 0;
+
+  getDeviceCount(&numDevices);
+
+  REQUIRE(numDevices != 0);
+
+  if (numDevices == 1) {
+    deviceList[0] = 0;
+    count = 1;
+  } else {
+    for (int i=0; i < numDevices; i++) {
+      if (i%2 == 0) {
+        deviceList[count] = i;
+        count++;
+      }
+    }
+  }
+
+  testRvdCvd(numDevices, deviceList, count);
+}
+#endif  // __HIP_PLATFORM_NVCC__
+
+#endif  // __linux__