From 8a5b1e02a557bda051387d51fe1362d27d44d2f2 Mon Sep 17 00:00:00 2001 From: sumanthtg Date: Wed, 7 Jul 2021 12:49:13 +0530 Subject: [PATCH] SWDEV-238517 - [dtest] Catch2 multiprocess tests for device files. Changes contain Catch2 device multiprocess tests which were covered under "hip/tests/src/runtimeApi/device" and some additional tests. These test files makes use of fork call or sets visible devices env variable and validates various device aspects. Change-Id: Iaaca37598d386104da9e5f37f92ab176bc8a2845 [ROCm/hip commit: 6a8fd2b76284a14052c100420d3d5a4804c0b3b9] --- projects/hip/tests/catch/CMakeLists.txt | 2 +- .../tests/catch/hipTestMain/CMakeLists.txt | 10 +- .../hip/tests/catch/multiproc/CMakeLists.txt | 7 + .../hipDeviceComputeCapabilityMproc.cc | 159 +++++ .../multiproc/hipDeviceGetPCIBusIdMproc.cc | 258 ++++++++ .../catch/multiproc/hipDeviceTotalMemMproc.cc | 161 +++++ .../multiproc/hipGetDeviceAttributeMproc.cc | 164 +++++ .../catch/multiproc/hipGetDeviceCountMproc.cc | 54 ++ .../multiproc/hipGetDevicePropertiesMproc.cc | 165 +++++ .../catch/multiproc/hipSetGetDeviceMproc.cc | 570 ++++++++++++++++++ 10 files changed, 1547 insertions(+), 3 deletions(-) create mode 100644 projects/hip/tests/catch/multiproc/hipDeviceComputeCapabilityMproc.cc create mode 100644 projects/hip/tests/catch/multiproc/hipDeviceGetPCIBusIdMproc.cc create mode 100644 projects/hip/tests/catch/multiproc/hipDeviceTotalMemMproc.cc create mode 100644 projects/hip/tests/catch/multiproc/hipGetDeviceAttributeMproc.cc create mode 100644 projects/hip/tests/catch/multiproc/hipGetDeviceCountMproc.cc create mode 100644 projects/hip/tests/catch/multiproc/hipGetDevicePropertiesMproc.cc create mode 100644 projects/hip/tests/catch/multiproc/hipSetGetDeviceMproc.cc diff --git a/projects/hip/tests/catch/CMakeLists.txt b/projects/hip/tests/catch/CMakeLists.txt index d5eee44edc..0fddc8f346 100644 --- a/projects/hip/tests/catch/CMakeLists.txt +++ b/projects/hip/tests/catch/CMakeLists.txt @@ -70,7 +70,7 @@ add_subdirectory(ABM) add_subdirectory(hipTestMain) add_subdirectory(stress) -if(UNIX AND HIP_PLATFORM MATCHES "amd") +if(UNIX) add_subdirectory(multiproc) endif() diff --git a/projects/hip/tests/catch/hipTestMain/CMakeLists.txt b/projects/hip/tests/catch/hipTestMain/CMakeLists.txt index 37d7d6568d..5102046a1e 100644 --- a/projects/hip/tests/catch/hipTestMain/CMakeLists.txt +++ b/projects/hip/tests/catch/hipTestMain/CMakeLists.txt @@ -37,9 +37,15 @@ catch_discover_tests(ABMTests PROPERTIES SKIP_REGULAR_EXPRESSION "HIP_SKIP_THIS add_dependencies(build_tests UnitTests ABMTests) # Add Multiproc tests as seperate binary -if(UNIX AND HIP_PLATFORM MATCHES "amd") +if(UNIX) add_executable(MultiProcTests EXCLUDE_FROM_ALL main.cc hip_test_context.cc) - set_property(TARGET MultiProcTests PROPERTY CXX_STANDARD 17) + + if(HIP_PLATFORM MATCHES "amd") + set_property(TARGET MultiProcTests PROPERTY CXX_STANDARD 17) + else() + target_compile_options(MultiProcTests PUBLIC -std=c++17) + endif() + target_link_libraries(MultiProcTests PRIVATE MultiProc stdc++fs) catch_discover_tests(MultiProcTests PROPERTIES SKIP_REGULAR_EXPRESSION "HIP_SKIP_THIS_TEST") diff --git a/projects/hip/tests/catch/multiproc/CMakeLists.txt b/projects/hip/tests/catch/multiproc/CMakeLists.txt index a782262380..eab78965b0 100644 --- a/projects/hip/tests/catch/multiproc/CMakeLists.txt +++ b/projects/hip/tests/catch/multiproc/CMakeLists.txt @@ -2,6 +2,13 @@ set(LINUX_TEST_SRC hipMallocConcurrency.cc childMalloc.cc + hipDeviceComputeCapabilityMproc.cc + hipDeviceGetPCIBusIdMproc.cc + hipDeviceTotalMemMproc.cc + hipGetDeviceAttributeMproc.cc + hipGetDeviceCountMproc.cc + hipGetDevicePropertiesMproc.cc + hipSetGetDeviceMproc.cc ) if(UNIX) diff --git a/projects/hip/tests/catch/multiproc/hipDeviceComputeCapabilityMproc.cc b/projects/hip/tests/catch/multiproc/hipDeviceComputeCapabilityMproc.cc new file mode 100644 index 0000000000..19be9dcf58 --- /dev/null +++ b/projects/hip/tests/catch/multiproc/hipDeviceComputeCapabilityMproc.cc @@ -0,0 +1,159 @@ +/* +Copyright (c) 2021-2022 Advanced Micro Devices, Inc. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR +IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/** + * hipDeviceComputeCapability tests + * Scenario: Validate behavior of hipDeviceComputeCapability for masked devices + */ + +#include +#ifdef __linux__ +#include +#include + + +#define MAX_SIZE 30 +#define VISIBLE_DEVICE 0 + +/** + * Fetches Gpu device count + */ +static void getDeviceCount(int *pdevCnt) { + int fd[2], val = 0; + pid_t childpid; + + // create pipe descriptors + pipe(fd); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); +#endif + + childpid = fork(); + + if (childpid > 0) { // Parent + close(fd[1]); + // parent will wait to read the device cnt + read(fd[0], &val, sizeof(val)); + + // close the read-descriptor + close(fd[0]); + + // wait for child exit + wait(NULL); + + *pdevCnt = val; + } else if (!childpid) { // Child + int devCnt = 1; + // writing only, no need for read-descriptor + close(fd[0]); + + HIP_CHECK(hipGetDeviceCount(&devCnt)); + // send the value on the write-descriptor: + write(fd[1], &devCnt, sizeof(devCnt)); + + // close the write descriptor: + close(fd[1]); + exit(0); + } else { // failure + *pdevCnt = 0; + return; + } +} + +/** + * Runs test on masked devices + */ +bool runMaskedDeviceTest(int actualNumGPUs) { + bool testResult = true; + int fd[2]; + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + hipError_t err; + char visibleDeviceString[MAX_SIZE] = {}; + snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); + HIP_CHECK(hipInit(0)); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); +#endif + + for (int count = 1; + count < actualNumGPUs; count++) { + int major, minor; + err = hipDeviceComputeCapability(&major, &minor, count); + if (err == hipSuccess) { + testResult = false; + } else { + printf("hipDeviceComputeCapability: Error Code Returned: '%s'(%d)\n", + hipGetErrorString(err), err); + } + } + close(fd[0]); + printf("testResult = %d \n", testResult); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + + } else if (cPid > 0) { // parent + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + return testResult; +} + +/** + * Validate behavior of hipDeviceComputeCapability for masked devices. + */ +TEST_CASE("Unit_hipDeviceGet_MaskedDevices") { + int count = -1; + constexpr int ReqGPUs = 2; + bool ret; + + getDeviceCount(&count); + + if (count >= ReqGPUs) { + ret = runMaskedDeviceTest(count); + REQUIRE(ret == true); + } else { + SUCCEED("Not enough GPUs to run the masked GPU tests"); + } +} + +#endif // __linux__ diff --git a/projects/hip/tests/catch/multiproc/hipDeviceGetPCIBusIdMproc.cc b/projects/hip/tests/catch/multiproc/hipDeviceGetPCIBusIdMproc.cc new file mode 100644 index 0000000000..5accbe20ed --- /dev/null +++ b/projects/hip/tests/catch/multiproc/hipDeviceGetPCIBusIdMproc.cc @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved. + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/* + * Tests to + * 1. Compare {pciDomainID, pciBusID, pciDeviceID} values + * hipDeviceGetPCIBusId vs lspci + * 2. Validate behavior of hipDeviceGetPCIBusId for masked devices + */ + +#include +#ifdef __linux__ +#include +#include + +#define MAX_DEVICE_LENGTH 20 +#define MAX_SIZE 30 +#define VISIBLE_DEVICE 0 + +namespace hipDeviceGetPCIBusIdTests { + +/** + * Fetches Gpu device count + */ +void getDeviceCount(int *pdevCnt) { + int fd[2], val = 0; + pid_t childpid; + + // create pipe descriptors + pipe(fd); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); +#endif + + childpid = fork(); + + if (childpid > 0) { // Parent + close(fd[1]); + // parent will wait to read the device cnt + read(fd[0], &val, sizeof(val)); + + // close the read-descriptor + close(fd[0]); + + // wait for child exit + wait(NULL); + + *pdevCnt = val; + } else if (!childpid) { // Child + int devCnt = 1; + // writing only, no need for read-descriptor + close(fd[0]); + + HIP_CHECK(hipGetDeviceCount(&devCnt)); + // send the value on the write-descriptor: + write(fd[1], &devCnt, sizeof(devCnt)); + + // close the write descriptor: + close(fd[1]); + exit(0); + } else { // failure + *pdevCnt = 0; + return; + } +} + +/** + * Runs test on masked devices + */ +bool testWithMaskedDevices(int actualNumGPUs) { + bool testResult = true; + int fd[2]; + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + hipError_t err; + char pciBusId[MAX_DEVICE_LENGTH]; + char visibleDeviceString[MAX_SIZE] = {}; + snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); + HIP_CHECK(hipInit(0)); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); +#endif + + for (int count = 1; + count < actualNumGPUs; count++) { + err = hipDeviceGetPCIBusId(pciBusId, MAX_DEVICE_LENGTH, count); + if (err == hipSuccess) { + testResult &= false; + } else { + printf("hipGetDeviceProperties: Error Code Returned: '%s'(%d)\n", + hipGetErrorString(err), err); + } + } + close(fd[0]); + printf("testResult = %d \n", testResult); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + + } else if (cPid > 0) { // parent + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + return testResult; +} + + +bool getPciBusId(int deviceCount, + char **hipDeviceList) { + for (int i = 0; i < deviceCount; i++) { + HIP_CHECK(hipDeviceGetPCIBusId(hipDeviceList[i], MAX_DEVICE_LENGTH, i)); + } + return true; +} +} // namespace hipDeviceGetPCIBusIdTests + + +/** + * Scenario: Validate behavior of hipDeviceGetPCIBusId for masked devices. + */ +TEST_CASE("Unit_hipDeviceGetPCIBusId_MaskedDevices") { + int count = -1; + constexpr int ReqGPUs = 2; + bool ret; + + hipDeviceGetPCIBusIdTests::getDeviceCount(&count); + + if (count >= ReqGPUs) { + ret = hipDeviceGetPCIBusIdTests::testWithMaskedDevices(count); + REQUIRE(ret == true); + } else { + SUCCEED("Not enough GPUs to run the masked GPU tests"); + } +} + +/* Compare {pciDomainID, pciBusID, pciDeviceID} values + * hipDeviceGetPCIBusId vs lspci + */ +TEST_CASE("Unit_hipDeviceGetPCIBusId_CheckPciBusIDWithLspci") { + FILE *fpipe; + { + // Check if lspci is installed, if not, don't proceed + char const *cmd = "lspci --version"; + char *lspciCheck{nullptr}; + constexpr auto MaxLen = 50; + char temp[MaxLen]{}; + + fpipe = popen(cmd, "r"); + REQUIRE_FALSE(fpipe == nullptr); + + lspciCheck = fgets(temp, MaxLen, fpipe); + pclose(fpipe); + + if (lspciCheck == nullptr) { + WARN("Skipping test as lspci is not found in system"); + return; + } + } + + int deviceCount = 0; + HIP_CHECK(hipGetDeviceCount(&deviceCount)); + REQUIRE_FALSE(deviceCount == 0); + // Allocate an array of pointer to characters + char **hipDeviceList = new char*[deviceCount]; + REQUIRE_FALSE(hipDeviceList == nullptr); + char **pciDeviceList = new char*[deviceCount]; + REQUIRE_FALSE(pciDeviceList == nullptr); + for (int i = 0; i < deviceCount; i++) { + hipDeviceList[i] = new char[MAX_DEVICE_LENGTH]; + REQUIRE_FALSE(hipDeviceList[i] == nullptr); + pciDeviceList[i] = new char[MAX_DEVICE_LENGTH]; + REQUIRE_FALSE(pciDeviceList[i] == nullptr); + } + + hipDeviceGetPCIBusIdTests::getPciBusId(deviceCount, hipDeviceList); + char const *command = nullptr; + // Get lspci device list and compare with hip device list + if ((TestContext::get()).isNvidia()) { + command = "lspci -D | grep controller | grep NVIDIA | " + "cut -d ' ' -f 1"; + } else { + command = "lspci -D | grep controller | grep AMD/ATI | " + "cut -d ' ' -f 1"; + } + fpipe = popen(command, "r"); + REQUIRE_FALSE(fpipe == nullptr); + + int index = 0; + int deviceMatchCount = 0; + constexpr auto cmpLen = 10; + while (fgets(pciDeviceList[index], MAX_DEVICE_LENGTH, fpipe)) { + bool bMatchFound = false; + for (int deviceNo = 0; deviceNo < deviceCount; deviceNo++) { + if (!strncasecmp(pciDeviceList[index], hipDeviceList[deviceNo], + cmpLen)) { + deviceMatchCount++; + bMatchFound = true; + } + } + if (bMatchFound == false) { + printf("PCI device: %s is not reported by HIP\n", + pciDeviceList[index]); + } + index++; + if (index >= deviceCount) break; + } + // Deallocate + for (int i = 0; i < deviceCount; i++) { + delete hipDeviceList[i]; + } + delete[] hipDeviceList; + for (int i = 0; i < deviceCount; i++) { + delete pciDeviceList[i]; + } + delete[] pciDeviceList; + pclose(fpipe); + + REQUIRE(deviceMatchCount == deviceCount); +} +#endif diff --git a/projects/hip/tests/catch/multiproc/hipDeviceTotalMemMproc.cc b/projects/hip/tests/catch/multiproc/hipDeviceTotalMemMproc.cc new file mode 100644 index 0000000000..03217d0ee4 --- /dev/null +++ b/projects/hip/tests/catch/multiproc/hipDeviceTotalMemMproc.cc @@ -0,0 +1,161 @@ +/* +Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR +IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/** + * hipDeviceTotalMem tests + * Scenario: Validate behavior of hipDeviceTotalMem for masked devices. + */ + +#include +#ifdef __linux__ +#include +#include + +#define MAX_SIZE 30 +#define VISIBLE_DEVICE 0 + +/** + * Fetches Gpu device count + */ +static void getDeviceCount(int *pdevCnt) { + int fd[2], val = 0; + pid_t childpid; + + // create pipe descriptors + pipe(fd); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); +#endif + + childpid = fork(); + + if (childpid > 0) { // Parent + close(fd[1]); + // parent will wait to read the device cnt + read(fd[0], &val, sizeof(val)); + + // close the read-descriptor + close(fd[0]); + + // wait for child exit + wait(NULL); + + *pdevCnt = val; + } else if (!childpid) { // Child + int devCnt = 1; + // writing only, no need for read-descriptor + close(fd[0]); + + HIP_CHECK(hipGetDeviceCount(&devCnt)); + // send the value on the write-descriptor: + write(fd[1], &devCnt, sizeof(devCnt)); + + // close the write descriptor: + close(fd[1]); + exit(0); + } else { // failure + *pdevCnt = 0; + return; + } +} + + +/** + * Func tries to fetch total memory of masked devices and returns pass/fail. + */ +static bool getTotalMemoryOfMaskedDevices(int actualNumGPUs) { + bool testResult = true; + int fd[2]; + + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + hipError_t err; + char visibleDeviceString[MAX_SIZE] = {}; + snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); + HIP_CHECK(hipInit(0)); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); +#endif + + for (int count = 1; + count < actualNumGPUs; count++) { + size_t totMem; + err = hipDeviceTotalMem(&totMem, count); + if (err == hipSuccess) { + testResult &= false; + } else { + printf("hipDeviceTotalMem: Error Code Returned: '%s'(%d)\n", + hipGetErrorString(err), err); + } + } + close(fd[0]); + printf("testResult = %d \n", testResult); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + + } else if (cPid > 0) { // parent + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + return testResult; +} + + +/** + * Scenario: Validate behavior of hipDeviceTotalMem for masked devices. + */ +TEST_CASE("Unit_hipDeviceTotalMem_MaskedDevices") { + int count = -1; + constexpr int ReqGPUs = 2; + bool ret; + + getDeviceCount(&count); + + if (count >= ReqGPUs) { + ret = getTotalMemoryOfMaskedDevices(count); + REQUIRE(ret == true); + } else { + SUCCEED("Not enough GPUs to run the masked GPU tests"); + } +} + +#endif diff --git a/projects/hip/tests/catch/multiproc/hipGetDeviceAttributeMproc.cc b/projects/hip/tests/catch/multiproc/hipGetDeviceAttributeMproc.cc new file mode 100644 index 0000000000..357deaedc1 --- /dev/null +++ b/projects/hip/tests/catch/multiproc/hipGetDeviceAttributeMproc.cc @@ -0,0 +1,164 @@ +/* +Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/** + * hipGetDeviceAttribute tests + * Scenario: Validate behavior of hipGetDeviceAttribute for masked devices. + */ + +#include +#include +#ifdef __linux__ +#include +#include + +#define MAX_SIZE 30 +#define VISIBLE_DEVICE 0 + +/** + * Fetches Gpu device count + */ +static void getDeviceCount(int *pdevCnt) { + int fd[2], val = 0; + pid_t childpid; + + // create pipe descriptors + pipe(fd); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); +#endif + + childpid = fork(); + + if (childpid > 0) { // Parent + close(fd[1]); + // parent will wait to read the device cnt + read(fd[0], &val, sizeof(val)); + + // close the read-descriptor + close(fd[0]); + + // wait for child exit + wait(NULL); + + *pdevCnt = val; + } else if (!childpid) { // Child + int devCnt = 1; + // writing only, no need for read-descriptor + close(fd[0]); + + HIP_CHECK(hipGetDeviceCount(&devCnt)); + // send the value on the write-descriptor: + write(fd[1], &devCnt, sizeof(devCnt)); + + // close the write descriptor: + close(fd[1]); + exit(0); + } else { // failure + *pdevCnt = 0; + return; + } +} + + +/** + * Tries to fetch device attribute of masked devices and returns pass/fail. + */ +static bool validateGetAttributeOfMaskedDevices(int actualNumGPUs) { + bool testResult = true; + int fd[2]; + + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + hipError_t err; + char visibleDeviceString[MAX_SIZE] = {}; + snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); + HIP_CHECK(hipInit(0)); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); +#endif + + for (int count = 1; + count < actualNumGPUs; count++) { + int pi = -1; + err = hipDeviceGetAttribute(&pi, hipDeviceAttributePciBusId, count); + if (err == hipSuccess) { + testResult &= false; + } else { + printf("hipDeviceGetAttribute: Error Code Returned: '%s'(%d)\n", + hipGetErrorString(err), err); + } + } + close(fd[0]); + printf("testResult = %d \n", testResult); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + + } else if (cPid > 0) { // parent + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + return testResult; +} + + +/** + * Scenario: Validate behavior of hipDeviceGetAttribute for masked devices. + */ +TEST_CASE("Unit_hipDeviceGetAttribute_MaskedDevices") { + int count = -1; + constexpr int ReqGPUs = 2; + bool ret; + + getDeviceCount(&count); + + if (count >= ReqGPUs) { + ret = validateGetAttributeOfMaskedDevices(count); + REQUIRE(ret == true); + } else { + SUCCEED("Not enough GPUs to run the masked GPU tests"); + } +} +#endif diff --git a/projects/hip/tests/catch/multiproc/hipGetDeviceCountMproc.cc b/projects/hip/tests/catch/multiproc/hipGetDeviceCountMproc.cc new file mode 100644 index 0000000000..c1613bd944 --- /dev/null +++ b/projects/hip/tests/catch/multiproc/hipGetDeviceCountMproc.cc @@ -0,0 +1,54 @@ +/* +Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANNTY OF ANY KIND, EXPRESS OR +IMPLIED, INNCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANNY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER INN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR INN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/** + * hipGetDeviceCount tests + * Scenario: Validates the value of numDevices when devices are hidden. + */ + +#include +#ifdef __linux__ +#include +#include + +#define MAX_SIZE 30 +#define VISIBLE_DEVICE 0 + +/** + * Validate behavior of hipGetDeviceCount for masked devices. + */ +TEST_CASE("Unit_hipGetDeviceCount_MaskedDevices") { + int numDevices = 0; + char visibleDeviceString[MAX_SIZE] = {}; + snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE); + +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); +#endif + + HIP_CHECK(hipGetDeviceCount(&numDevices)); + REQUIRE(numDevices == 1); +} +#endif diff --git a/projects/hip/tests/catch/multiproc/hipGetDevicePropertiesMproc.cc b/projects/hip/tests/catch/multiproc/hipGetDevicePropertiesMproc.cc new file mode 100644 index 0000000000..0a74d4b086 --- /dev/null +++ b/projects/hip/tests/catch/multiproc/hipGetDevicePropertiesMproc.cc @@ -0,0 +1,165 @@ +/* +Copyright (c) 2020-Present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/** + * Scenario: Validate behavior of hipGetDeviceProperties for masked devices. + */ + +#include +#ifdef __linux__ +#include +#include + + +#define MAX_SIZE 30 +#define VISIBLE_DEVICE 0 + +/** + * Fetches Gpu device count + */ +static void getDeviceCount(int *pdevCnt) { + int fd[2], val = 0; + pid_t childpid; + + // create pipe descriptors + pipe(fd); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); +#endif + + childpid = fork(); + + if (childpid > 0) { // Parent + close(fd[1]); + // parent will wait to read the device cnt + read(fd[0], &val, sizeof(val)); + + // close the read-descriptor + close(fd[0]); + + // wait for child exit + wait(NULL); + + *pdevCnt = val; + } else if (!childpid) { // Child + int devCnt = 1; + // writing only, no need for read-descriptor + close(fd[0]); + + HIP_CHECK(hipGetDeviceCount(&devCnt)); + // send the value on the write-descriptor: + write(fd[1], &devCnt, sizeof(devCnt)); + + // close the write descriptor: + close(fd[1]); + exit(0); + } else { // failure + *pdevCnt = 0; + return; + } +} + + + +/** + * Tries to fetch device properties of masked devices and returns pass/fail. + */ +static bool validateGetPropsOfMaskedDevices(int actualNumGPUs) { + bool testResult = true; + int fd[2]; + + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + hipError_t err; + char visibleDeviceString[MAX_SIZE] = {}; + snprintf(visibleDeviceString, MAX_SIZE, "%d", VISIBLE_DEVICE); + + // disable visible_devices env from shell +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); + HIP_CHECK(hipInit(0)); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); +#endif + + for (int count = 1; + count < actualNumGPUs; count++) { + hipDeviceProp_t prop; + err = hipGetDeviceProperties(&prop, count); + if (err == hipSuccess) { + testResult &= false; + } else { + printf("hipGetDeviceProperties: Error Code Returned: '%s'(%d)\n", + hipGetErrorString(err), err); + } + } + close(fd[0]); + printf("testResult = %d \n", testResult); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + + } else if (cPid > 0) { // parent + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + return testResult; +} + + + +/** + * Scenario: Validate behavior of hipGetDeviceProperties for masked devices. + */ +TEST_CASE("Unit_hipGetDeviceProperties_MaskedDevices") { + int count = -1; + constexpr int ReqGPUs = 2; + bool ret; + + getDeviceCount(&count); + + if (count >= ReqGPUs) { + ret = validateGetPropsOfMaskedDevices(count); + REQUIRE(ret == true); + } else { + SUCCEED("Not enough GPUs to run the masked GPU tests"); + } +} +#endif diff --git a/projects/hip/tests/catch/multiproc/hipSetGetDeviceMproc.cc b/projects/hip/tests/catch/multiproc/hipSetGetDeviceMproc.cc new file mode 100644 index 0000000000..6e88492446 --- /dev/null +++ b/projects/hip/tests/catch/multiproc/hipSetGetDeviceMproc.cc @@ -0,0 +1,570 @@ +/* + * Copyright (c) 2020-present Advanced Micro Devices, Inc. All rights reserved. + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/* + * Test designed to run on Linux based platforms + * Verifies functionality of + * -- hipSetDevice and hipGetDevice with different ROCR_VISIBLE_DEVICES and + * HIP_VISIBLE_DEVICES values set + */ + +#include + +#ifdef __linux__ +#include +#include + + +#define MAX_SIZE 30 + +/** + * Fetches Gpu device count + */ +static void getDeviceCount(int *pdevCnt) { + int fd[2], val = 0; + pid_t childpid; + + // create pipe descriptors + pipe(fd); + + // disable visible_devices env from shell + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + + + childpid = fork(); + + if (childpid > 0) { // Parent + close(fd[1]); + // parent will wait to read the device cnt + read(fd[0], &val, sizeof(val)); + + // close the read-descriptor + close(fd[0]); + + // wait for child exit + wait(NULL); + + *pdevCnt = val; + } else if (!childpid) { // Child + int devCnt = 1; + // writing only, no need for read-descriptor + close(fd[0]); + + HIP_CHECK(hipGetDeviceCount(&devCnt)); + + // send the value on the write-descriptor: + write(fd[1], &devCnt, sizeof(devCnt)); + + // close the write descriptor: + close(fd[1]); + exit(0); + } else { // failure + *pdevCnt = 0; + } +} + + +// Pass either -1 in deviceNumber or invalid device number +static void testInvalidDevice(int numDevices, bool useRocrEnv, + int deviceNumber) { + bool testResult = true; + int device; + int tempCount = 0; + int setDeviceErrorCheck = 0; + int getDeviceErrorCheck = 0; + int getDeviceCountErrorCheck = 0; + int fd[2]; + pipe(fd); + + pid_t cPid; + cPid = fork(); + + char visibleDeviceString[MAX_SIZE] = {}; + snprintf(visibleDeviceString, MAX_SIZE, "%d", deviceNumber); + + if (cPid == 0) { // child + hipError_t err; +#ifdef __HIP_PLATFORM_NVCC__ + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); +#else + if (true == useRocrEnv) { + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + } else { + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); + } +#endif + err = hipGetDeviceCount(&tempCount); + if (err != hipSuccess) { + getDeviceCountErrorCheck = 1; + } + for (int i = 0; i < numDevices; i++) { + err = hipSetDevice(i); + if (err != hipSuccess) { + setDeviceErrorCheck+= 1; + } + + err = hipGetDevice(&device); + if (err != hipSuccess) { + getDeviceErrorCheck+= 1; + } + } + + if ((getDeviceCountErrorCheck == 1) && (setDeviceErrorCheck == numDevices) + && (getDeviceErrorCheck == numDevices)) { + testResult = true; + + } else { + printf("Test failed for invalid device, getDeviceCountErrorCheck %d," + "setDeviceErrorCheck %d, getDeviceErrorCheck %d\n", + getDeviceCountErrorCheck, setDeviceErrorCheck, + getDeviceErrorCheck); + + testResult = false; + } + + close(fd[0]); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + + } else if (cPid > 0) { // parent + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + REQUIRE(testResult == true); +} + + +static void testValidDevices(int numDevices, bool useRocrEnv, int *deviceList, + int deviceListLength) { + bool testResult = true; + int tempCount = 0; + int device; + int setDeviceErrorCheck = 0; + int getDeviceErrorCheck = 0; + int getDeviceCountErrorCheck = 0; + int *deviceListPtr = deviceList; + char visibleDeviceString[MAX_SIZE] = {}; + + if ((NULL == deviceList) || ((deviceListLength < 1) || + deviceListLength > numDevices)) { + INFO("Invalid argument for number of devices. Skipping current test"); + REQUIRE(false); + } + + for (int i = 0; i < deviceListLength; i++) { + snprintf(visibleDeviceString + strlen(visibleDeviceString), MAX_SIZE, "%d,", + *deviceListPtr++); + } + + visibleDeviceString[strlen(visibleDeviceString)-1] = 0; + + int fd[2]; + pipe(fd); + + pid_t cPid; + cPid = fork(); + + if (cPid == 0) { +#ifdef __HIP_PLATFORM_NVCC__ + unsetenv("CUDA_VISIBLE_DEVICES"); + setenv("CUDA_VISIBLE_DEVICES", visibleDeviceString, 1); +#else + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + if (true == useRocrEnv) { + setenv("ROCR_VISIBLE_DEVICES", visibleDeviceString, 1); + } else { + setenv("HIP_VISIBLE_DEVICES", visibleDeviceString, 1); + } +#endif + + + hipError_t err; + err = hipGetDeviceCount(&tempCount); + + if (tempCount == deviceListLength) { + getDeviceCountErrorCheck = 1; + } else { + printf("hipGetDeviceCount failed. return value: %u\n", hipError_t(err)); + } + + for (int i = 0; i < numDevices; i++) { + err = hipSetDevice(i); + if (err != hipSuccess) { + setDeviceErrorCheck+= 1; + } + + err = hipGetDevice(&device); + if (err != hipSuccess) { + getDeviceErrorCheck+= 1; + } + } + + if ((getDeviceCountErrorCheck == 1) && (setDeviceErrorCheck == + (numDevices-deviceListLength)) && (getDeviceErrorCheck == 0)) { + testResult = true; + + } else { + printf("Test failed for device count %d\n", deviceListLength); + testResult = false; + } + + close(fd[0]); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + + } else if (cPid > 0) { + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + REQUIRE(testResult == true); +} + + +static void Initialize(int *deviceList, int numDevices, int count, + char min_visibleDeviceString[], char max_visibleDeviceString[]) { + int *deviceListPtr = deviceList; + for (int i =0; i < count; i++) { + if (i == count-1) { + snprintf(min_visibleDeviceString + strlen(min_visibleDeviceString), + MAX_SIZE, "%d", *deviceListPtr++); + } else { + snprintf(min_visibleDeviceString + strlen(min_visibleDeviceString), + MAX_SIZE, "%d,", *deviceListPtr++); + } + } + for (int i =0; i < numDevices; i++) { + if (i == numDevices-1) { + snprintf(max_visibleDeviceString + strlen(max_visibleDeviceString), + MAX_SIZE, "%d", i); + } else { + snprintf(max_visibleDeviceString + strlen(max_visibleDeviceString), + MAX_SIZE, "%d,", i); + } + } +} + +static void testMaxRvdMinHvd(int numDevices, int *deviceList, int count) { + bool testResult = true; + int device; + int validateCount = 0; + char min_visibleDeviceString[MAX_SIZE] = {0}; + char max_visibleDeviceString[MAX_SIZE] = {0}; + int fd[2]; + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + Initialize(deviceList, numDevices, + count, min_visibleDeviceString, max_visibleDeviceString); + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", max_visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", min_visibleDeviceString, 1); + HIP_CHECK(hipGetDeviceCount(&numDevices)); + for (int i = 0; i < numDevices; i++) { + HIP_CHECK(hipSetDevice(i)); + HIP_CHECK(hipGetDevice(&device)); + if (device == i) { + validateCount+= 1; + } + } + if (count != validateCount) { + testResult = false; + } + } else if (cPid > 0) { + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + REQUIRE(testResult == true); +} + +static void testRvdCvd(int numDevices, int *deviceList, int count) { + bool testResult = true; + int device; + int validateCount = 0; + char min_visibleDeviceString[MAX_SIZE] = {0}; + char max_visibleDeviceString[MAX_SIZE] = {0}; + int fd[2]; + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + Initialize(deviceList, numDevices, count, + min_visibleDeviceString, max_visibleDeviceString); + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", max_visibleDeviceString, 1); + setenv("CUDA_VISIBLE_DEVICES", min_visibleDeviceString, 1); + HIP_CHECK(hipGetDeviceCount(&numDevices)); + for (int i = 0; i < numDevices; i++) { + HIP_CHECK(hipSetDevice(i)); + HIP_CHECK(hipGetDevice(&device)); + if (device == i) { + validateCount+= 1; + } + } + if (count != validateCount) { + testResult = false; + } + } else if (cPid > 0) { + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + REQUIRE(testResult == true); +} + +static void testMinRvdMaxHvd(int numDevices, int *deviceList, int count) { + bool testResult = true; + int device; + int validateCount = 0; + char min_visibleDeviceString[MAX_SIZE] = {0}; + char max_visibleDeviceString[MAX_SIZE] = {0}; + int fd[2]; + pipe(fd); + pid_t cPid; + cPid = fork(); + if (cPid == 0) { // child + Initialize(deviceList, numDevices, count, + min_visibleDeviceString, max_visibleDeviceString); + unsetenv("ROCR_VISIBLE_DEVICES"); + unsetenv("HIP_VISIBLE_DEVICES"); + setenv("ROCR_VISIBLE_DEVICES", min_visibleDeviceString, 1); + setenv("HIP_VISIBLE_DEVICES", max_visibleDeviceString, 1); + HIP_CHECK(hipGetDeviceCount(&numDevices)); + for (int i = 0; i < numDevices; i++) { + HIP_CHECK(hipSetDevice(i)); + HIP_CHECK(hipGetDevice(&device)); + if (device == i) { + validateCount+= 1; + } + } + if (count != validateCount) { + testResult = false; + } + close(fd[0]); + write(fd[1], &testResult, sizeof(testResult)); + close(fd[1]); + exit(0); + } else if (cPid > 0) { + close(fd[1]); + read(fd[0], &testResult, sizeof(testResult)); + close(fd[0]); + wait(NULL); + } else { + printf("fork() failed\n"); + HIP_ASSERT(false); + } + + REQUIRE(testResult == true); +} + +/** + * Scenario sets Invalid visible device list and checks behavior. + */ +TEST_CASE("Unit_hipSetDevice_InvalidVisibleDeviceList") { + int numDevices = 0; + + getDeviceCount(&numDevices); + REQUIRE(numDevices != 0); + + SECTION("Test setting -1 to HIP_VISIBLE_DEVICES") { + testInvalidDevice(numDevices, false, -1); + } + + SECTION("Test setting invalid device to HIP_VISIBLE_DEVICES") { + testInvalidDevice(numDevices, false, numDevices); + } +#ifndef __HIP_PLATFORM_NVCC__ + SECTION("Test setting -1 to ROCR_VISIBLE_DEVICES") { + testInvalidDevice(numDevices, true, -1); + } + + SECTION("Test setting invalid device to ROCR_VISIBLE_DEVICES") { + testInvalidDevice(numDevices, true, numDevices); + } +#endif +} + +/** + * Scenario sets valid visible device list and checks behavior. + */ +TEST_CASE("Unit_hipSetDevice_ValidVisibleDeviceList") { + int numDevices = 0; + int deviceList[MAX_SIZE]; + + getDeviceCount(&numDevices); + REQUIRE(numDevices != 0); + + // Test for all available devices + for (int i = 0; i < numDevices; i++) { + deviceList[i] = i; + } + + SECTION("Test setting valid hip visible device list") { + testValidDevices(numDevices, false, deviceList, numDevices); + } +#ifndef __HIP_PLATFORM_NVCC__ + SECTION("Test setting valid rocr visible device list") { + testValidDevices(numDevices, true, deviceList, numDevices); + } +#endif +} + +/** + * Scenario sets subset of available devices and checks behavior. + */ +TEST_CASE("Unit_hipSetDevice_SubsetOfAvailableDevices") { + int numDevices = 0; + int deviceList[MAX_SIZE]; + int deviceListLength = 1; + + getDeviceCount(&numDevices); + REQUIRE(numDevices != 0); + + // Test for subset of available gpus + for (int i=0; i < deviceListLength; i++) { + deviceList[i] = deviceListLength-1-i; + } + +#ifndef __HIP_PLATFORM_NVCC__ + testValidDevices(numDevices, true, deviceList, + deviceListLength); +#endif + testValidDevices(numDevices, false, deviceList, + deviceListLength); +} + +#ifndef __HIP_PLATFORM_NVCC__ +/* Following tests apply only for AMD Platforms */ + +/** + * Scenario tests getDevice behavior with Minimal Len of RVD + * and Maximal Len of HVD + */ +TEST_CASE("Unit_hipSetDevice_MinRvdMaxHvdDevicesList") { + int numDevices = 0; + int deviceList[MAX_SIZE]; + int count = 0; + + getDeviceCount(&numDevices); + + REQUIRE(numDevices != 0); + + if (numDevices == 1) { + deviceList[0] = 0; + count = 1; + } else { + for (int i=0; i < numDevices; i++) { + if (i%2 == 0) { + deviceList[count] = i; + count++; + } + } + } + + testMinRvdMaxHvd(numDevices, deviceList, count); +} + +/** + * Scenario tests getDevice behavior with Maximal Len of RVD + * and Minimal Len of HVD + */ +TEST_CASE("Unit_hipSetDevice_MaxRvdMinHvdDevicesList") { + int numDevices = 0; + int deviceList[MAX_SIZE]; + int count = 0; + + getDeviceCount(&numDevices); + + REQUIRE(numDevices != 0); + + if (numDevices == 1) { + deviceList[0] = 0; + count = 1; + } else { + for (int i=0; i < numDevices; i++) { + if (i%2 == 0) { + deviceList[count] = i; + count++; + } + } + } + + testMaxRvdMinHvd(numDevices, deviceList, count); +} + +/** + * Scenario tests getDevice behavior with combination of RVD and CVD + */ +TEST_CASE("Unit_hipSetDevice_RvdCvdDevicesList") { + int numDevices = 0; + int deviceList[MAX_SIZE]; + int count = 0; + + getDeviceCount(&numDevices); + + REQUIRE(numDevices != 0); + + if (numDevices == 1) { + deviceList[0] = 0; + count = 1; + } else { + for (int i=0; i < numDevices; i++) { + if (i%2 == 0) { + deviceList[count] = i; + count++; + } + } + } + + testRvdCvd(numDevices, deviceList, count); +} +#endif // __HIP_PLATFORM_NVCC__ + +#endif // __linux__