Test cases for HIP_VISIBLE_DEVICES/CUDA_VISIBLE_DEVICES.

hipEnvVar is the base test case, to be called by hipEnvVarDriver
at the run time.
Test case includes tests for normal use case of the environment
variable, invalid value/sequence and use CUDA_VISIBLE_DEVICES as a
alternative.

hipamd/src/hip_hcc.cpp

@@ -300,7 +300,7 @@ unsigned g_deviceCnt;
 //Forward Declarations:
 //=================================================================================================
 INLINE bool ihipIsValidDevice(unsigned deviceIndex);
-INLINE bool ihipIsVisibleDevice(unsigned deviceIndex);
+
 //=================================================================================================
 // Implementation:
 //=================================================================================================
@@ -967,25 +967,23 @@ void ihipInit()
     for (int i=0; i<accs.size(); i++) {
         // check if the device id is included in the HIP_VISIBLE_DEVICES env variable
         if (! accs[i].get_is_emulated()) {
-            //if (ihipIsVisibleDevice(i-1) && !g_visible_device)
+            //if (std::find(g_hip_visible_devices.begin(), g_hip_visible_devices.end(), (i-1)) == g_hip_visible_devices.end() && g_visible_device)
-            if (ihipIsVisibleDevice(i-1))
+            if (std::find(g_hip_visible_devices.begin(), g_hip_visible_devices.end(), (i-1)) == g_hip_visible_devices.end() && g_visible_device)
             {
-                //Ignore if the device is not in visible devices list
+                //If device is not in visible devices list, ignore
                 continue;
             }
-            //std::cout << "The visible GPU number is " << i << std::endl;
             g_devices[g_deviceCnt].init(g_deviceCnt, accs[i]);
-            //std::cout << "index of g_devices = "<< g_deviceCnt << std::endl;
             g_deviceCnt++;
         }
     }
-    //std::cout << "g_deviceCnt = " << g_deviceCnt << std::endl;
 
     // If HIP_VISIBLE_DEVICES is not set, make sure all devices are initialized
     if(!g_visible_device)
         assert(deviceCnt == g_deviceCnt);
 
     tprintf(TRACE_API, "pid=%u %-30s\n", getpid(), "<ihipInit>");
+
 }
 
 INLINE bool ihipIsValidDevice(unsigned deviceIndex)
@@ -994,12 +992,12 @@ INLINE bool ihipIsValidDevice(unsigned deviceIndex)
     return (deviceIndex < g_deviceCnt);
 }
 
-// check if the device ID is set as visible
+/*// check if the device ID is set as visible*/
-INLINE bool ihipIsVisibleDevice(unsigned deviceIndex)
+//INLINE bool ihipIsVisibleDevice(unsigned deviceIndex)
-{
+//{
-    return std::find(g_hip_visible_devices.begin(), g_hip_visible_devices.end(),
+    //return std::find(g_hip_visible_devices.begin(), g_hip_visible_devices.end(),
-            (int)deviceIndex) != g_hip_visible_devices.end();
+            //(int)deviceIndex) != g_hip_visible_devices.end();
-}
+/*}*/
 
 //---
 INLINE ihipDevice_t *ihipGetTlsDefaultDevice()

hipamd/tests/src/CMakeLists.txt

@@ -110,6 +110,8 @@ make_hip_executable (hip_clz hip_clz.cpp)
 make_hip_executable (hip_brev hip_brev.cpp) 
 make_hip_executable (hip_ffs hip_ffs.cpp) 
 make_hip_executable (hipGetDeviceAttribute hipGetDeviceAttribute.cpp) 
+make_hip_executable (hipEnvVar hipEnvVar.cpp) 
+make_hip_executable (hipEnvVarDriver hipEnvVarDriver.cpp) 
 make_hip_executable (hipMemcpy hipMemcpy.cpp) 
 make_hip_executable (hipMemcpyAsync hipMemcpyAsync.cpp) 
 make_hip_executable (hipMemset hipMemset.cpp) 
@@ -137,6 +139,7 @@ make_test(hipMemset --N 10    --memsetval 0x42 )  # small copy, just 10 bytes.
 make_test(hipMemset --N 10013 --memsetval 0x5a )  # oddball size.
 make_test(hipMemset --N 256M  --memsetval 0xa6 )  # big copy
 make_test(hipGridLaunch " " )
+make_test(hipEnvVarDriver " " )
 make_test(hipPointerAttrib " " )
 make_test(hipMultiThreadStreams " " )
 

hipamd/tests/src/hipEnvVar.cpp

@@ -28,24 +28,27 @@ THE SOFTWARE.
 #include <string>
 #include <hip_runtime.h>
 
-int debug = 0;
+using namespace std;
 
 void usage() {
     printf("hipEnvVar [otpions]\n\
-    -a,\t\ttotal number ofavailable GPUs and their pciBusID\n\
+    -c,\t\ttotal number ofavailable GPUs and their pciBusID\n\
-    -s,\t\tselect one GPU and return its pciBusID\n\
+    -d,\t\tselect one GPU and return its pciBusID\n\
+    -v,\t\tsend the list to HIP_VISIBLE_DEVICES env var\n\
     -h,\t\tshow this help message\n\
     ");
 }
 int main(int argc, char **argv)
 {
+   //string str = getenv("HIP_VISIBLE_DEVICES");
+    //std::cout << "The current env HIP_VISIBLE_DEVICES is"<<str << std::endl;
     extern char *optarg;
     extern int optind;
     int c, err = 0;
-    int retDevCnt=0, retDevInfo=0;
+    int retDevCnt=0, retDevInfo=0, setEnvVar=0;
     int device=0;
-    //std::cout << "reach here!!" << std::endl;
+    string env;
-    while ((c = getopt(argc, argv, "cd:h")) != -1)
+    while ((c = getopt(argc, argv, "cd:v:h")) != -1)
         switch (c) {
         case 'c':
             retDevCnt = true;
@@ -54,6 +57,10 @@ int main(int argc, char **argv)
             retDevInfo = true;
             device = atoi(optarg);
             break;
+        case 'v':
+            setEnvVar = true;
+            env = optarg;
+            break;
         case 'h':
             usage();
             return 0;
@@ -66,6 +73,21 @@ int main(int argc, char **argv)
             err = 1;
             break;
         }
+
+    if (setEnvVar ) {
+        //env = "export HIP_VISIBLE_DEVICES=" + env;
+        //cout<<"The received env var is: "<<env<<endl;
+        setenv("HIP_VISIBLE_DEVICES",env.c_str(),1);
+        cout<<"set env HIP_VISIBLE_DEVICES = "<< env.c_str()<<endl;
+        //verify if the environment variable is set
+        char* pPath;
+        pPath = getenv ("HIP_VISIBLE_DEVICES");
+        if(pPath!=NULL)
+            printf("HIP_VISIBLE_DEVICES is %s\n", pPath);
+        else
+            printf("HIP_VISIBLE_DEVICES is not set\n");
+    }
+
     // device init
     int devCount=0;
     hipGetDeviceCount(&devCount);
@@ -81,8 +103,10 @@ int main(int argc, char **argv)
                device, 0, devCount -1);
         return -1;
     }
+
     if (retDevCnt) {
-        std::cout << "Total number of devices visible in system is "<< devCount  << std::endl;
+        //std::cout << "Total number of devices visible in system is "<< devCount  << std::endl;
+        std::cout << devCount  << std::endl;
     }
     if (retDevInfo) {
         hipSetDevice(device);
@@ -90,13 +114,11 @@ int main(int argc, char **argv)
 
         hipDeviceGetProperties(&devProp, device);
         if (devProp.major < 1) {
-            printf("Device %d does not support HIP\n", device);
+            printf("%d does not support HIP\n", device);
             return -1;
         }
-
+        std::cout << devProp.pciBusID << std::endl;
-        std::cout << "The selected device pciBusID is " << devProp.pciBusID << std::endl;
     }
-
     exit(0);
 }
 

hipamd/tests/src/hipEnvVarDriver.cpp

@@ -13,26 +13,91 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
 EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
-THE USE OR OTHER DEALINGS IN TcHE SOFTWARE. */
+THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
 
 #include <iostream>
+#include <vector>
 #include <stdio.h>
-
+#include <stdlib.h>
+#include <assert.h>
 using namespace std;
 
-int main() {
+//./hipEnvVar -c -d 0 -h
-    FILE *in;
+  //putenv("SomeVariable=SomeValue");
-    char buff[512];
+ //putenv("export HIP_VISIBLE_DEVICES=0,1,2,3");
 
-    if(!(in = popen("ls -sail", "r"))){
+int getDeviceNumber(){
+   FILE *in;
+   char buff[512];
+   string str;
+   if(!(in = popen("./hipEnvVar -c", "r"))){
         return 1;
     }
-
+    fgets(buff, sizeof(buff), in);
-    while(fgets(buff, sizeof(buff), in)!=NULL){
-        cout << buff;
-    }
     pclose(in);
-
+    return atoi(buff);
-    return 0;
+}
-
+
+int getDevicePCIBusNum(int deviceID){
+    FILE *in;
+    char buff[512];
+    string str = "./hipEnvVar -d ";
+    str += std::to_string(deviceID);
+    if(!(in = popen(str.c_str(), "r"))){
+        return 1;
+    }
+    fgets(buff, sizeof(buff), in);
+    pclose(in);
+    return atoi(buff);
+}
+
+int main() {
+    unsetenv("HIP_VISIBLE_DEVICES");
+    //collect the device pci bus ID for all devices
+    int totalDeviceNum = getDeviceNumber();
+    std::cout << "The total number of available devices is " << totalDeviceNum<< std::endl
+        <<"Valid index range is 0 - "<<totalDeviceNum-1<<std::endl;
+    std::vector<int> devPCINum;
+    for (int i = 0; i < totalDeviceNum ; i++) {
+        devPCINum.push_back(getDevicePCIBusNum(i));
+        std::cout <<"The collected device PCI Bus ID of Device "<<i<<" is "
+            << getDevicePCIBusNum(i) << std::endl;
+    }
+
+    //select each of the available devices to be the target device,
+    //query the returned device pci bus number, check if match the database
+    for (int i = 0; i < totalDeviceNum ; i++) {
+        setenv("HIP_VISIBLE_DEVICES",(char*)std::to_string(i).c_str(),1);
+        //cout<<"HIP_VISIBLE_DEVICES is "<<i<<" data in vector is "<<devPCINum[i]<<endl;
+        //std::cout <<"Returned pci number is"<< getDevicePCIBusNum(0) << std::endl;
+        if (devPCINum[i] != getDevicePCIBusNum(0)) {
+            std::cout << "The returned PciBusID is not correct"
+                << std::endl;
+            exit(-1);
+        } else {
+            continue;
+        }
+    }
+
+    //check when set an invalid device number
+    setenv("HIP_VISIBLE_DEVICES","1000,0,1",1);
+    assert(getDeviceNumber() == 0);
+
+    if(totalDeviceNum > 2){
+        setenv("HIP_VISIBLE_DEVICES","0,1,1000,2",1);
+        assert(getDeviceNumber() == 2);
+
+        setenv("HIP_VISIBLE_DEVICES","0,1,2",1);
+        assert(getDeviceNumber() == 3);
+        // test if CUDA_VISIBLE_DEVICES will be accepted by the runtime
+        unsetenv("HIP_VISIBLE_DEVICES");
+        setenv("CUDA_VISIBLE_DEVICES","0,1,2",1);
+        assert(getDeviceNumber() == 3);
+    }
+
+    setenv("HIP_VISIBLE_DEVICES","-100,0,1",1);
+    assert(getDeviceNumber() == 0);
+
+    std::cout << "Passed!" << std::endl;
+    return 0;
 }