P4 to Git Change 1595124 by skudchad@skudchad_test2_win_opencl on 2018/08/17 14:05:25

SWDEV-145570 - [HIP] Implement environment variables and subsequent changes for HIP. This gets hipEnvVar passing

	ReviewBoardURL = http://ocltc.amd.com/reviews/r/15641/diff/

Affected files ...

... //depot/stg/opencl/drivers/opencl/api/hip/hip_device.cpp#15 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.cpp#224 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#310 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#104 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#95 edit
... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#297 edit


[ROCm/clr commit: 812ec48f9e]

projects/clr/rocclr/runtime/device/device.cpp

@@ -224,6 +224,8 @@ Device::~Device() {
   }
 }
 
+bool amd::Device::validOrdinal_ = true;
+
 bool Device::create() {
   vaCacheAccess_ = new amd::Monitor("VA Cache Ops Lock", true);
   if (NULL == vaCacheAccess_) {

projects/clr/rocclr/runtime/device/device.hpp

@@ -1591,6 +1591,13 @@ class Device : public RuntimeObject {
   // P2P devices that are accessible from the current device
   std::vector<cl_device_id> p2pDevices_;
 
+  //! Check for invalid ordinal passed via environment var
+  bool isOrdinalValid() { return validOrdinal_; }
+
+  //! Write to validOrdinal_
+  static void setvalidOrdinal(bool val) { validOrdinal_ = val; }
+
+
  protected:
   //! Enable the specified extension
   char* getExtensionString();
@@ -1601,6 +1608,7 @@ class Device : public RuntimeObject {
   BlitProgram* blitProgram_;      //!< Blit program info
   static AppProfile appProfile_;  //!< application profile
   HwDebugManager* hwDebugMgr_;    //!< Hardware Debug manager
+  static bool validOrdinal_;      //!< Valid Ordinal
 
  private:
   bool IsTypeMatching(cl_device_type type, bool offlineDevices);

projects/clr/rocclr/runtime/device/pal/paldevice.cpp

@@ -1092,9 +1092,13 @@ device::Program* Device::createProgram(amd::option::Options* options) {
 typedef std::unordered_map<int, bool> requestedDevices_t;
 
 //! Parses the requested list of devices to be exposed to the user.
-static void parseRequestedDeviceList(requestedDevices_t& requestedDevices) {
+static void parseRequestedDeviceList(requestedDevices_t& requestedDevices,
+                                     uint32_t numDevices) {
   int requestedDeviceCount = 0;
-  const char* requestedDeviceList = GPU_DEVICE_ORDINAL;
+
+  const char* requestedDeviceList = IS_HIP ? ((HIP_VISIBLE_DEVICES[0] != '\0') ?
+                                    HIP_VISIBLE_DEVICES : CUDA_VISIBLE_DEVICES)
+                                    : GPU_DEVICE_ORDINAL;
 
   char* pch = strtok(const_cast<char*>(requestedDeviceList), ",");
   while (pch != nullptr) {
@@ -1107,17 +1111,27 @@ static void parseRequestedDeviceList(requestedDevices_t& requestedDevices) {
         break;
       }
     }
-    if (currentDeviceIndex < 0) {
+    if (currentDeviceIndex < 0 ||
+      static_cast<uint32_t>(currentDeviceIndex) >= numDevices) {
       deviceIdValid = false;
     }
     // Get next token.
     pch = strtok(nullptr, ",");
-    if (!deviceIdValid) {
-      continue;
+
+    // FIXME Allow atleast one valid deviceId so compilation etc doesnt break
+    // but set validOrdinal_ flag
+    if (!deviceIdValid && (requestedDevices.size() != 0)) {
+      // Exit the loop as anything to the right of invalid deviceId
+      // has to be discarded unless its the first one
+      break;
+    }
+
+    if (!deviceIdValid && (requestedDevices.size() == 0)) {
+      Device::setvalidOrdinal(false);
     }
 
     // Requested device is valid.
-    requestedDevices[currentDeviceIndex] = true;
+    requestedDevices[currentDeviceIndex] = deviceIdValid;
   }
 }
 
@@ -1182,14 +1196,21 @@ bool Device::init() {
 
   uint ordinal = 0;
   const char* selectDeviceByName = nullptr;
-  if (!flagIsDefault(GPU_DEVICE_ORDINAL)) {
+
+  if (IS_HIP) {
+    if (HIP_VISIBLE_DEVICES[0] != '\0' || CUDA_VISIBLE_DEVICES[0] != '\0') {
+      useDeviceList = true;
+      parseRequestedDeviceList(requestedDevices, numDevices);
+    }
+  } else if (GPU_DEVICE_ORDINAL[0] != '\0') {
     useDeviceList = true;
-    parseRequestedDeviceList(requestedDevices);
+    parseRequestedDeviceList(requestedDevices, numDevices);
   } else if (!flagIsDefault(GPU_DEVICE_NAME)) {
     selectDeviceByName = GPU_DEVICE_NAME;
   }
 
   bool foundDevice = false;
+
   // Loop through all active devices and initialize the device info structure
   for (; ordinal < numDevices; ++ordinal) {
     // Create the GPU device object
@@ -1197,6 +1218,10 @@ bool Device::init() {
     bool result = (nullptr != d) && d->create(deviceList[ordinal]);
     if (useDeviceList) {
       result &= (requestedDevices.find(ordinal) != requestedDevices.end());
+      if (!result) {
+        delete d;
+        break;
+      }
     }
     if (result && ((nullptr == selectDeviceByName) || ('\0' == selectDeviceByName[0]) ||
                    (strstr(selectDeviceByName, d->info().name_) != nullptr))) {

projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp

@@ -403,19 +403,37 @@ bool Device::init() {
     return false;
   }
 
-  std::vector<bool> selectedDevices;
-  selectedDevices.resize(gpu_agents_.size(), true);
+  std::unordered_map<int, bool> selectedDevices;
+  bool useDeviceList = false;
 
-  if (!flagIsDefault(GPU_DEVICE_ORDINAL)) {
-    std::fill(selectedDevices.begin(), selectedDevices.end(), false);
+  if ((GPU_DEVICE_ORDINAL != '\0') || (HIP_VISIBLE_DEVICES[0] != '\0')
+      || (CUDA_VISIBLE_DEVICES != '\0')) {
+    useDeviceList = true;
+    std::string ordinals = IS_HIP ? ((HIP_VISIBLE_DEVICES[0] != '\0') ?
+                           HIP_VISIBLE_DEVICES : CUDA_VISIBLE_DEVICES)
+                           : GPU_DEVICE_ORDINAL;
 
-    std::string ordinals(GPU_DEVICE_ORDINAL);
     size_t end, pos = 0;
     do {
+      bool deviceIdValid = true;
       end = ordinals.find_first_of(',', pos);
-      size_t index = atoi(ordinals.substr(pos, end - pos).c_str());
-      selectedDevices.resize(index + 1);
-      selectedDevices[index] = true;
+      int index = atoi(ordinals.substr(pos, end - pos).c_str());
+      if (index < 0 || static_cast<size_t>(index) >= gpu_agents_.size()) {
+        deviceIdValid = false;
+      }
+      // FIXME Allow atleast one valid deviceId so compilation etc doesnt break
+      // but set validOrdinal_ flag
+      if (!deviceIdValid && (selectedDevices.size() != 0)) {
+        // Exit the loop as anything to the right of invalid deviceId
+        // has to be discarded unless its the first one
+        break;
+      }
+
+      if (!deviceIdValid && (selectedDevices.size() == 0)) {
+        Device::setvalidOrdinal(false);
+      }
+
+      selectedDevices[index] = deviceIdValid;
       pos = end + 1;
     } while (end != std::string::npos);
   }
@@ -538,10 +556,13 @@ bool Device::init() {
       }
     }
 
-    if (selectedDevices[ordinal++] &&
-        (flagIsDefault(GPU_DEVICE_NAME) || GPU_DEVICE_NAME == 0 || GPU_DEVICE_NAME[0] == '\0' ||
-         !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) {
-      roc_device.release()->registerDevice();
+    if (!useDeviceList && (flagIsDefault(GPU_DEVICE_NAME) || GPU_DEVICE_NAME == 0
+        || GPU_DEVICE_NAME[0] == '\0' || !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) {
+        roc_device.release()->registerDevice();
+    } else if (useDeviceList && selectedDevices[ordinal++]) {
+        roc_device.release()->registerDevice();
+    } else {
+      break;
     }
   }
 

projects/clr/rocclr/runtime/utils/flags.hpp

@@ -205,7 +205,10 @@ release(bool, GPU_FORCE_WAVE_SIZE_32, false,                                  \
         "Forces WaveSize32 compilation in SC")                                \
 release(uint, GPU_MAX_COMMAND_BUFFERS, 8,                                     \
          "The maximum number of command buffers allocated per queue")         \
-
+release(cstring, HIP_VISIBLE_DEVICES, "",                                             \
+        "Only devices whose index is present in the sequence are visible to HIP")    \
+release(cstring, CUDA_VISIBLE_DEVICES, "",                                            \
+        "Only devices whose index is present in the sequence are visible to HIP")    \
 
 namespace amd {