From 812ec48f9e1b5b27c1b3e7d8d52502062796c59d Mon Sep 17 00:00:00 2001 From: foreman Date: Fri, 17 Aug 2018 14:28:54 -0400 Subject: [PATCH] P4 to Git Change 1595124 by skudchad@skudchad_test2_win_opencl on 2018/08/17 14:05:25 SWDEV-145570 - [HIP] Implement environment variables and subsequent changes for HIP. This gets hipEnvVar passing ReviewBoardURL = http://ocltc.amd.com/reviews/r/15641/diff/ Affected files ... ... //depot/stg/opencl/drivers/opencl/api/hip/hip_device.cpp#15 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.cpp#224 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/device.hpp#310 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#104 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#95 edit ... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#297 edit --- rocclr/runtime/device/device.cpp | 2 ++ rocclr/runtime/device/device.hpp | 8 +++++ rocclr/runtime/device/pal/paldevice.cpp | 41 ++++++++++++++++----- rocclr/runtime/device/rocm/rocdevice.cpp | 45 +++++++++++++++++------- rocclr/runtime/utils/flags.hpp | 5 ++- 5 files changed, 80 insertions(+), 21 deletions(-) diff --git a/rocclr/runtime/device/device.cpp b/rocclr/runtime/device/device.cpp index a40cd8c94d..bfb7e23bda 100644 --- a/rocclr/runtime/device/device.cpp +++ b/rocclr/runtime/device/device.cpp @@ -224,6 +224,8 @@ Device::~Device() { } } +bool amd::Device::validOrdinal_ = true; + bool Device::create() { vaCacheAccess_ = new amd::Monitor("VA Cache Ops Lock", true); if (NULL == vaCacheAccess_) { diff --git a/rocclr/runtime/device/device.hpp b/rocclr/runtime/device/device.hpp index 3c1c8813b4..58ffdc4b5f 100644 --- a/rocclr/runtime/device/device.hpp +++ b/rocclr/runtime/device/device.hpp @@ -1591,6 +1591,13 @@ class Device : public RuntimeObject { // P2P devices that are accessible from the current device std::vector p2pDevices_; + //! Check for invalid ordinal passed via environment var + bool isOrdinalValid() { return validOrdinal_; } + + //! Write to validOrdinal_ + static void setvalidOrdinal(bool val) { validOrdinal_ = val; } + + protected: //! Enable the specified extension char* getExtensionString(); @@ -1601,6 +1608,7 @@ class Device : public RuntimeObject { BlitProgram* blitProgram_; //!< Blit program info static AppProfile appProfile_; //!< application profile HwDebugManager* hwDebugMgr_; //!< Hardware Debug manager + static bool validOrdinal_; //!< Valid Ordinal private: bool IsTypeMatching(cl_device_type type, bool offlineDevices); diff --git a/rocclr/runtime/device/pal/paldevice.cpp b/rocclr/runtime/device/pal/paldevice.cpp index 5770a50952..1f050f8fdb 100644 --- a/rocclr/runtime/device/pal/paldevice.cpp +++ b/rocclr/runtime/device/pal/paldevice.cpp @@ -1092,9 +1092,13 @@ device::Program* Device::createProgram(amd::option::Options* options) { typedef std::unordered_map requestedDevices_t; //! Parses the requested list of devices to be exposed to the user. -static void parseRequestedDeviceList(requestedDevices_t& requestedDevices) { +static void parseRequestedDeviceList(requestedDevices_t& requestedDevices, + uint32_t numDevices) { int requestedDeviceCount = 0; - const char* requestedDeviceList = GPU_DEVICE_ORDINAL; + + const char* requestedDeviceList = IS_HIP ? ((HIP_VISIBLE_DEVICES[0] != '\0') ? + HIP_VISIBLE_DEVICES : CUDA_VISIBLE_DEVICES) + : GPU_DEVICE_ORDINAL; char* pch = strtok(const_cast(requestedDeviceList), ","); while (pch != nullptr) { @@ -1107,17 +1111,27 @@ static void parseRequestedDeviceList(requestedDevices_t& requestedDevices) { break; } } - if (currentDeviceIndex < 0) { + if (currentDeviceIndex < 0 || + static_cast(currentDeviceIndex) >= numDevices) { deviceIdValid = false; } // Get next token. pch = strtok(nullptr, ","); - if (!deviceIdValid) { - continue; + + // FIXME Allow atleast one valid deviceId so compilation etc doesnt break + // but set validOrdinal_ flag + if (!deviceIdValid && (requestedDevices.size() != 0)) { + // Exit the loop as anything to the right of invalid deviceId + // has to be discarded unless its the first one + break; + } + + if (!deviceIdValid && (requestedDevices.size() == 0)) { + Device::setvalidOrdinal(false); } // Requested device is valid. - requestedDevices[currentDeviceIndex] = true; + requestedDevices[currentDeviceIndex] = deviceIdValid; } } @@ -1182,14 +1196,21 @@ bool Device::init() { uint ordinal = 0; const char* selectDeviceByName = nullptr; - if (!flagIsDefault(GPU_DEVICE_ORDINAL)) { + + if (IS_HIP) { + if (HIP_VISIBLE_DEVICES[0] != '\0' || CUDA_VISIBLE_DEVICES[0] != '\0') { + useDeviceList = true; + parseRequestedDeviceList(requestedDevices, numDevices); + } + } else if (GPU_DEVICE_ORDINAL[0] != '\0') { useDeviceList = true; - parseRequestedDeviceList(requestedDevices); + parseRequestedDeviceList(requestedDevices, numDevices); } else if (!flagIsDefault(GPU_DEVICE_NAME)) { selectDeviceByName = GPU_DEVICE_NAME; } bool foundDevice = false; + // Loop through all active devices and initialize the device info structure for (; ordinal < numDevices; ++ordinal) { // Create the GPU device object @@ -1197,6 +1218,10 @@ bool Device::init() { bool result = (nullptr != d) && d->create(deviceList[ordinal]); if (useDeviceList) { result &= (requestedDevices.find(ordinal) != requestedDevices.end()); + if (!result) { + delete d; + break; + } } if (result && ((nullptr == selectDeviceByName) || ('\0' == selectDeviceByName[0]) || (strstr(selectDeviceByName, d->info().name_) != nullptr))) { diff --git a/rocclr/runtime/device/rocm/rocdevice.cpp b/rocclr/runtime/device/rocm/rocdevice.cpp index 98df526728..9530aba59c 100644 --- a/rocclr/runtime/device/rocm/rocdevice.cpp +++ b/rocclr/runtime/device/rocm/rocdevice.cpp @@ -403,19 +403,37 @@ bool Device::init() { return false; } - std::vector selectedDevices; - selectedDevices.resize(gpu_agents_.size(), true); + std::unordered_map selectedDevices; + bool useDeviceList = false; - if (!flagIsDefault(GPU_DEVICE_ORDINAL)) { - std::fill(selectedDevices.begin(), selectedDevices.end(), false); + if ((GPU_DEVICE_ORDINAL != '\0') || (HIP_VISIBLE_DEVICES[0] != '\0') + || (CUDA_VISIBLE_DEVICES != '\0')) { + useDeviceList = true; + std::string ordinals = IS_HIP ? ((HIP_VISIBLE_DEVICES[0] != '\0') ? + HIP_VISIBLE_DEVICES : CUDA_VISIBLE_DEVICES) + : GPU_DEVICE_ORDINAL; - std::string ordinals(GPU_DEVICE_ORDINAL); size_t end, pos = 0; do { + bool deviceIdValid = true; end = ordinals.find_first_of(',', pos); - size_t index = atoi(ordinals.substr(pos, end - pos).c_str()); - selectedDevices.resize(index + 1); - selectedDevices[index] = true; + int index = atoi(ordinals.substr(pos, end - pos).c_str()); + if (index < 0 || static_cast(index) >= gpu_agents_.size()) { + deviceIdValid = false; + } + // FIXME Allow atleast one valid deviceId so compilation etc doesnt break + // but set validOrdinal_ flag + if (!deviceIdValid && (selectedDevices.size() != 0)) { + // Exit the loop as anything to the right of invalid deviceId + // has to be discarded unless its the first one + break; + } + + if (!deviceIdValid && (selectedDevices.size() == 0)) { + Device::setvalidOrdinal(false); + } + + selectedDevices[index] = deviceIdValid; pos = end + 1; } while (end != std::string::npos); } @@ -538,10 +556,13 @@ bool Device::init() { } } - if (selectedDevices[ordinal++] && - (flagIsDefault(GPU_DEVICE_NAME) || GPU_DEVICE_NAME == 0 || GPU_DEVICE_NAME[0] == '\0' || - !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) { - roc_device.release()->registerDevice(); + if (!useDeviceList && (flagIsDefault(GPU_DEVICE_NAME) || GPU_DEVICE_NAME == 0 + || GPU_DEVICE_NAME[0] == '\0' || !strcmp(GPU_DEVICE_NAME, roc_device->info_.name_))) { + roc_device.release()->registerDevice(); + } else if (useDeviceList && selectedDevices[ordinal++]) { + roc_device.release()->registerDevice(); + } else { + break; } } diff --git a/rocclr/runtime/utils/flags.hpp b/rocclr/runtime/utils/flags.hpp index d7f7f45714..981fbb01a9 100644 --- a/rocclr/runtime/utils/flags.hpp +++ b/rocclr/runtime/utils/flags.hpp @@ -205,7 +205,10 @@ release(bool, GPU_FORCE_WAVE_SIZE_32, false, \ "Forces WaveSize32 compilation in SC") \ release(uint, GPU_MAX_COMMAND_BUFFERS, 8, \ "The maximum number of command buffers allocated per queue") \ - +release(cstring, HIP_VISIBLE_DEVICES, "", \ + "Only devices whose index is present in the sequence are visible to HIP") \ +release(cstring, CUDA_VISIBLE_DEVICES, "", \ + "Only devices whose index is present in the sequence are visible to HIP") \ namespace amd {