diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h index 94f8921da3..476134b09e 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h @@ -124,7 +124,6 @@ typedef enum { RSMI_CLK_TYPE_FIRST = RSMI_CLK_TYPE_SYS, RSMI_CLK_TYPE_MEM, //!< Memory clock - RSMI_CLK_TYPE_LAST = RSMI_CLK_TYPE_MEM } rsmi_clk_type; @@ -239,6 +238,24 @@ typedef struct { uint64_t frequency[RSMI_MAX_NUM_FREQUENCIES]; } rsmi_frequencies; +/** + * @brief This structure holds information about the possible PCIe + * bandwidths. Specifically, the possible transfer rates and their + * associated numbers of lanes are stored here. + */ +typedef struct { + /** + * Transfer rates (T/s) that are possible + */ + rsmi_frequencies transfer_rate; + + /** + * List of lanes for corresponding transfer rate. + * Only the first num_supported bandwidths are valid. + */ + uint32_t lanes[RSMI_MAX_NUM_FREQUENCIES]; +} rsmi_pcie_bandwidth; + /** * @brief Initialize Rocm SMI. @@ -274,13 +291,58 @@ rsmi_status_t rsmi_shut_down(void); */ rsmi_status_t rsmi_num_monitor_devices(uint32_t *num_devices); +/** + * @brief Get the list of possible pci bandwidths that are available. + * + * @details Given a device index @p dv_ind and a pointer to a to an + * rsmi_pcie_bandwidth structure @p bandwidth, this function will fill in + * @p bandwidth with the possible T/s values and associated number of lanes, + * and indication of the current selection. + * + * @param[in] dv_ind a device index + * + * @param[inout] bandwidth a pointer to a caller provided rsmi_pcie_bandwidth + * structure to which the frequency information will be written + * + * @retval RSMI_STATUS_SUCCESS is returned upon successful call. + * + */ +rsmi_status_t +rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth *bandwidth); + +/** + * @brief Control the set of allowed PCIe bandwidths that can be used. + * + * @details Given a device index @p dv_ind and a 64 bit bitmask @p bw_bitmask, + * this function will limit the set of allowable bandwidths. If a bit in @p + * bw_bitmask has a value of 1, then the frequency (as ordered in an + * rsmi_frequencies returned by rsmi_dev_get_gpu_clk_freq()) corresponding + * to that bit index will be allowed. + * + * This function will change the performance level to + * ::RSMI_DEV_PERF_LEVEL_MANUAL in order to modify the set of allowable + * band_widths. Caller will need to set to ::RSMI_DEV_PERF_LEVEL_AUTO in order + * to get back to default state. + * + * All bits with indices greater than or equal to + * rsmi_pcie_bandwidth.transfer_rate.num_supported will be ignored. + * + * @param[in] dv_ind a device index + * + * @param[in] bw_bitmask A bitmask indicating the indices of the + * bandwidths that are to be enabled (1) and disabled (0). Only the lowest + * rsmi_pcie_bandwidth.transfer_rate.num_supported bits of this mask are + * relevant. + */ +rsmi_status_t rsmi_dev_pci_bandwidth_set(uint32_t dv_ind, uint64_t bw_bitmask); + /** * @brief Get the unique PCI device identifier associated for a device * - * @details Give a device index @p dev_ind and a pointer to a uint64_t @p + * @details Give a device index @p dv_ind and a pointer to a uint64_t @p * bdfid, this function will write the Bus/Device/Function PCI identifier - * (BDFID) associated with device @p dev_ind to the value pointed to by - * @bdfid. + * (BDFID) associated with device @p dv_ind to the value pointed to by + * @p bdfid. * * @param[in] dv_ind a device index * @@ -290,7 +352,7 @@ rsmi_status_t rsmi_num_monitor_devices(uint32_t *num_devices); * @retval RSMI_STATUS_SUCCESS is returned upon successful call. */ -rsmi_status_t rsmi_dev_pci_id_get(uint32_t dev_ind, uint64_t *bdfid); +rsmi_status_t rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid); /** * @brief Get the device id associated with the device with provided device @@ -436,7 +498,7 @@ rsmi_status_t rsmi_dev_gpu_clk_freq_get(uint32_t dv_ind, * specified clock. * * @details Given a device index @p dv_ind, a clock type @p clk_type, and a - * 32 bit bitmask @p freq_bitmask, this function will limit the set of + * 64 bit bitmask @p freq_bitmask, this function will limit the set of * allowable frequencies. If a bit in @p freq_bitmask has a value of 1, then * the frequency (as ordered in an rsmi_frequencies returned by * rsmi_dev_get_gpu_clk_freq()) corresponding to that bit index will be @@ -739,8 +801,8 @@ rsmi_dev_power_max_get(uint32_t dv_ind, uint32_t sensor_ind, uint64_t *power); * * @param[in] dv_ind a device index * - * @param[in] sensor_ind a 0-based sensor index. Normally, this will be 0. - * If a device has more than one sensor, it could be greater than 0. + * @param[in] sensor_ind a 0-based sensor index. Normally, this will be 0. + * If a device has more than one sensor, it could be greater than 0. * * @param[inout] status a pointer to rsmi_power_profile_status that will be * populated by a call to this function @@ -769,7 +831,7 @@ rsmi_dev_power_profile_presets_get(uint32_t dv_ind, uint32_t sensor_ind, * @param[in] profile a rsmi_power_profile_preset_masks that hold the mask * of the desired new power profile * - * @retval RSMI_STATUS_SUCCESS is returned upon successful call. + * @retval RSMI_STATUS_SUCCESS is returned upon successful call. * */ rsmi_status_t @@ -786,11 +848,31 @@ rsmi_dev_power_profile_set(uint32_t dv_ind, uint32_t sensor_ind, * @param[inout] status_string A pointer to a const char * which will be made * to point to a description of the provided error code * - * @retval RSMI_STATUS_SUCCESS is returned upon successful call + * @retval RSMI_STATUS_SUCCESS is returned upon successful call * */ rsmi_status_t rsmi_status_string(rsmi_status_t status, const char **status_string); + +/** + * @brief Get percentage of time device is busy doing any processing + * + * @details Given a device index @p dv_ind, this function returns the + * percentage of time that the specified device is busy. The device is + * considered busy if any one or more of its sub-blocks are working, and idle + * if none of the sub-blocks are working. + * + * @param[in] dv_ind a device index + * + * @param[inout] busy_percent a pointer to the uint32_t to which the busy + * percent will be written + * + * @retval RSMI_STATUS_SUCCESS is returned upon successful call + * + */ +rsmi_status_t +rsmi_dev_busy_percent_get(uint32_t dv_ind, uint32_t *busy_percent); + #ifdef __cplusplus } #endif // __cplusplus diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h index 8a3c6448d4..61f7774660 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h @@ -64,7 +64,9 @@ enum DevInfoTypes { kDevDevID, kDevGPUMClk, kDevGPUSClk, - kDevPowerProfileMode + kDevPCIEBW, + kDevPowerProfileMode, + kDevUsage, }; class Device { @@ -78,7 +80,10 @@ class Device { const std::shared_ptr& power_monitor() {return power_monitor_;} void set_power_monitor(std::shared_ptr pm) {power_monitor_ = pm;} +#if 0 // This is not being used right now. int readDevInfo(DevInfoTypes type, uint32_t *val); +#endif + int readDevInfo(DevInfoTypes type, std::string *val); int readDevInfo(DevInfoTypes type, std::vector *retVec); int writeDevInfo(DevInfoTypes type, uint64_t val); diff --git a/projects/rocm-smi-lib/src/rocm_smi.cc b/projects/rocm-smi-lib/src/rocm_smi.cc index 249a932ca8..1f210d8a3a 100755 --- a/projects/rocm-smi-lib/src/rocm_smi.cc +++ b/projects/rocm-smi-lib/src/rocm_smi.cc @@ -102,20 +102,23 @@ static rsmi_status_t errno_to_rsmi_status(uint32_t err) { case 0: return RSMI_STATUS_SUCCESS; case EACCES: return RSMI_STATUS_PERMISSION; case EPERM: return RSMI_STATUS_NOT_SUPPORTED; - case ENOENT: return RSMI_STATUS_FILE_ERROR; + case ENOENT: + case EISDIR: return RSMI_STATUS_FILE_ERROR; default: return RSMI_STATUS_UNKNOWN_ERROR; } } /** - * Parse a string of the form ": <|*>" + * Parse a string of the form: + * ": <|*>" */ -static uint32_t freq_string_to_int(std::string freq_line, bool *is_curr) { +static uint64_t freq_string_to_int(const std::vector &freq_lines, + bool *is_curr, uint32_t lanes[], int i) { assert(is_curr != nullptr); - std::istringstream fs(freq_line); + std::istringstream fs(freq_lines[i]); uint32_t ind; - uint32_t freq; + float freq; std::string junk; std::string units_str; std::string star_str; @@ -127,7 +130,7 @@ static uint32_t freq_string_to_int(std::string freq_line, bool *is_curr) { fs >> star_str; if (is_curr != nullptr) { - if (freq_line.find("*") != std::string::npos) { + if (freq_lines[i].find("*") != std::string::npos) { *is_curr = true; } else { *is_curr = false; @@ -135,18 +138,33 @@ static uint32_t freq_string_to_int(std::string freq_line, bool *is_curr) { } uint32_t multiplier = 0; - if (units_str == "Mhz") { - multiplier = 1000000; - } else if (units_str == "Ghz") { - multiplier = 1000000000; - } else if (units_str == "Khz") { - multiplier = 1000; - } else if (units_str == "Hz") { - multiplier = 1; - } else { - assert(!"Unexpected units for frequency"); + switch (units_str[0]) { + case 'G': // GT or GHz + multiplier = 1000000000; + break; + + case 'M': // MT or MHz + multiplier = 1000000; + break; + + case 'K': // KT or KHz + multiplier = 1000; + break; + + case 'T': // Transactions + case 'H': // Hertz + multiplier = 1; + break; + default: + assert(!"Unexpected units for frequency"); } + if (star_str[0] == 'x') { + assert(lanes != nullptr && "Lanes are provided but null lanes pointer"); + if (lanes) { + lanes[i] = std::stoi(star_str.substr(1), nullptr); + } + } return freq*multiplier; } @@ -430,7 +448,7 @@ rsmi_dev_perf_level_set(int32_t dv_ind, rsmi_dev_perf_level perf_level) { } static rsmi_status_t get_frequencies(amd::smi::DevInfoTypes type, - uint32_t dv_ind, rsmi_frequencies *f) { + uint32_t dv_ind, rsmi_frequencies *f, uint32_t *lanes = nullptr) { TRY std::vector val_vec; rsmi_status_t ret; @@ -450,7 +468,7 @@ static rsmi_status_t get_frequencies(amd::smi::DevInfoTypes type, f->current = RSMI_MAX_NUM_FREQUENCIES + 1; // init to an invalid value for (uint32_t i = 0; i < f->num_supported; ++i) { - f->frequency[i] = freq_string_to_int(val_vec[i], ¤t); + f->frequency[i] = freq_string_to_int(val_vec, ¤t, lanes, i); // Our assumption is that frequencies are read in from lowest to highest. // Check that that is true. @@ -665,6 +683,60 @@ rsmi_dev_name_get(uint32_t dv_ind, char *name, size_t len) { CATCH } +rsmi_status_t +rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth *b) { + TRY + assert(b != nullptr); + + if (b == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + + return get_frequencies(amd::smi::kDevPCIEBW, dv_ind, + &b->transfer_rate, b->lanes); + + CATCH +} + +rsmi_status_t +rsmi_dev_pci_bandwidth_set(uint32_t dv_ind, uint64_t bw_bitmask) { + rsmi_status_t ret; + rsmi_pcie_bandwidth bws; + + TRY + ret = rsmi_dev_pci_bandwidth_get(dv_ind, &bws); + + if (ret != RSMI_STATUS_SUCCESS) { + return ret; + } + + assert(bws.transfer_rate.num_supported <= RSMI_MAX_NUM_FREQUENCIES); + + amd::smi::RocmSMI smi = amd::smi::RocmSMI::getInstance(); + + // Above call to rsmi_dev_pci_bandwidth_get() should have emitted an error + // if assert below is not true + assert(dv_ind < smi.monitor_devices().size()); + + std::string freq_enable_str = + bitfield_to_freq_string(bw_bitmask, bws.transfer_rate.num_supported); + + std::shared_ptr dev = smi.monitor_devices()[dv_ind]; + assert(dev != nullptr); + + ret = rsmi_dev_perf_level_set(dv_ind, RSMI_DEV_PERF_LEVEL_MANUAL); + if (ret != RSMI_STATUS_SUCCESS) { + return ret; + } + + uint32_t ret_i; + ret_i = dev->writeDevInfo(amd::smi::kDevPCIEBW, freq_enable_str); + + return errno_to_rsmi_status(ret_i); + + CATCH +} + rsmi_status_t rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_ind, rsmi_temperature_metric metric, int64_t *temperature) { @@ -1026,3 +1098,22 @@ rsmi_status_string(rsmi_status_t status, const char **status_string) { return RSMI_STATUS_SUCCESS; CATCH } + +rsmi_status_t +rsmi_dev_busy_percent_get(uint32_t dv_ind, uint32_t *busy_percent) { + TRY + std::string val_str; + rsmi_status_t ret = get_dev_value_str(amd::smi::kDevUsage, dv_ind, + &val_str); + if (ret != RSMI_STATUS_SUCCESS) { + return ret; + } + + errno = 0; + *busy_percent = strtoul(val_str.c_str(), nullptr, 10); + assert(errno == 0); + + return RSMI_STATUS_SUCCESS; + + CATCH +} diff --git a/projects/rocm-smi-lib/src/rocm_smi_device.cc b/projects/rocm-smi-lib/src/rocm_smi_device.cc index 1f1a63cdc9..98cbb48422 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_device.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_device.cc @@ -64,7 +64,10 @@ static const char *kDevDevIDFName = "device"; static const char *kDevOverDriveLevelFName = "pp_sclk_od"; static const char *kDevGPUSClkFName = "pp_dpm_sclk"; static const char *kDevGPUMClkFName = "pp_dpm_mclk"; -static const char *kDevPowerProfileModeName = "pp_power_profile_mode"; +static const char *kDevGPUPCIEClkFname = "pp_dpm_pcie"; +static const char *kDevPowerProfileModeFName = "pp_power_profile_mode"; +static const char *kDevUsageFName = "gpu_busy_percent"; + static const char *kDevPerfLevelAutoStr = "auto"; static const char *kDevPerfLevelLowStr = "low"; static const char *kDevPerfLevelHighStr = "high"; @@ -81,7 +84,9 @@ static const std::map kDevAttribNameMap = { {kDevDevID, kDevDevIDFName}, {kDevGPUMClk, kDevGPUMClkFName}, {kDevGPUSClk, kDevGPUSClkFName}, - {kDevPowerProfileMode, kDevPowerProfileModeName}, + {kDevPCIEBW, kDevGPUPCIEClkFname}, + {kDevPowerProfileMode, kDevPowerProfileModeFName}, + {kDevUsage, kDevUsageFName}, }; static const std::map kDevPerfLvlMap = { @@ -114,7 +119,6 @@ Device::Device(std::string p, RocmSMI_env_vars const *e) : path_(p), env_(e) { Device:: ~Device() { } -// TODO(cfreehil): cache values that are constant int Device::readDevInfoStr(DevInfoTypes type, std::string *retStr) { auto tempPath = path_; @@ -193,6 +197,7 @@ int Device::writeDevInfo(DevInfoTypes type, uint64_t val) { case kDevGPUMClk: // integer (index within num-freq range) case kDevGPUSClk: // integer (index within num-freq range) + case kDevPCIEBW: // integer (index within num-freq range) case kDevDevID: // string (read-only) default: break; @@ -205,6 +210,7 @@ int Device::writeDevInfo(DevInfoTypes type, std::string val) { switch (type) { case kDevGPUMClk: case kDevGPUSClk: + case kDevPCIEBW: return writeDevInfoStr(type, val); case kDevOverDriveLevel: @@ -242,12 +248,12 @@ int Device::readDevInfoMultiLineStr(DevInfoTypes type, return 0; } +#if 0 int Device::readDevInfo(DevInfoTypes type, uint32_t *val) { assert(val != nullptr); std::string tempStr; int ret; - switch (type) { case kDevDevID: ret = readDevInfoStr(type, &tempStr); @@ -255,6 +261,7 @@ int Device::readDevInfo(DevInfoTypes type, uint32_t *val) { *val = std::stoi(tempStr, 0, 16); break; + case kDevUsage: case kDevOverDriveLevel: ret = readDevInfoStr(type, &tempStr); RET_IF_NONZERO(ret); @@ -266,13 +273,14 @@ int Device::readDevInfo(DevInfoTypes type, uint32_t *val) { } return 0; } - +#endif int Device::readDevInfo(DevInfoTypes type, std::vector *val) { assert(val != nullptr); switch (type) { case kDevGPUMClk: case kDevGPUSClk: + case kDevPCIEBW: case kDevPowerProfileMode: return readDevInfoMultiLineStr(type, val); break; @@ -289,6 +297,7 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) { switch (type) { case kDevPerfLevel: + case kDevUsage: case kDevOverDriveLevel: case kDevDevID: return readDevInfoStr(type, val); diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/rsmi_sanity.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/rsmi_sanity.cc index 3c167dd5e0..d99980e472 100755 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/functional/rsmi_sanity.cc +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/functional/rsmi_sanity.cc @@ -451,10 +451,66 @@ static rsmi_status_t test_set_freq(uint32_t dv_ind) { return RSMI_STATUS_SUCCESS; } -static void print_frequencies(rsmi_frequencies *f) { +static rsmi_status_t test_set_pci_bw(uint32_t dv_ind) { + rsmi_status_t ret; + rsmi_pcie_bandwidth bw; + uint32_t freq_bitmask; + + print_test_header("PCIe Bandwidth Control", dv_ind); + + ret = rsmi_dev_pci_bandwidth_get(dv_ind, &bw); + CHK_ERR_RET(ret) + + IF_VERB(STANDARD) { + std::cout << "Initial PCIe is " << bw.transfer_rate.current << std::endl; + } + + // First set the bitmask to all supported bandwidths + freq_bitmask = ~(~0 << bw.transfer_rate.num_supported); + + // Then, set the bitmask to all bandwidths besides the initial BW + freq_bitmask ^= (1 << bw.transfer_rate.current); + + std::string freq_bm_str = + std::bitset(freq_bitmask).to_string(); + + freq_bm_str.erase(0, std::min(freq_bm_str.find_first_not_of('0'), + freq_bm_str.size()-1)); + + IF_VERB(STANDARD) { + std::cout << "Setting bandwidth mask to " << "0b" << freq_bm_str << + " ..." << std::endl; + } + ret = rsmi_dev_pci_bandwidth_set(dv_ind, freq_bitmask); + CHK_ERR_RET(ret) + + ret = rsmi_dev_pci_bandwidth_get(dv_ind, &bw); + CHK_ERR_RET(ret) + + IF_VERB(STANDARD) { + std::cout << "Bandwidth is now index " << bw.transfer_rate.current << + std::endl; + std::cout << "Resetting mask to all bandwidths." << std::endl; + } + ret = rsmi_dev_pci_bandwidth_set(dv_ind, 0xFFFFFFFF); + CHK_ERR_RET(ret) + + ret = rsmi_dev_perf_level_set(dv_ind, RSMI_DEV_PERF_LEVEL_AUTO); + CHK_ERR_RET(ret) + + return RSMI_STATUS_SUCCESS; +} + +static void print_frequencies(rsmi_frequencies *f, uint32_t *l=nullptr) { assert(f != nullptr); for (uint32_t j = 0; j < f->num_supported; ++j) { std::cout << "\t** " << j << ": " << f->frequency[j]; + if (l != nullptr) { + std::cout << "T/s; x" << l[j]; + } else { + std::cout << "Hz"; + } + if (j == f->current) { std::cout << " *"; } @@ -500,6 +556,7 @@ void TestSanity::Run(void) { uint32_t val_ui32; rsmi_dev_perf_level pfl; rsmi_frequencies f; + rsmi_pcie_bandwidth b; uint32_t num_monitor_devs = 0; @@ -542,6 +599,14 @@ void TestSanity::Run(void) { std::cout << f.num_supported << std::endl; print_frequencies(&f); } + err = rsmi_dev_pci_bandwidth_get(i, &b); + CHK_ERR_ASRT(err) + IF_VERB(STANDARD) { + std::cout << "\t**Supported PCIe bandwidths: "; + std::cout << b.transfer_rate.num_supported << std::endl; + print_frequencies(&b.transfer_rate, b.lanes); + } + err = rsmi_dev_gpu_clk_freq_get(i, RSMI_CLK_TYPE_SYS, &f); CHK_ERR_ASRT(err) IF_VERB(STANDARD) { @@ -549,6 +614,23 @@ void TestSanity::Run(void) { std::cout << f.num_supported << std::endl; print_frequencies(&f); } + err = rsmi_dev_busy_percent_get(i, &val_ui32); + if (err != RSMI_STATUS_SUCCESS) { + if (err == RSMI_STATUS_FILE_ERROR) { + IF_VERB(STANDARD) { + std::cout << "\t**GPU Busy Percent: Not supported on this machine" + << std::endl; + } + } else { + CHK_ERR_ASRT(err) + } + } else { + IF_VERB(STANDARD) { + std::cout << "\t**GPU Busy Percent (Percent Idle):" << std::dec << + val_ui32 << " (" << 100 - val_ui32 << ")" << std::endl; + } + } + char name[20]; err = rsmi_dev_name_get(i, name, 20); CHK_ERR_ASRT(err) @@ -672,6 +754,9 @@ void TestSanity::Run(void) { err = test_set_freq(i); CHK_RSMI_PERM_ERR(err) + err = test_set_pci_bw(i); + CHK_RSMI_PERM_ERR(err) + err = test_set_fan_speed(i); CHK_RSMI_PERM_ERR(err) diff --git a/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc b/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc index d6b548cf9d..20f13ec56a 100755 --- a/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc +++ b/projects/rocm-smi-lib/tests/rocm_smi_test/main.cc @@ -103,24 +103,6 @@ TEST(rsmitst, RSMISanityTest) { RunGenericTest(&tst); } -#if 0 - -TEST(rocrtstFunc, IPC) { - IPCTest ipc; - RunGenericTest(&ipc); -} - -TEST(rocrtstFunc, MemoryAccessTests) { - MemoryAccessTest mt; - RunCustomTestProlog(&mt); - mt.CPUAccessToGPUMemoryTest(); - mt.GPUAccessToCPUMemoryTest(); - RunCustomTestEpilog(&mt); -} - - - -#endif int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); @@ -130,7 +112,7 @@ int main(int argc, char** argv) { // Set some default values settings.verbosity = 1; settings.monitor_verbosity = 1; - settings.num_iterations = 5; + settings.num_iterations = 1; if (ProcessCmdline(&settings, argc, argv)) {