AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Partition EBUSY with RSMI_STATUS_BUSY & invalid GPU Metrics check

Browse Source 
* Updates:
   - [API/CLI] rsmi_dev_*_partition_set &
     rsmi_dev_*_partition_reset - exposed RSMI_STATUS_BUSY for
     EBUSY writes + cleaned up accidental map insertions
     (maplookup[] can insert values that are not in the map,
     map.at(key) fixes this potential issue)
   - [API] rsmi_dev_gpu_metrics_info_get() - returns
     RSMI_STATUS_NOT_SUPPORTED for unsupported metric tables
     outside of 1v1/1v2/1v3
   - [API] writeDevInfoStr() - exposes RSMI_STATUS_BUSY for
     EBUSY write errors; kept backward compatibility
     for other writes which do not care about these states
   - [API] rsmi_dev_od_volt_info_get()
      & rsmi_dev_od_volt_curve_regions_get() have better logging
     + Expose more details on why they are erroring
   - [Utils/logs/example] Expose AMD GPU gfx target version to aid in
     system troubleshooting
   - [Utils] Added test methods that look at od volt
     freq & regions into here - for easier access across
     several tests
   - [Utils] Updated getRSMIStatusString(new argument - fullstatus;
     default to true for backwards compatibility)
     -> true shows shortened RSMI STATUS response
   - [Utils] Added splitString to cut out noisy return responses
     (used in getRSMIStatusString(), when fullstatus = true)
   - [Utils] Added getFileCreationDate() to expose build date
     of the library - helpful for local builds or experimental builds
   - [Utils] Macro cleanup
   - [Example] Added a few gpu_metric checks - helpful for upcoming
     updates
   - [Device] SYSFS/DebugFS - now have better r/w displayed in logs
   - [LOGS] Expose library build date - see above for details
   - [Tests] Add more warnings/errors to test builds
   - [Tests] Moved up Partition tests for ordered test runs - helped
     identify issues with GPU BUSY writes
   - [Tests] compute_partition_read_write - handles RSMI_STATUS_BUSY
     with waits for busy status found & cleaned up how we checked
     for partition changes - with RSMI responses exposed more clearly
   - [Tests] perf_determinism - multi gpu now properly runs through
     with full resets as needed
   - [Tests] volt_freq_curv_read - better error handling with more
     verbose output

Change-Id: Ie94c6abb6a9aab95c345996d3ad3843cf6734977
Signed-off-by: Charis Poag <Charis.Poag@amd.com>


[ROCm/rocm_smi_lib commit: 57b6135e54]
This commit is contained in:
Charis Poag
2023-10-23 21:37:31 -05:00
parent e142177077
commit e89751e202
18 changed files with 701 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h
+18 -10
View File
@@ -363,16 +363,16 @@ typedef rsmi_clk_type_t rsmi_clk_type;
*/
typedef enum {
RSMI_COMPUTE_PARTITION_INVALID = 0,
RSMI_COMPUTE_PARTITION_CPX, //!< Core mode (CPX)- Per-chip XCC with
//!< shared memory
RSMI_COMPUTE_PARTITION_SPX, //!< Single GPU mode (SPX)- All XCCs work
//!< together with shared memory
RSMI_COMPUTE_PARTITION_DPX, //!< Dual GPU mode (DPX)- Half XCCs work
//!< together with shared memory
RSMI_COMPUTE_PARTITION_TPX, //!< Triple GPU mode (TPX)- One-third XCCs
//!< work together with shared memory
RSMI_COMPUTE_PARTITION_QPX //!< Quad GPU mode (QPX)- Quarter XCCs
//!< work together with shared memory
RSMI_COMPUTE_PARTITION_CPX = 1, //!< Core mode (CPX)- Per-chip XCC with
//!< shared memory
RSMI_COMPUTE_PARTITION_SPX = 2, //!< Single GPU mode (SPX)- All XCCs work
//!< together with shared memory
RSMI_COMPUTE_PARTITION_DPX = 3, //!< Dual GPU mode (DPX)- Half XCCs work
//!< together with shared memory
RSMI_COMPUTE_PARTITION_TPX = 4, //!< Triple GPU mode (TPX)- One-third XCCs
//!< work together with shared memory
RSMI_COMPUTE_PARTITION_QPX = 5, //!< Quad GPU mode (QPX)- Quarter XCCs
//!< work together with shared memory
} rsmi_compute_partition_type_t;
/// \cond Ignore in docs.
typedef rsmi_compute_partition_type_t rsmi_compute_partition_type;
@@ -3783,6 +3783,8 @@ rsmi_dev_compute_partition_get(uint32_t dv_ind, char *compute_partition,
* unavailable for current device
* @retval ::RSMI_STATUS_NOT_SUPPORTED installed software or hardware does not
* support this function
* @retval ::RSMI_STATUS_BUSY A resource or mutex could not be acquired
* because it is already being used - device is busy
*
*/
rsmi_status_t
@@ -3802,6 +3804,8 @@ rsmi_dev_compute_partition_set(uint32_t dv_ind,
* @retval ::RSMI_STATUS_PERMISSION function requires root access
* @retval ::RSMI_STATUS_NOT_SUPPORTED installed software or hardware does not
* support this function
* @retval ::RSMI_STATUS_BUSY A resource or mutex could not be acquired
* because it is already being used - device is busy
*
*/
rsmi_status_t rsmi_dev_compute_partition_reset(uint32_t dv_ind);
@@ -3866,6 +3870,8 @@ rsmi_dev_memory_partition_get(uint32_t dv_ind, char *memory_partition,
* support this function
* @retval ::RSMI_STATUS_AMDGPU_RESTART_ERR could not successfully restart
* the amdgpu driver
* @retval ::RSMI_STATUS_BUSY A resource or mutex could not be acquired
* because it is already being used - device is busy
*
*/
rsmi_status_t
@@ -3887,6 +3893,8 @@ rsmi_dev_memory_partition_set(uint32_t dv_ind,
* support this function
* @retval ::RSMI_STATUS_AMDGPU_RESTART_ERR could not successfully restart
* the amdgpu driver
* @retval ::RSMI_STATUS_BUSY A resource or mutex could not be acquired
* because it is already being used - device is busy
*
*/
rsmi_status_t rsmi_dev_memory_partition_reset(uint32_t dv_ind);
projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h
+2 -1
View File
@@ -260,7 +260,8 @@ class Device {
std::vector<std::string> *retVec);
int readDevInfoBinary(DevInfoTypes type, std::size_t b_size,
void *p_binary_data);
int writeDevInfoStr(DevInfoTypes type, std::string valStr);
int writeDevInfoStr(DevInfoTypes type, std::string valStr,
bool returnWriteErr = false);
rsmi_status_t run_amdgpu_property_reinforcement_query(const AMDGpuPropertyQuery_t& amdgpu_property_query);
projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h
+3
View File
@@ -84,6 +84,9 @@ class KFDNode {
int get_total_memory(uint64_t* total);
int get_used_memory(uint64_t* used);
// Get gfx target version from kfd
int get_gfx_target_version(uint64_t* gfx_target_version);
private:
uint32_t node_indx_;
uint32_t amdgpu_dev_index_;
projects/rocm-smi-lib/include/rocm_smi/rocm_smi_utils.h
+16 -5
View File
@@ -51,6 +51,8 @@
#include <sstream>
#include <iomanip>
#include <type_traits>
#include <tuple>
#include <queue>
#include "rocm_smi/rocm_smi_device.h"
@@ -97,10 +99,10 @@ rsmi_status_t
GetDevBinaryBlob(amd::smi::DevInfoTypes type,
uint32_t dv_ind, std::size_t b_size, void* p_binary_data);
rsmi_status_t ErrnoToRsmiStatus(int err);
std::string getRSMIStatusString(rsmi_status_t ret);
std::string getRSMIStatusString(rsmi_status_t ret, bool fullStatus = true);
std::tuple<bool, std::string, std::string, std::string, std::string,
std::string, std::string, std::string, std::string,
std::string, std::string, std::string>
std::string, std::string, std::string, std::string, std::string>
getSystemDetails(void);
void logSystemDetails(void);
rsmi_status_t getBDFString(uint64_t bdf_id, std::string& bfd_str);
@@ -109,11 +111,20 @@ void logHexDump(const char *desc, const void *addr, const size_t len,
bool isSystemBigEndian();
std::string getBuildType();
std::string getMyLibPath();
std::string getFileCreationDate(std::string path);
int subDirectoryCountInPath(const std::string path);
std::queue<std::string> getAllDeviceGfxVers();
std::string monitor_type_string(amd::smi::MonitorTypes type);
std::string power_type_string(RSMI_POWER_TYPE type);
std::string splitString(std::string str, char delim);
std::string print_rsmi_od_volt_freq_data_t(rsmi_od_volt_freq_data_t *odv);
std::string print_rsmi_od_volt_freq_regions(uint32_t num_regions,
rsmi_freq_volt_region_t *regions);
bool is_sudo_user();
rsmi_status_t rsmi_get_gfx_target_version(uint32_t dv_ind,
std::string *gfx_version);
template <typename T>
std::string print_int_as_hex(T i, bool showHexNotation=true) {
std::string print_int_as_hex(T i, bool showHexNotation = true) {
std::stringstream ss;
if (showHexNotation) {
ss << "0x" << std::setfill('0') << std::setw(sizeof(T) * 2) << std::hex;
@@ -132,7 +143,7 @@ std::string print_int_as_hex(T i, bool showHexNotation=true) {
}
ss << std::dec;
return ss.str();
};
}
template <typename T>
std::string print_unsigned_int(T i) {
@@ -263,7 +274,7 @@ class ScopedAcquire {
LockType* lock_;
bool doRelease;
/// @brief: Disable copiable and assignable ability.
DISALLOW_COPY_AND_ASSIGN(ScopedAcquire);
DISALLOW_COPY_AND_ASSIGN(ScopedAcquire)
};
} // namespace smi