AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add rocmtools support

Browse Source 
This commit adds integration with ROCmTools

Additional changes:
- Fix DEB and RPM installation issue when systemd is not present
- Fix typos in rdc.h
- Wrap negative values in parentheses in rdc.h
- CMAKE: Improve rocm_smi searching
- README: Improve formatting, add info about ROCmTools

Metrics added: 700-714
Metrics can be listed with `rdci dmon --list-all`
Majority of the metrics are only supported by Instict (MI) series GPUs
700 RDC_FI_PROF_ELAPSED_CYCLES should be available on most devices
See README for more information

Change-Id: I907d3eacdc92fc5588ca6c76c2fa1ce0ad900770
Signed-off-by: Galantsev, Dmitrii <dmitrii.galantsev@amd.com>


[ROCm/rdc commit: 861a843ed7]
This commit is contained in:
Galantsev, Dmitrii
2022-12-13 14:37:59 -06:00
parent 001461e975
commit eccb4e202c
26 changed files with 1061 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rdc/include/rdc/rdc.h
+31 -12
View File
@@ -104,11 +104,11 @@ typedef enum {
//! ID used to represent an invalid GPU
#define GPU_ID_INVALID -1
#define GPU_ID_INVALID (-1)
//! Used to specify all GPUs
#define RDC_GROUP_ALL_GPUS -1000
#define RDC_GROUP_ALL_GPUS (-1000)
//! Used to specify all stats fields
#define RDC_JOB_STATS_FIELDS -1000
#define RDC_JOB_STATS_FIELDS (-1000)
/**
* @brief The max rdc field string length
@@ -223,6 +223,25 @@ typedef enum {
RDC_FI_ECC_UMC_SEC, //!< UMC Single Error Correction
RDC_FI_ECC_UMC_DED, //!< UMC Double Error Detection
/**
* @brief ROC-profiler related fields
*/
RDC_FI_PROF_ELAPSED_CYCLES = 700, //!< Number of elapsed cycles over all SMs
RDC_FI_PROF_ACTIVE_WAVES, //!< Number of Active Waves
RDC_FI_PROF_ACTIVE_CYCLES, //!< Number of Active Cycles
RDC_FI_PROF_CU_OCCUPANCY, //!< Active Waves / maximum active Waves supported
RDC_FI_PROF_CU_UTILIZATION, //!< Total active cycles / Total elapsed cycles
RDC_FI_PROF_FETCH_SIZE, //!< Number of kilobytes fetched from video memory
RDC_FI_PROF_WRITE_SIZE, //!< Number of kilobytes written to video memory
RDC_FI_PROF_FLOPS_16, //!< Number of fp16 OPS / second
RDC_FI_PROF_FLOPS_32, //!< Number of fp32 OPS / second
RDC_FI_PROF_FLOPS_64, //!< Number of fp64 OPS / second
RDC_FI_PROF_GFLOPS_16, //!< Number of fp16 GOPS / second
RDC_FI_PROF_GFLOPS_32, //!< Number of fp32 GOPS / second
RDC_FI_PROF_GFLOPS_64, //!< Number of fp64 GOPS / second
RDC_FI_PROF_MEMR_BW_KBPNS, //!< HBM Read Bandwidth in kilobytes / nanosecond
RDC_FI_PROF_MEMW_BW_KBPNS, //!< HBM Write Bandwidth in kilobytes / nanosecond
/*
* @brief Raw XGMI counter events
*/
@@ -253,7 +272,7 @@ typedef enum {
//!< neighbor 1
RDC_EVNT_XGMI_1_BEATS_TX, //!< Data beats sent to
//!< neighbor 1; Each beat
//!< represnts 32 bytes
//!< represents 32 bytes
// "Composite" events. These events have additional processing beyond
// the value provided by the rocm_smi library.
@@ -328,7 +347,7 @@ typedef struct {
uint64_t energy_consumed; //!< GPU Energy consumed
uint64_t ecc_correct; //!< Correctable errors
uint64_t ecc_uncorrect; //!< Uncorrtable errors
uint64_t ecc_uncorrect; //!< Uncorrectable errors
rdc_stats_summary_t pcie_tx; //!< Bytes sent over PCIe stats
rdc_stats_summary_t pcie_rx; //!< Bytes received over PCIe stats
rdc_stats_summary_t power_usage; //!< GPU Power usage stats
@@ -348,7 +367,7 @@ typedef struct {
uint32_t num_gpus; //!< Number of GPUs used by job
rdc_gpu_usage_info_t summary; //!< Job usage summary statistics
//!< (overall)
rdc_gpu_usage_info_t gpus[16]; //!< Job usage summary staticstics by GPU
rdc_gpu_usage_info_t gpus[16]; //!< Job usage summary statistics by GPU
} rdc_job_info_t;
/**
@@ -958,18 +977,18 @@ rdc_status_t rdc_field_unwatch(rdc_handle_t p_rdc_handle,
/**
* @brief Run the diagnostic test cases
*
* @details Run the diagnostic test cases at differenet levles.
* @details Run the diagnostic test cases at different levels.
*
* @param[in] p_rdc_handle The RDC handler.
*
* @param[in] group_id The GPU group id.
*
* @param[in] level The level decides how long the test will run.
* The RDC_DIAG_LVL_SHORT only take a few seconds, and the
* The RDC_DIAG_LVL_SHORT only take a few seconds, and the
* the RDC_DIAG_LVL_LONG may take up to 15 minutes.
*
*
* @param[inout] response The detail results of the tests run.
*
*
* @retval ::RDC_ST_OK is returned upon successful call.
*/
rdc_status_t rdc_diagnostic_run(
@@ -988,9 +1007,9 @@ rdc_status_t rdc_diagnostic_run(
* @param[in] group_id The GPU group id.
*
* @param[in] test_case The test case to run.
*
*
* @param[inout] result The results of the test.
*
*
* @retval ::RDC_ST_OK is returned upon successful call.
*/
rdc_status_t rdc_test_case_run(
projects/rdc/include/rdc_lib/impl/RdcModuleMgrImpl.h
+7 -4
View File
@@ -23,21 +23,24 @@ THE SOFTWARE.
#define INCLUDE_RDC_LIB_IMPL_RDCMODULEMGRIMPL_H_
#include <memory>
#include "rdc_lib/RdcModuleMgr.h"
#include "rdc_lib/RdcMetricFetcher.h"
#include "rdc_lib/RdcModuleMgr.h"
#include "rdc_lib/RdcTelemetry.h"
#include "rdc_lib/impl/RdcRasLib.h"
#include "rdc_lib/impl/RdcSmiLib.h"
#include "rdc_lib/impl/RdcRocpLib.h"
#include "rdc_lib/impl/RdcRocrLib.h"
#include "rdc_lib/impl/RdcSmiLib.h"
namespace amd {
namespace rdc {
class RdcModuleMgrImpl: public RdcModuleMgr {
class RdcModuleMgrImpl : public RdcModuleMgr {
public:
RdcTelemetryPtr get_telemetry_module() override;
RdcDiagnosticPtr get_diagnostic_module() override;
explicit RdcModuleMgrImpl(const RdcMetricFetcherPtr& fetcher);
private:
// Function module
RdcTelemetryPtr rdc_telemetry_module_;
@@ -48,10 +51,10 @@ class RdcModuleMgrImpl: public RdcModuleMgr {
RdcSmiLibPtr smi_lib_;
RdcMetricFetcherPtr fetcher_;
RdcRocrLibPtr rocr_lib_;
RdcRocpLibPtr rocp_lib_;
};
} // namespace rdc
} // namespace amd
#endif // INCLUDE_RDC_LIB_IMPL_RDCMODULEMGRIMPL_H_
projects/rdc/include/rdc_lib/impl/RdcRocpLib.h
+92
View File
@@ -0,0 +1,92 @@
/*
Copyright (c) 2022 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef INCLUDE_RDC_LIB_IMPL_RDCROCPLIB_H_
#define INCLUDE_RDC_LIB_IMPL_RDCROCPLIB_H_
#include <cstdint>
#include <memory>
#include <vector>
#include "rdc_lib/RdcLibraryLoader.h"
#include "rdc_lib/RdcTelemetry.h"
namespace amd {
namespace rdc {
class RdcRocpLib : public RdcTelemetry {
public:
/* Telemetry */
// get support field ids
rdc_status_t rdc_telemetry_fields_query(
uint32_t field_ids[MAX_NUM_FIELDS],
uint32_t* field_count) override;
// Fetch
rdc_status_t rdc_telemetry_fields_value_get(
rdc_gpu_field_t* fields,
uint32_t fields_count,
rdc_field_value_f callback,
void* user_data) override;
rdc_status_t rdc_telemetry_fields_watch(
rdc_gpu_field_t* fields,
uint32_t fields_count) override;
rdc_status_t rdc_telemetry_fields_unwatch(
rdc_gpu_field_t* fields,
uint32_t fields_count) override;
uint64_t get_profiler_version();
explicit RdcRocpLib(const char* lib_name);
~RdcRocpLib();
private:
RdcLibraryLoader lib_loader_;
rdc_status_t (*telemetry_fields_query_)(
uint32_t field_ids[MAX_NUM_FIELDS],
uint32_t* field_count);
rdc_status_t (*telemetry_fields_value_get_)(
rdc_gpu_field_t* fields,
uint32_t fields_count,
rdc_field_value_f callback,
void* user_data);
rdc_status_t (*telemetry_fields_watch_)(
rdc_gpu_field_t* fields,
uint32_t fields_count);
rdc_status_t (*telemetry_fields_unwatch_)(
rdc_gpu_field_t* fields,
uint32_t fields_count);
};
using RdcRocpLibPtr = std::shared_ptr<RdcRocpLib>;
} // namespace rdc
} // namespace amd
#endif // INCLUDE_RDC_LIB_IMPL_RDCROCPLIB_H_
projects/rdc/include/rdc_lib/impl/RdcTelemetryModule.h
+3 -1
View File
@@ -29,6 +29,7 @@ THE SOFTWARE.
#include "rdc_lib/RdcTelemetry.h"
#include "rdc_lib/impl/RdcRasLib.h"
#include "rdc_lib/impl/RdcSmiLib.h"
#include "rdc_lib/impl/RdcRocpLib.h"
#include "rdc_lib/RdcMetricFetcher.h"
namespace amd {
@@ -50,7 +51,8 @@ class RdcTelemetryModule : public RdcTelemetry {
uint32_t fields_count);
RdcTelemetryModule(const RdcSmiLibPtr& smi_lib,
const RdcRasLibPtr& ras_module);
const RdcRasLibPtr& ras_module,
const RdcRocpLibPtr& rocp_module);
private:
//< Helper function to dispatch fields to module
projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h
+137
View File
@@ -0,0 +1,137 @@
/*
Copyright (c) 2022 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef RDC_MODULES_RDC_ROCP_RDCROCPBASE_H_
#define RDC_MODULES_RDC_ROCP_RDCROCPBASE_H_
#include <rocmtools.h>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <map>
#include <string>
#include <typeinfo>
#include <unordered_map>
#include "rdc/rdc.h"
namespace amd {
namespace rdc {
/**
* @brief Map of RDC fields to rocmtools counters
*
* See metrics.xml in rocmtools for more info.
* RDC_CALC fields are calculated over time by RDC.
*/
static const std::unordered_map<rdc_field_t, const char*> counter_map_k = {
{RDC_FI_PROF_ELAPSED_CYCLES, "GRBM_COUNT"},
{RDC_FI_PROF_ACTIVE_WAVES, "SQ_WAVES"},
{RDC_FI_PROF_ACTIVE_CYCLES, "SQ_BUSY_CU_CYCLES"},
{RDC_FI_PROF_CU_OCCUPANCY, "CU_OCCUPANCY"},
{RDC_FI_PROF_CU_UTILIZATION, "CU_UTILIZATION"},
{RDC_FI_PROF_FETCH_SIZE, "FETCH_SIZE"},
{RDC_FI_PROF_WRITE_SIZE, "WRITE_SIZE"},
{RDC_FI_PROF_FLOPS_16, "TOTAL_16_OPS"},
{RDC_FI_PROF_FLOPS_32, "TOTAL_32_OPS"},
{RDC_FI_PROF_FLOPS_64, "TOTAL_64_OPS"},
// fields below require special handling
{RDC_FI_PROF_GFLOPS_16, "TOTAL_16_OPS"},
{RDC_FI_PROF_GFLOPS_32, "TOTAL_32_OPS"},
{RDC_FI_PROF_GFLOPS_64, "TOTAL_64_OPS"},
{RDC_FI_PROF_MEMR_BW_KBPNS, "FETCH_SIZE"},
{RDC_FI_PROF_MEMW_BW_KBPNS, "WRITE_SIZE"},
};
/// Common interface for RocP tests and samples
class RdcRocpBase {
typedef std::pair<uint32_t, rdc_field_t> pair_gpu_field_t;
typedef struct session_info_t {
rocmtools_session_id_t id{};
std::chrono::
time_point<std::chrono::system_clock, std::chrono::nanoseconds>
start_time;
std::chrono::
time_point<std::chrono::system_clock, std::chrono::nanoseconds>
stop_time;
} session_info_t;
public:
RdcRocpBase();
RdcRocpBase(const RdcRocpBase&) = default;
RdcRocpBase(RdcRocpBase&&) = delete;
RdcRocpBase& operator=(const RdcRocpBase&) = delete;
RdcRocpBase& operator=(RdcRocpBase&&) = delete;
~RdcRocpBase();
/**
* @brief Lookup ROCProfiler counter
*
* @param[in] field An existing field already added to sessions dictionary
* @param[out] value A pointer that will be populated with returned value
*
* @retval ::ROCMTOOLS_STATUS_SUCCESS The function has been executed
* successfully.
*/
rdc_status_t rocp_lookup(pair_gpu_field_t gpu_field, double* value);
/**
* @brief Destroy ROCmTools session responsible for monitoring a given
* field
*
* @details While rocmtools supports multiple fields per ID - it has a
* limit to how many counters it can query internally.
* To avoid concerning ourselves with said limit, we limit each session to
* 1 field.
* In the future this can be optimized to allow for multiple fields per
* session.
*
* @param[in] field A field to start monitoring
*
* @retval ::ROCMTOOLS_STATUS_SUCCESS The function has been executed
* successfully.
*/
rdc_status_t create_session(pair_gpu_field_t gpu_field);
/**
* @brief Destroy ROCmTools session responsible for monitoring a given
* field
*
* @param[in] field A field to stop monitoring
*
* @retval ::ROCMTOOLS_STATUS_SUCCESS The function has been executed
* successfully.
*/
rdc_status_t destroy_session(pair_gpu_field_t gpu_field);
protected:
private:
std::map<pair_gpu_field_t, session_info_t> sessions;
/**
* @brief Convert from rocmtools status into RDC status
*/
rdc_status_t Rocp2RdcError(rocmtools_status_t rocm_status);
};
} // namespace rdc
} // namespace amd
#endif // RDC_MODULES_RDC_ROCP_RDCROCPBASE_H_
projects/rdc/include/rdc_modules/rdc_rocp/RdcTelemetryLib.h
+27
View File
@@ -0,0 +1,27 @@
/*
Copyright (c) 2021 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef RDC_MODULES_RDC_DIAGNOSTIC_RDCDIAGNOSTICLIB_H_
#define RDC_MODULES_RDC_DIAGNOSTIC_RDCDIAGNOSTICLIB_H_
#include "rdc/rdc.h"
#include "rdc_lib/RdcTelemetryLibInterface.h"
#endif // RDC_MODULES_RDC_DIAGNOSTIC_RDCDIAGNOSTICLIB_H_