AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GPU HW Counters via rocprofiler (#84)

Browse Source 
* Initial support for GPU hardware counters

* Update find modules for roctracer and rocprofiler

- /opt/rocm/{rocprofiler,roctracer} path is deprecated so tweak search procedure

* Improve ConfigCPack for MPI

* Update rocprofiler

- rocm_metrics()
- minor cleanup

* Update rocm find modules

* declare rocm_metrics + call in omnitrace-avail

* relocate omnitrace-launch-compiler

* REALPATH and find_modules

* Examples cmake (may drop)

* omnitrace-avail

- hw_counter categories
- init rocm

* setenv updates for rocprofiler in library.cpp and dl.cpp

* get_rocm_events config

* gpu::hip_device_count()

* rocm_metrics returns hardware_counters::info

* - relocated library/components/roctracer_callbacks.* to library/roctracer.*
- relocated library/components/rocprofiler.* to library/rocprofiler.*
- cleaned up rocprofiler.hpp
- added perfetto output of rocprofiler
- added timemory output of rocprofiler
- renamed omni.roctracer thread to roctracer.hip
- added roctracer.hsa thread name
- updated timemory submodule to support std::variant
- updated timemory submodule to support = in config value
- updated timemory submodule to support standalone storage
- updated timemory submodule to support new hw counter apis
- updated timemory submodule to prevent label/description caching in data_tracker

* update omnitrace-avail info_type generation

* Update timemory submodule

* rocprofiler component

* cmake formatting

* omnitrace-avail handle no GPUs

- Add -c command-line option for --categories
- support verbosity

* hsa_rsrc_factory throws exceptions

- throw exceptions to avoid aborting on HSA_STATUS_ERROR_NOT_INITIALIZED when advantageous
- removed duplicate specialization of is_available for component::rocprofiler

* rocprofiler symbols for when disabled

* Fix warning in omnitrace-avail

- std::stringstream from initializer list would use explicit constructor

* Fix finalization after settings are deleted

* Reorganized rocprofiler source

* Updated formatting

* Miscellaneous tweaks

- added using statements from timemory
- tweaked the main and thread bundle names
- fixed timemory header includes

[ROCm/rocprofiler-systems commit: 4208b5654c]
This commit is contained in:
Jonathan R. Madsen
2022-07-17 21:52:09 -05:00
committed by GitHub
parent aa7f2f63af
commit 7d1989f5a4
51 changed files with 3973 additions and 390 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocprofiler-systems/source/bin/omnitrace-avail/avail.cpp
+58 -10
View File
@@ -30,6 +30,8 @@
#include "info_type.hpp"
#include "library/config.hpp"
#include "library/gpu.hpp"
#include "library/rocprofiler.hpp"
#include <timemory/components.hpp>
#include <timemory/components/definition.hpp>
@@ -57,6 +59,12 @@
#include <utility>
#include <vector>
#if defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0
# include <hip/hip_runtime.h>
#elif !defined(OMNITRACE_USE_HIP)
# define OMNITRACE_USE_HIP 0
#endif
#if defined(TIMEMORY_UNIX)
# include <sys/ioctl.h> // ioctl() and TIOCGWINSZ
# include <unistd.h> // for STDOUT_FILENO
@@ -98,6 +106,8 @@ void
write_hw_counter_info(std::ostream&, const array_t<bool, N>& = {},
const array_t<bool, N>& = {}, const array_t<string_t, N>& = {});
int gpu_count = 0;
//--------------------------------------------------------------------------------------//
int
@@ -125,6 +135,8 @@ main(int argc, char** argv)
}
}
}
_category_options.emplace("hw_counters::CPU");
_category_options.emplace("hw_counters::GPU");
array_t<bool, TOTAL> options = { false, false, false, false, false, false, false };
array_t<string_t, TOTAL> fields = {};
@@ -164,6 +176,11 @@ main(int argc, char** argv)
parser.add_argument({ "--debug" }, "Enable debug messages")
.max_count(1)
.action([](parser_t& p) { debug_msg = p.get<bool>("debug"); });
parser.add_argument({ "--verbose" }, "Enable informational messages")
.max_count(1)
.action([](parser_t& p) {
verbose_level = (p.get_count("verbose") == 0) ? 1 : p.get<int>("verbose");
});
parser.add_argument({ "-a", "--all" }, "Print all available info")
.max_count(1)
.action([&](parser_t& p) {
@@ -297,7 +314,7 @@ main(int argc, char** argv)
"Display the output filename for the component")
.max_count(1);
parser
.add_argument({ "--categories" },
.add_argument({ "-c", "--categories" },
"Display the category information (use --list-categories to see "
"the available categories)")
.dtype("string")
@@ -399,6 +416,20 @@ main(int argc, char** argv)
return EXIT_FAILURE;
}
#if OMNITRACE_USE_HIP > 0
// initialize HIP and call rocm_metrics() which add choices to OMNITRACE_ROCM_EVENTS
// setting
auto _status = hipGetDeviceCount(&gpu_count);
if(gpu_count > 0 && _status == hipSuccess)
{
(void) omnitrace::rocprofiler::rocm_metrics();
}
else
{
verbprintf(0, "No HIP devices found. GPU HW counters will not be available\n");
}
#endif
auto _parser_set_if_exists = [&parser](auto& _var, const std::string& _opt) {
using Tp = decay_t<decltype(_var)>;
if(parser.exists(_opt)) _var = parser.get<Tp>(_opt);
@@ -923,10 +954,13 @@ write_hw_counter_info(std::ostream& os, const array_t<bool, N>& options,
static_assert(N >= num_hw_counter_options,
"Error! Too few hw counter options + fields");
using width_type = array_t<int64_t, N>;
using width_bool = array_t<bool, N>;
using width_type = array_t<int64_t, N>;
using width_bool = array_t<bool, N>;
using hwcounter_info_t = std::vector<tim::hardware_counters::info>;
auto _papi_events = tim::papi::available_events_info();
auto _rocm_events =
(gpu_count > 0) ? omnitrace::rocprofiler::rocm_metrics() : hwcounter_info_t{};
auto _process_counters = [](auto& _events, int32_t _offset) {
for(auto& itr : _events)
@@ -939,13 +973,25 @@ write_hw_counter_info(std::ostream& os, const array_t<bool, N>& options,
int32_t _offset = 0;
_offset += _process_counters(_papi_events, _offset);
_offset += _process_counters(_rocm_events, _offset);
using hwcounter_info_t = std::vector<tim::hardware_counters::info>;
auto fields = std::vector<hwcounter_info_t>{ _papi_events };
auto subcategories = std::vector<std::string>{ "CPU", "GPU", "" };
array_t<string_t, N> _labels = { "HARDWARE COUNTER", "AVAILABLE", "SUMMARY",
auto fields = std::vector<hwcounter_info_t>{ _papi_events, _rocm_events };
auto subcategories = std::vector<std::string>{ "CPU", "GPU", "" };
array_t<string_t, N> _labels = { "HARDWARE COUNTER", "AVAILABLE", "SUMMARY",
"DESCRIPTION" };
array_t<bool, N> _center = { false, true, false, false };
array_t<bool, N> _center = { false, true, false, false };
for(size_t i = 0; i < subcategories.size(); ++i)
{
if(i >= fields.size()) break;
if(!category_view.empty() && category_view.count(subcategories.at(i)) == 0 &&
category_view.count(std::string{ "hw_counters::" } + subcategories.at(i)) == 0)
fields.at(i).clear();
if(!is_category_selected(subcategories.at(i)) &&
!is_category_selected(std::string{ "hw_counters::" } + subcategories.at(i)))
fields.at(i).clear();
if(fields.at(i).empty()) subcategories.at(i).clear();
}
width_type _widths;
width_bool _wusing;
@@ -986,14 +1032,15 @@ write_hw_counter_info(std::ostream& os, const array_t<bool, N>& options,
os << "\n" << banner(_widths, _wusing, '-');
size_t nitr = 0;
size_t nout = 0;
for(const auto& fitr : fields)
{
auto idx = nitr++;
if(idx < subcategories.size())
{
if(!markdown && idx != 0) os << banner(_widths, _wusing, '-');
if(subcategories.at(idx).length() > 0)
if(!markdown && nout != 0) os << banner(_widths, _wusing, '-');
if(!subcategories.at(idx).empty())
{
os << global_delim;
if(options[0])
@@ -1008,6 +1055,7 @@ write_hw_counter_info(std::ostream& os, const array_t<bool, N>& options,
}
os << "\n";
if(!markdown) os << banner(_widths, _wusing, '-');
++nout;
}
}
else
projects/rocprofiler-systems/source/bin/omnitrace-avail/common.cpp
+7 -3
View File
@@ -38,7 +38,6 @@ bool alphabetical = false;
bool available_only = false;
bool all_info = false;
bool force_brief = false;
bool debug_msg = false;
bool case_insensitive = false;
bool regex_hl = false;
bool expand_keys = false;
@@ -50,7 +49,11 @@ int32_t padding = 4;
str_vec_t regex_keys = {};
str_vec_t category_regex_keys = {};
str_set_t category_view = {};
std::stringstream lerr = {};
std::stringstream lerr{};
bool debug_msg = tim::get_env<bool>("OMNITRACE_DEBUG_AVAIL", settings::debug());
int32_t verbose_level =
tim::get_env<int32_t>("OMNITRACE_VERBOSE_AVAIL", settings::verbose());
// explicit setting names to exclude
std::set<std::string> settings_exclude = {
@@ -321,7 +324,8 @@ process_categories(parser_t& p, const str_set_t& _category_options)
if(_category_options.count(itr) == 0)
{
if(!_is_shorthand("component") && !_is_shorthand("settings"))
if(!_is_shorthand("component") && !_is_shorthand("settings") &&
!_is_shorthand("hw_counters"))
throw std::runtime_error(
itr + " is not a valid category. Use --list-categories to view "
"valid categories");
projects/rocprofiler-systems/source/bin/omnitrace-avail/common.hpp
+34
View File
@@ -99,6 +99,7 @@ extern int32_t max_width;
extern int32_t num_cols;
extern int32_t min_width;
extern int32_t padding;
extern int32_t verbose_level;
extern str_vec_t regex_keys;
extern str_vec_t category_regex_keys;
extern str_set_t category_view;
@@ -143,3 +144,36 @@ remove(std::string inp, const std::set<std::string>& entries);
bool
file_exists(const std::string&);
// control debug printf statements
#define errprintf(LEVEL, ...) \
{ \
if(werror || LEVEL < 0) \
{ \
if(debug_msg || verbose_level >= LEVEL) \
fprintf(stderr, "[omnitrace][avail] Error! " __VA_ARGS__); \
char _buff[FUNCNAMELEN]; \
sprintf(_buff, "[omnitrace][avail] Error! " __VA_ARGS__); \
throw std::runtime_error(std::string{ _buff }); \
} \
else \
{ \
if(debug_msg || verbose_level >= LEVEL) \
fprintf(stderr, "[omnitrace][avail] Warning! " __VA_ARGS__); \
} \
fflush(stderr); \
}
// control verbose printf statements
#define verbprintf(LEVEL, ...) \
{ \
if(debug_msg || verbose_level >= LEVEL) \
fprintf(stderr, "[omnitrace][avail] " __VA_ARGS__); \
fflush(stderr); \
}
#define verbprintf_bare(LEVEL, ...) \
{ \
if(debug_msg || verbose_level >= LEVEL) fprintf(stderr, __VA_ARGS__); \
fflush(stderr); \
}
projects/rocprofiler-systems/source/bin/omnitrace-avail/generate_config.cpp
+4 -3
View File
@@ -230,7 +230,7 @@ generate_config(std::string _config_file, const std::set<std::string>& _config_f
_ar->startNode();
(*_ar)(cereal::make_nvp("version", std::string{ OMNITRACE_VERSION_STRING }));
(*_ar)(cereal::make_nvp("date", tim::get_local_datetime("%F_%H.%M", &_time)));
settings::serialize_settings(*_ar);
settings::serialize_settings(*_ar, *_settings);
_ar->finishNode();
};
@@ -311,6 +311,7 @@ generate_config(std::string _config_file, const std::set<std::string>& _config_f
});
else
{
_settings->ordering();
std::sort(_data.begin(), _data.end(), [](auto _lhs, auto _rhs) {
auto _lomni = _lhs->get_categories().count("omnitrace") > 0;
auto _romni = _rhs->get_categories().count("omnitrace") > 0;
@@ -419,7 +420,7 @@ update_choices(std::shared_ptr<settings> _settings)
{
std::vector<info_type> _info = get_component_info<TIMEMORY_NATIVE_COMPONENTS_END>();
if(settings::verbose() >= 2 || settings::debug())
if(_settings->get_verbose() >= 2 || _settings->get_debug())
printf("[omnitrace-avail] # of component found: %zu\n", _info.size());
_info.erase(std::remove_if(_info.begin(), _info.end(),
@@ -442,7 +443,7 @@ update_choices(std::shared_ptr<settings> _settings)
_component_choices.reserve(_info.size());
for(const auto& itr : _info)
_component_choices.emplace_back(itr.id_type());
if(settings::verbose() >= 2 || settings::debug())
if(_settings->get_verbose() >= 2 || _settings->get_debug())
printf("[omnitrace-avail] # of component choices: %zu\n",
_component_choices.size());
_settings->find("OMNITRACE_TIMEMORY_COMPONENTS")
projects/rocprofiler-systems/source/bin/omnitrace-avail/info_type.cpp
+1
View File
@@ -30,6 +30,7 @@
#include "library/components/mpi_gotcha.hpp"
#include "library/components/omnitrace.hpp"
#include "library/components/pthread_gotcha.hpp"
#include "library/components/rocprofiler.hpp"
#include "library/components/roctracer.hpp"
#include "library/components/user_region.hpp"