[rocprofiler-systems] Update logging to use spdlog library (#2428)
## Motivation - Structured logging with proper log levels (TRACE, DEBUG, INFO, WARNING, ERROR, CRITICAL) - Better performance through compile-time formatting - Consistent formatting using fmt library - Runtime log level control via arguments and environment variables - Easier maintenance and debugging capabilities ## Technical Details - Added spdlog as a submodule and integrated it into CMake build system - Created new `rocprofiler-systems-logger` library wrapping spdlog functionality - Replaced custom logging macros (`ROCPROFSYS_VERBOSE`, `ROCPROFSYS_DEBUG`, `ROCPROFSYS_FATAL`, `ROCPROFSYS_REQUIRE`, `ROCPROFSYS_CI_THROW`, etc.) with spdlog equivalents (`LOG_DEBUG`, `LOG_WARNING`, `LOG_CRITICAL`, etc.) - Implemented log level control through command-line arguments and environment variables - Converted assertion macros to proper error handling with exceptions and std::abort()
Šī revīzija ir iekļauta:
Milan Radosavljevic
revīziju iesūtīja
GitHub
GitHub
vecāks
499127c0b9
revīzija
318d13870f
@@ -98,3 +98,7 @@
|
||||
[submodule "projects/rocprofiler-systems/external/filesystem"]
|
||||
path = projects/rocprofiler-systems/external/filesystem
|
||||
url = https://github.com/gulrak/filesystem.git
|
||||
[submodule "projects/rocprofiler-systems/external/spdlog"]
|
||||
path = projects/rocprofiler-systems/external/spdlog
|
||||
url = https://github.com/gabime/spdlog.git
|
||||
tag = v1.16.0
|
||||
|
||||
@@ -9,6 +9,9 @@ Full documentation for ROCm Systems Profiler is available at [https://rocm.docs.
|
||||
### Added
|
||||
|
||||
- Documentation for `--trace-legacy` / `-L` CLI flag for direct tracing mode.
|
||||
- Added dependency to `spdlog` library.
|
||||
- Added environment variable `ROCPROFSYS_LOG_LEVEL` which control level of logging.
|
||||
- Available log levels: `critical`, `error`, `warning`, `info`(default), `debug`, `trace` and `off`.
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -18,6 +21,7 @@ Full documentation for ROCm Systems Profiler is available at [https://rocm.docs.
|
||||
- `--trace` / `-T` CLI flag enables tracing with cached mode by default.
|
||||
- `--trace-legacy` / `-L` CLI flag enables legacy direct mode for tracing.
|
||||
- Changed thread storage allocation from a hard-coded 4096-element array to a compile-time computed size derived from the ROCPROFSYS_MAX_THREADS configuration flag.
|
||||
- Changed logging module to use `spdlog` library.
|
||||
|
||||
### Resolved issues
|
||||
|
||||
@@ -30,6 +34,7 @@ Full documentation for ROCm Systems Profiler is available at [https://rocm.docs.
|
||||
### Deprecated
|
||||
|
||||
- `ROCPROFSYS_USE_PERFETTO` environment variable (use `ROCPROFSYS_TRACE`).
|
||||
- `ROCPROFSYS_VERBOSE` and `ROCPROFSYS_DEBUG` environment variables (use `ROCPROFSYS_LOG_LEVEL`).
|
||||
|
||||
## ROCm Systems Profiler 1.3.0 for ROCm 7.2.0
|
||||
|
||||
|
||||
@@ -236,6 +236,11 @@ rocprofiler_systems_add_option(ROCPROFSYS_BUILD_CODECOV "Build for code coverage
|
||||
rocprofiler_systems_add_option(ROCPROFSYS_INSTALL_PERFETTO_TOOLS
|
||||
"Install perfetto tools (i.e. traced, perfetto, etc.)" OFF
|
||||
)
|
||||
|
||||
rocprofiler_systems_add_option(ROCPROFSYS_BUILD_SPDLOG
|
||||
"Enable building spdlog library internally" ON
|
||||
)
|
||||
|
||||
rocprofiler_systems_add_option(ROCPROFSYS_BUILD_SQLITE3
|
||||
"Enable building sqlite3 library internally" ON
|
||||
)
|
||||
|
||||
@@ -62,6 +62,9 @@ rocprofiler_systems_add_interface_library(rocprofiler-systems-sqlite3
|
||||
rocprofiler_systems_add_interface_library(rocprofiler-systems-json
|
||||
"Use nlohmann/json for json data handling"
|
||||
)
|
||||
rocprofiler_systems_add_interface_library(rocprofiler-systems-spdlog
|
||||
"Provides spdlog library"
|
||||
)
|
||||
rocprofiler_systems_add_interface_library(rocprofiler-systems-timemory
|
||||
"Provides timemory libraries"
|
||||
)
|
||||
@@ -675,6 +678,14 @@ include(Perfetto)
|
||||
|
||||
include(SQLite3)
|
||||
|
||||
# ----------------------------------------------------------------------------------------#
|
||||
#
|
||||
# Spdlog
|
||||
#
|
||||
# ----------------------------------------------------------------------------------------#
|
||||
|
||||
include(Spdlog)
|
||||
|
||||
# ----------------------------------------------------------------------------------------#
|
||||
#
|
||||
# NlohmannJson
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
include_guard(GLOBAL)
|
||||
|
||||
if(ROCPROFSYS_BUILD_SPDLOG)
|
||||
message(STATUS "Building spdlog from source!")
|
||||
|
||||
include(FetchContent)
|
||||
|
||||
rocprofiler_systems_checkout_git_submodule(
|
||||
RELATIVE_PATH external/spdlog
|
||||
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
|
||||
REPO_URL https://github.com/gabime/spdlog.git
|
||||
TEST_FILE CMakeLists.txt
|
||||
REPO_BRANCH "v1.16.0"
|
||||
)
|
||||
|
||||
FetchContent_Declare(spdlog SOURCE_DIR ${PROJECT_SOURCE_DIR}/external/spdlog)
|
||||
|
||||
set(SPDLOG_BUILD_SHARED OFF CACHE BOOL "" FORCE)
|
||||
set(SPDLOG_BUILD_EXAMPLE OFF CACHE BOOL "" FORCE)
|
||||
set(SPDLOG_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
||||
set(SPDLOG_INSTALL OFF CACHE BOOL "" FORCE)
|
||||
set(SPDLOG_FMT_EXTERNAL OFF CACHE BOOL "" FORCE)
|
||||
|
||||
# Spdlog workaround for building static library
|
||||
set(_ROCPROFSYS_BUILD_SHARED_LIBS_BACKUP ${BUILD_SHARED_LIBS})
|
||||
set(BUILD_SHARED_LIBS OFF)
|
||||
|
||||
FetchContent_MakeAvailable(spdlog)
|
||||
|
||||
set(BUILD_SHARED_LIBS ${_ROCPROFSYS_BUILD_SHARED_LIBS_BACKUP})
|
||||
unset(_ROCPROFSYS_BUILD_SHARED_LIBS_BACKUP)
|
||||
|
||||
target_link_libraries(rocprofiler-systems-spdlog INTERFACE spdlog::spdlog)
|
||||
else()
|
||||
message(STATUS "Using system spdlog library")
|
||||
find_package(spdlog REQUIRED)
|
||||
target_link_libraries(rocprofiler-systems-spdlog INTERFACE spdlog::spdlog)
|
||||
endif()
|
||||
@@ -100,6 +100,7 @@ Required third-party packages
|
||||
|
||||
* `libunwind <https://www.nongnu.org/libunwind/>`_ for call-stack sampling
|
||||
* `SQLite <https://github.com/sqlite/sqlite>`_ for database output
|
||||
* `spdlog <https://github.com/gabime/spdlog>`_ for logging
|
||||
|
||||
Any of the third-party packages required by Dyninst, along with Dyninst itself, can be built and installed
|
||||
during the ROCm Systems Profiler build. The following list indicates the package, the version,
|
||||
@@ -112,6 +113,7 @@ while Dyninst requires TBB), and the CMake option to build the package alongside
|
||||
"Dyninst", "13.0", "ROCm Systems Profiler", "``ROCPROFSYS_BUILD_DYNINST`` (default: OFF)"
|
||||
"Libunwind", "", "ROCm Systems Profiler", "``ROCPROFSYS_BUILD_LIBUNWIND`` (default: ON)"
|
||||
"Nlohmann/JSON", "", "ROCm Systems Profiler", "``ROCPROFSYS_BUILD_NLOHMANN_JSON`` (default: ON)"
|
||||
"spdlog", "", "ROCm Systems Profiler", "``ROCPROFSYS_BUILD_SPDLOG`` (default: ON)"
|
||||
"SQLite", "", "ROCm Systems Profiler", "``ROCPROFSYS_BUILD_SQLITE`` (default: OFF)"
|
||||
"TBB", "2018.6", "Dyninst", "``ROCPROFSYS_BUILD_TBB`` (default: OFF)"
|
||||
"ElfUtils", "0.178", "Dyninst", "``ROCPROFSYS_BUILD_ELFUTILS`` (default: OFF)"
|
||||
|
||||
+1
Submodule projects/rocprofiler-systems/external/spdlog added at 486b55554f
+2
-2
@@ -51,7 +51,7 @@ struct component_categories
|
||||
};
|
||||
(void) _cleanup; // unused but set if sizeof...(Tp) == 0
|
||||
|
||||
TIMEMORY_FOLD_EXPRESSION(_v.emplace(TIMEMORY_JOIN(
|
||||
ROCPROFSYS_FOLD_EXPRESSION(_v.emplace(TIMEMORY_JOIN(
|
||||
"::", "component", _cleanup(rocprofsys::utility::demangle<Tp>(), "tim::"))));
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@ struct component_categories<void>
|
||||
template <size_t... Idx>
|
||||
void operator()(std::set<std::string>& _v, std::index_sequence<Idx...>) const
|
||||
{
|
||||
TIMEMORY_FOLD_EXPRESSION(component_categories<comp::enumerator_t<Idx>>{}(_v));
|
||||
ROCPROFSYS_FOLD_EXPRESSION(component_categories<comp::enumerator_t<Idx>>{}(_v));
|
||||
}
|
||||
|
||||
void operator()(std::set<std::string>& _v) const
|
||||
|
||||
@@ -400,6 +400,15 @@ parse_args(int argc, char** argv, std::vector<char*>& _env,
|
||||
std::min<int>(_cols - parser.get_help_width() - 8, 120));
|
||||
|
||||
parser.start_group("DEBUG OPTIONS", "");
|
||||
|
||||
parser.add_argument({ "--log-level" }, "Log level")
|
||||
.max_count(1)
|
||||
.dtype("string")
|
||||
.choices({ "trace", "debug", "info", "warn", "error", "critical", "off" })
|
||||
.action([&](parser_t& p) {
|
||||
update_env(_env, "ROCPROFSYS_LOG_LEVEL", p.get<std::string>("log-level"));
|
||||
});
|
||||
|
||||
parser.add_argument({ "--monochrome" }, "Disable colorized output")
|
||||
.max_count(1)
|
||||
.dtype("bool")
|
||||
@@ -410,17 +419,26 @@ parse_args(int argc, char** argv, std::vector<char*>& _env,
|
||||
update_env(_env, "ROCPROFSYS_MONOCHROME", (_monochrome) ? "1" : "0");
|
||||
update_env(_env, "MONOCHROME", (_monochrome) ? "1" : "0");
|
||||
});
|
||||
parser.add_argument({ "--debug" }, "Debug output")
|
||||
parser.add_argument({ "--debug" }, "[DEPRECATED Use --log-level=debug] Debug output")
|
||||
.max_count(1)
|
||||
.action([&](parser_t& p) {
|
||||
update_env(_env, "ROCPROFSYS_DEBUG", p.get<bool>("debug"));
|
||||
update_env(_env, "ROCPROFSYS_LOG_LEVEL", "debug");
|
||||
});
|
||||
parser.add_argument({ "-v", "--verbose" }, "Verbose output")
|
||||
parser
|
||||
.add_argument({ "-v", "--verbose" },
|
||||
"[DEPRECATED Use --log-level=trace] Verbose output")
|
||||
.count(1)
|
||||
.action([&](parser_t& p) {
|
||||
auto _v = p.get<int>("verbose");
|
||||
verbose = _v;
|
||||
update_env(_env, "ROCPROFSYS_VERBOSE", _v);
|
||||
|
||||
constexpr std::array<const char*, 5> log_levels = { "off", "info", "debug",
|
||||
"debug", "trace" };
|
||||
|
||||
auto index = std::clamp(_v + 1, 0, static_cast<int>(log_levels.size() - 1));
|
||||
update_env(_env, "ROCPROFSYS_LOG_LEVEL", log_levels[index]);
|
||||
});
|
||||
|
||||
std::string _config_file = {};
|
||||
|
||||
@@ -294,21 +294,47 @@ parse_args(int argc, char** argv, std::vector<char*>& _env)
|
||||
update_mode::REPLACE, ":", updated_envs,
|
||||
original_envs);
|
||||
});
|
||||
parser.add_argument({ "--debug" }, "Debug output")
|
||||
|
||||
parser.add_argument({ "--log-level" }, "Log level")
|
||||
.max_count(1)
|
||||
.dtype("string")
|
||||
.choices({ "trace", "debug", "info", "warn", "error", "critical", "off" })
|
||||
.action([&](parser_t& p) {
|
||||
rocprofsys::common::update_env(
|
||||
_env, "ROCPROFSYS_LOG_LEVEL", p.get<std::string>("log-level"),
|
||||
update_mode::REPLACE, ":", updated_envs, original_envs);
|
||||
});
|
||||
|
||||
parser.add_argument({ "--debug" }, "[DEPRECATED Use --log-level=debug] Debug output")
|
||||
.max_count(1)
|
||||
.action([&](parser_t& p) {
|
||||
rocprofsys::common::update_env(_env, "ROCPROFSYS_DEBUG", p.get<bool>("debug"),
|
||||
update_mode::REPLACE, ":", updated_envs,
|
||||
original_envs);
|
||||
|
||||
rocprofsys::common::update_env(_env, "ROCPROFSYS_LOG_LEVEL", "debug",
|
||||
update_mode::REPLACE, ":", updated_envs,
|
||||
original_envs);
|
||||
});
|
||||
parser.add_argument({ "-v", "--verbose" }, "Verbose output")
|
||||
parser
|
||||
.add_argument({ "-v", "--verbose" },
|
||||
"[DEPRECATED Use --log-level=trace] Verbose output")
|
||||
.count(1)
|
||||
.action([&](parser_t& p) {
|
||||
auto _v = p.get<int>("verbose");
|
||||
verbose = _v;
|
||||
|
||||
rocprofsys::common::update_env(_env, "ROCPROFSYS_VERBOSE", _v,
|
||||
update_mode::REPLACE, ":", updated_envs,
|
||||
original_envs);
|
||||
|
||||
constexpr std::array<const char*, 5> log_levels = { "off", "info", "debug",
|
||||
"debug", "trace" };
|
||||
|
||||
auto index = std::clamp(_v + 1, 0, static_cast<int>(log_levels.size() - 1));
|
||||
rocprofsys::common::update_env(_env, "ROCPROFSYS_LOG_LEVEL",
|
||||
log_levels[index], update_mode::REPLACE, ":",
|
||||
updated_envs, original_envs);
|
||||
});
|
||||
|
||||
parser.start_group("GENERAL OPTIONS",
|
||||
|
||||
@@ -42,6 +42,7 @@ target_link_libraries(
|
||||
$<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-perfetto>
|
||||
$<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-sqlite3>
|
||||
$<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-json>
|
||||
$<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-logger>
|
||||
$<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-timemory>
|
||||
$<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-elfutils>
|
||||
$<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-bfd>
|
||||
@@ -64,6 +65,7 @@ target_link_libraries(
|
||||
add_subdirectory(common)
|
||||
add_subdirectory(core)
|
||||
add_subdirectory(binary)
|
||||
add_subdirectory(logger)
|
||||
|
||||
# ------------------------------------------------------------------------------#
|
||||
#
|
||||
|
||||
@@ -36,7 +36,6 @@
|
||||
#include "core/binary/fwd.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/locking.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/utility.hpp"
|
||||
@@ -53,6 +52,8 @@
|
||||
#include <timemory/utility/join.hpp>
|
||||
#include <timemory/utility/procfs/maps.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <dlfcn.h>
|
||||
@@ -75,8 +76,7 @@ parse_line_info(const std::string& _name, bool _process_dwarf, bool _process_bfd
|
||||
auto& _bfd = _info.bfd;
|
||||
_bfd = std::make_shared<bfd_file>(_name);
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "[binary] Reading line info for '%s'...\n",
|
||||
_name.c_str());
|
||||
LOG_INFO("[binary] Reading line info for '{}'...", _name);
|
||||
|
||||
if(_bfd && _bfd->is_good())
|
||||
{
|
||||
@@ -125,8 +125,8 @@ parse_line_info(const std::string& _name, bool _process_dwarf, bool _process_bfd
|
||||
_info.sort();
|
||||
}
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "[binary] Reading line info for '%s'... %zu entries\n",
|
||||
_bfd->name.c_str(), _info.symbols.size());
|
||||
LOG_DEBUG("[binary] Reading line info for '{}'... {} entries", _bfd->name,
|
||||
_info.symbols.size());
|
||||
|
||||
return _info;
|
||||
}
|
||||
@@ -222,7 +222,7 @@ lookup_ipaddr_entry(uintptr_t _addr, unw_context_t* _context_p,
|
||||
{
|
||||
static auto _exclude_range = []() {
|
||||
auto _maps = ::tim::procfs::maps::iterate_program_headers();
|
||||
auto _exclude_range_v = std::set<address_range_t>{};
|
||||
auto _exclude_range_v = std::set<address_range>{};
|
||||
auto _insert_exclude_range = [&_maps,
|
||||
&_exclude_range_v](const std::string& _v) {
|
||||
auto _base_v = std::string_view{ filepath::basename(_v) };
|
||||
@@ -233,7 +233,7 @@ lookup_ipaddr_entry(uintptr_t _addr, unw_context_t* _context_p,
|
||||
_real_v == _v)
|
||||
{
|
||||
_exclude_range_v.emplace(
|
||||
address_range_t{ mitr.load_address, mitr.last_address });
|
||||
address_range{ mitr.load_address, mitr.last_address });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -23,11 +23,12 @@
|
||||
#include "link_map.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
|
||||
#include <timemory/utility/filepath.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <dlfcn.h>
|
||||
#include <link.h>
|
||||
@@ -138,14 +139,12 @@ get_link_map(const char* _lib, const std::string& _exclude_linked_by,
|
||||
auto _name = (!_lib) ? config::get_exe_realpath() : std::string{ _lib };
|
||||
for(const auto& itr : _fini_chain)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "[linkmap][%s]: %s\n", filepath::basename(_name),
|
||||
itr.real().c_str());
|
||||
LOG_DEBUG("[linkmap][{}]: {}", filepath::basename(_name), itr.real());
|
||||
}
|
||||
|
||||
for(const auto& itr : _excl_chain)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(3, "[linkmap][%s]: %s\n", _exclude_linked_by.c_str(),
|
||||
link_file{ itr }.real().c_str());
|
||||
LOG_DEBUG("[linkmap][{}]: {}", _exclude_linked_by, link_file{ itr }.real());
|
||||
}
|
||||
|
||||
return _fini_chain;
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
|
||||
#if !defined(TIMEMORY_USE_BFD)
|
||||
# error "BFD support not enabled"
|
||||
|
||||
@@ -9,7 +9,6 @@ set(core_sources
|
||||
${CMAKE_CURRENT_LIST_DIR}/config.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/constraint.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/cpu.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/debug.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/dynamic_library.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/exception.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gpu.cpp
|
||||
@@ -37,8 +36,6 @@ set(core_headers
|
||||
${CMAKE_CURRENT_LIST_DIR}/config.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/constraint.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/cpu.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/debug.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/defines.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/demangler.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/dynamic_library.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/exception.hpp
|
||||
@@ -50,7 +47,6 @@ set(core_headers
|
||||
${CMAKE_CURRENT_LIST_DIR}/node_info.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/perf.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/perfetto.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/redirect.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/rocprofiler-sdk.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/state.hpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/timemory.hpp
|
||||
|
||||
@@ -21,7 +21,9 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "agent_manager.hpp"
|
||||
#include "debug.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
|
||||
@@ -42,11 +44,10 @@ agent_manager::agent_manager(std::vector<std::shared_ptr<agent>> agents)
|
||||
void
|
||||
agent_manager::insert_agent(agent& _agent)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
3, "Inserting agent with device handle: %lu, and agent id: %ld, device type: %s",
|
||||
_agent.device_id,
|
||||
(_agent.type == agent_type::GPU ? _gpu_agents_cnt : _cpu_agents_cnt),
|
||||
(_agent.type == agent_type::GPU ? "GPU" : "CPU"));
|
||||
LOG_TRACE("Inserting agent with device handle: {}, and agent id: {}, device type: {}",
|
||||
_agent.device_id,
|
||||
(_agent.type == agent_type::GPU ? _gpu_agents_cnt : _cpu_agents_cnt),
|
||||
(_agent.type == agent_type::GPU ? "GPU" : "CPU"));
|
||||
|
||||
_agent.device_type_index =
|
||||
(_agent.type == agent_type::GPU ? _gpu_agents_cnt++ : _cpu_agents_cnt++);
|
||||
@@ -56,18 +57,17 @@ agent_manager::insert_agent(agent& _agent)
|
||||
const agent&
|
||||
agent_manager::get_agent_by_type_index(size_t type_index, agent_type type) const
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3, "Getting agent for type: %s, with type index: %ld\n",
|
||||
(type == agent_type::GPU) ? "GPU" : "CPU", type_index);
|
||||
LOG_TRACE("Getting agent for type: {}, with type index: {}",
|
||||
(type == agent_type::GPU ? "GPU" : "CPU"), type_index);
|
||||
auto _agent =
|
||||
std::find_if(_agents.begin(), _agents.end(), [&](const auto& agent_ptr) {
|
||||
return agent_ptr->type == type && agent_ptr->device_type_index == type_index;
|
||||
});
|
||||
if(_agent == _agents.end())
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "Agent not found for type index: " << type_index
|
||||
<< ", type: " << (type == agent_type::GPU ? "GPU" : "CPU");
|
||||
throw std::out_of_range(oss.str());
|
||||
throw std::out_of_range(
|
||||
fmt::format("Agent not found for type index: {}, type: {}", type_index,
|
||||
(type == agent_type::GPU ? "GPU" : "CPU")));
|
||||
}
|
||||
return **_agent;
|
||||
}
|
||||
@@ -75,18 +75,17 @@ agent_manager::get_agent_by_type_index(size_t type_index, agent_type type) const
|
||||
const agent&
|
||||
agent_manager::get_agent_by_id(size_t device_id, agent_type type) const
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3, "Getting agent for device id: %ld, type %s\n", device_id,
|
||||
(type == agent_type::GPU) ? "GPU" : "CPU");
|
||||
LOG_TRACE("Getting agent for device id: {}, type {}", device_id,
|
||||
(type == agent_type::GPU ? "GPU" : "CPU"));
|
||||
auto _agent =
|
||||
std::find_if(_agents.begin(), _agents.end(), [&](const auto& agent_ptr) {
|
||||
return agent_ptr->type == type && agent_ptr->device_id == device_id;
|
||||
});
|
||||
if(_agent == _agents.end())
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "Agent not found for device id: " << device_id
|
||||
<< ", type: " << (type == agent_type::GPU ? "GPU" : "CPU");
|
||||
throw std::out_of_range(oss.str());
|
||||
throw std::out_of_range(fmt::format("Agent not found for device id: {}, type: {}",
|
||||
device_id,
|
||||
(type == agent_type::GPU ? "GPU" : "CPU")));
|
||||
}
|
||||
return **_agent;
|
||||
}
|
||||
@@ -94,18 +93,17 @@ agent_manager::get_agent_by_id(size_t device_id, agent_type type) const
|
||||
const agent&
|
||||
agent_manager::get_agent_by_handle(uint64_t device_handle, agent_type type) const
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3, "Getting agent for device handle: %ld, type %s\n",
|
||||
device_handle, (type == agent_type::GPU ? "GPU" : "CPU"));
|
||||
LOG_TRACE("Getting agent for device handle: {}, type {}", device_handle,
|
||||
(type == agent_type::GPU ? "GPU" : "CPU"));
|
||||
auto _agent =
|
||||
std::find_if(_agents.begin(), _agents.end(), [&](const auto& agent_ptr) {
|
||||
return agent_ptr->type == type && agent_ptr->handle == device_handle;
|
||||
});
|
||||
if(_agent == _agents.end())
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "Agent not found for device handle: " << device_handle
|
||||
<< ", type: " << (type == agent_type::GPU ? "GPU" : "CPU");
|
||||
throw std::out_of_range(oss.str());
|
||||
throw std::out_of_range(
|
||||
fmt::format("Agent not found for device handle: {}, type: {}", device_handle,
|
||||
(type == agent_type::GPU ? "GPU" : "CPU")));
|
||||
}
|
||||
return **_agent;
|
||||
}
|
||||
@@ -113,16 +111,15 @@ agent_manager::get_agent_by_handle(uint64_t device_handle, agent_type type) cons
|
||||
const agent&
|
||||
agent_manager::get_agent_by_handle(size_t device_handle) const
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3, "Getting agent for device handle: %ld\n", device_handle);
|
||||
LOG_TRACE("Getting agent for device handle: {}", device_handle);
|
||||
auto _agent =
|
||||
std::find_if(_agents.begin(), _agents.end(), [&](const auto& agent_ptr) {
|
||||
return agent_ptr->handle == device_handle;
|
||||
});
|
||||
if(_agent == _agents.end())
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "Agent not found for device handle: " << device_handle;
|
||||
throw std::out_of_range(oss.str());
|
||||
throw std::out_of_range(
|
||||
fmt::format("Agent not found for device handle: {}", device_handle));
|
||||
}
|
||||
return **_agent;
|
||||
}
|
||||
@@ -130,8 +127,8 @@ agent_manager::get_agent_by_handle(size_t device_handle) const
|
||||
std::vector<std::shared_ptr<agent>>
|
||||
agent_manager::get_agents_by_type(agent_type type) const
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3, "Getting agent for device type: %s\n",
|
||||
type == agent_type::GPU ? "GPU" : "CPU");
|
||||
LOG_TRACE("Getting agent for device type: {}",
|
||||
(type == agent_type::GPU ? "GPU" : "CPU"));
|
||||
|
||||
std::vector<std::shared_ptr<agent>> agents;
|
||||
std::copy_if(std::begin(_agents), std::end(_agents), std::back_inserter(agents),
|
||||
|
||||
@@ -23,10 +23,11 @@
|
||||
#include "core/amd_smi.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/gpu.hpp"
|
||||
#include "timemory.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#if defined(ROCPROFSYS_USE_ROCM) && ROCPROFSYS_USE_ROCM > 0
|
||||
namespace rocprofsys
|
||||
{
|
||||
@@ -54,8 +55,8 @@ get_setting_name(std::string _v)
|
||||
__VA_ARGS__ }); \
|
||||
if(!_ret.second) \
|
||||
{ \
|
||||
ROCPROFSYS_PRINT("Warning! Duplicate setting: %s / %s\n", \
|
||||
get_setting_name(ENV_NAME).c_str(), ENV_NAME); \
|
||||
LOG_WARNING("Duplicate setting: {} / {}", get_setting_name(ENV_NAME), \
|
||||
ENV_NAME); \
|
||||
} \
|
||||
return _config->find(ENV_NAME)->second; \
|
||||
}()
|
||||
@@ -68,10 +69,10 @@ config_settings(const std::shared_ptr<settings>& _config)
|
||||
|
||||
std::string default_metrics = "busy, temp, power, mem_usage";
|
||||
// No distinction between busy and activity shown in description
|
||||
std::string jpeg_activity_support = "";
|
||||
std::string vcn_activity_support = "";
|
||||
std::string xgmi_support = "";
|
||||
std::string pcie_support = "";
|
||||
std::string jpeg_activity_support{};
|
||||
std::string vcn_activity_support{};
|
||||
std::string xgmi_support{};
|
||||
std::string pcie_support{};
|
||||
|
||||
size_t device_count = gpu::get_processor_count();
|
||||
for(size_t i = 0; i < device_count; i++)
|
||||
|
||||
@@ -32,6 +32,8 @@
|
||||
#include <timemory/utility/filepath.hpp>
|
||||
#include <timemory/utility/join.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace argparse
|
||||
@@ -216,6 +218,20 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
|
||||
_parser.start_group("DEBUG OPTIONS", "");
|
||||
|
||||
if(_data.environ_filter("log_level", _data))
|
||||
{
|
||||
_parser.add_argument({ "--log-level" }, "Log level")
|
||||
.max_count(1)
|
||||
.dtype("string")
|
||||
.choices({ "trace", "debug", "info", "warn", "error", "critical", "off" })
|
||||
.action([&](parser_t& p) {
|
||||
update_env(_data, "ROCPROFSYS_LOG_LEVEL",
|
||||
p.get<std::string>("log-level"));
|
||||
});
|
||||
|
||||
_data.processed_environs.emplace("log_level");
|
||||
}
|
||||
|
||||
if(_data.environ_filter("monochrome", _data))
|
||||
{
|
||||
_parser.add_argument({ "--monochrome" }, "Disable colorized output")
|
||||
@@ -234,24 +250,35 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
|
||||
|
||||
if(_data.environ_filter("debug", _data))
|
||||
{
|
||||
_parser.add_argument({ "--debug" }, "Debug output")
|
||||
_parser
|
||||
.add_argument({ "--debug" },
|
||||
"[DEPRECATED Use --log-level=debug] Debug output")
|
||||
.max_count(1)
|
||||
.action([&](parser_t& p) {
|
||||
update_env(_data, "ROCPROFSYS_DEBUG", p.get<bool>("debug"));
|
||||
});
|
||||
.action(
|
||||
[&](parser_t&) { update_env(_data, "ROCPROFSYS_LOG_LEVEL", "debug"); });
|
||||
|
||||
_data.processed_environs.emplace("debug");
|
||||
}
|
||||
|
||||
if(_data.environ_filter("verbose", _data))
|
||||
{
|
||||
_parser.add_argument({ "-v", "--verbose" }, "Verbose output")
|
||||
_parser
|
||||
.add_argument({ "-v", "--verbose" },
|
||||
"[DEPRECATED Use --log-level=trace] Verbose output")
|
||||
.count(1)
|
||||
.dtype("integral")
|
||||
.action([&](parser_t& p) {
|
||||
auto _v = p.get<int>("verbose");
|
||||
_data.verbose = _v;
|
||||
update_env(_data, "ROCPROFSYS_VERBOSE", _v);
|
||||
|
||||
constexpr std::array<const char*, 5> log_levels = { "off", "info",
|
||||
"debug", "debug",
|
||||
"trace" };
|
||||
|
||||
auto index =
|
||||
std::clamp(_v + 1, 0, static_cast<int>(log_levels.size() - 1));
|
||||
update_env(_data, "ROCPROFSYS_LOG_LEVEL", log_levels[index]);
|
||||
});
|
||||
|
||||
_data.processed_environs.emplace("verbose");
|
||||
@@ -1172,8 +1199,7 @@ add_group_arguments(parser_t& _parser, const std::string& _group_name, parser_da
|
||||
}
|
||||
else
|
||||
{
|
||||
TIMEMORY_PRINTF_WARNING(stderr, "Warning! Option %s (%s) is not enabled\n",
|
||||
_name.c_str(), itr->get_env_name().c_str());
|
||||
LOG_WARNING("Option {} ({}) is not enabled", _name, itr->get_env_name());
|
||||
_parser.add_argument({ _opt_name }, itr->get_description())
|
||||
.action([&](parser_t& p) {
|
||||
using namespace timemory::join;
|
||||
|
||||
@@ -21,7 +21,8 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "binary/address_range.hpp"
|
||||
#include "debug.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
@@ -36,9 +37,11 @@ address_range::address_range(uintptr_t _low, uintptr_t _high)
|
||||
: low{ _low }
|
||||
, high{ _high }
|
||||
{
|
||||
TIMEMORY_REQUIRE(high >= low)
|
||||
<< "Error! address_range high must be >= low. low=" << as_hex(low)
|
||||
<< ", high=" << as_hex(high) << "\n";
|
||||
if(high < low)
|
||||
{
|
||||
throw std::invalid_argument(fmt::format(
|
||||
"address_range high must be >= low. low={:X}, high={:X}", low, high));
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
@@ -58,6 +61,19 @@ address_range::as_string(int _depth) const
|
||||
return _ss.str();
|
||||
}
|
||||
|
||||
std::string
|
||||
address_range::as_hex() const
|
||||
{
|
||||
const auto c_width = 16;
|
||||
const auto _as_hex_util = [](auto _v, size_t _width) {
|
||||
return fmt::format("0x{:0{}x}", _v, _width);
|
||||
};
|
||||
|
||||
return (is_range()) ? fmt::format("{}-{}", _as_hex_util(low, c_width),
|
||||
_as_hex_util(high, c_width))
|
||||
: _as_hex_util(low, c_width);
|
||||
}
|
||||
|
||||
uintptr_t
|
||||
address_range::size() const
|
||||
{
|
||||
@@ -171,8 +187,10 @@ address_range&
|
||||
address_range::operator+=(address_range _v)
|
||||
{
|
||||
if(!contiguous_with(_v))
|
||||
throw exception<std::runtime_error>(
|
||||
"attempting to add two address ranges that are not contiguous");
|
||||
{
|
||||
throw std::invalid_argument(
|
||||
"Error! attempting to add two address ranges that are not contiguous");
|
||||
}
|
||||
|
||||
low = std::min(low, _v.low);
|
||||
high = std::max(high, _v.high);
|
||||
|
||||
@@ -69,6 +69,8 @@ struct address_range
|
||||
uintptr_t size() const;
|
||||
explicit operator bool() const { return is_valid(); }
|
||||
|
||||
std::string as_hex() const;
|
||||
|
||||
template <typename ArchiveT>
|
||||
void serialize(ArchiveT& ar, const unsigned)
|
||||
{
|
||||
|
||||
@@ -24,10 +24,11 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/constraint.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
#include "core/utility.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
@@ -44,8 +45,7 @@ configure_categories(bool _enable, const std::set<std::string>& _categories)
|
||||
auto _name = trait::name<Tp>::value;
|
||||
if(_categories.count(_name) > 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(3, "%s category: %s\n", (_enable) ? "Enabling" : "Disabling",
|
||||
_name);
|
||||
LOG_DEBUG("{} category: {}", _enable ? "Enabling" : "Disabling", _name);
|
||||
trait::runtime_enabled<Tp>::set(_enable);
|
||||
}
|
||||
}
|
||||
@@ -61,7 +61,7 @@ configure_categories(bool _enable, const std::set<std::string>& _categories,
|
||||
void
|
||||
configure_categories(bool _enable, const std::set<std::string>& _categories)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "%s categories...\n", (_enable) ? "Enabling" : "Disabling");
|
||||
LOG_DEBUG("{} categories...", (_enable) ? "Enabling" : "Disabling");
|
||||
|
||||
configure_categories(
|
||||
_enable, _categories,
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
#include "common/defines.h"
|
||||
#include "common/static_object.hpp"
|
||||
#include "constraint.hpp"
|
||||
#include "debug.hpp"
|
||||
#include "defines.hpp"
|
||||
#include "gpu.hpp"
|
||||
#include "mproc.hpp"
|
||||
@@ -56,6 +55,8 @@
|
||||
#include <timemory/utility/signals.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
@@ -143,8 +144,8 @@ using utility::parse_numeric_range;
|
||||
__VA_ARGS__ }); \
|
||||
if(!_ret.second) \
|
||||
{ \
|
||||
ROCPROFSYS_PRINT("Warning! Duplicate setting: %s / %s\n", \
|
||||
get_setting_name(ENV_NAME).c_str(), ENV_NAME); \
|
||||
LOG_WARNING("Duplicate setting: {} / {}", get_setting_name(ENV_NAME), \
|
||||
ENV_NAME); \
|
||||
} \
|
||||
return _config->find(ENV_NAME)->second; \
|
||||
}()
|
||||
@@ -157,8 +158,8 @@ using utility::parse_numeric_range;
|
||||
std::set<std::string>{ "custom", "rocprofsys", __VA_ARGS__ }); \
|
||||
if(!_ret.second) \
|
||||
{ \
|
||||
ROCPROFSYS_PRINT("Warning! Duplicate setting: %s / %s\n", \
|
||||
get_setting_name(ENV_NAME).c_str(), ENV_NAME); \
|
||||
LOG_WARNING("Duplicate setting: {} / {}", get_setting_name(ENV_NAME), \
|
||||
ENV_NAME); \
|
||||
} \
|
||||
return _config->find(ENV_NAME)->second; \
|
||||
}()
|
||||
@@ -174,8 +175,8 @@ using utility::parse_numeric_range;
|
||||
std::vector<std::string>{ CMD_LINE }); \
|
||||
if(!_ret.second) \
|
||||
{ \
|
||||
ROCPROFSYS_PRINT("Warning! Duplicate setting: %s / %s\n", \
|
||||
get_setting_name(ENV_NAME).c_str(), ENV_NAME); \
|
||||
LOG_WARNING("Duplicate setting: {} / {}", get_setting_name(ENV_NAME), \
|
||||
ENV_NAME); \
|
||||
} \
|
||||
return _config->find(ENV_NAME)->second; \
|
||||
}()
|
||||
@@ -191,7 +192,7 @@ auto cfg_fini_callbacks = std::vector<std::function<void()>>{};
|
||||
void
|
||||
finalize()
|
||||
{
|
||||
ROCPROFSYS_DEBUG("[rocprofsys_finalize] Disabling signal handling...\n");
|
||||
LOG_DEBUG("[rocprofsys_finalize] Disabling signal handling...");
|
||||
tim::signals::disable_signal_detection();
|
||||
_settings_are_configured() = false;
|
||||
for(const auto& itr : cfg_fini_callbacks)
|
||||
@@ -216,9 +217,11 @@ configure_settings(bool _init)
|
||||
if(is_ci_value && get_state() < State::Init)
|
||||
{
|
||||
timemory_print_demangled_backtrace<64>();
|
||||
ROCPROFSYS_THROW("config::configure_settings() called before "
|
||||
"rocprofsys_init_library. state = %s",
|
||||
std::to_string(get_state()).c_str());
|
||||
|
||||
auto message = fmt::format("config::configure_settings() called before "
|
||||
"rocprofsys_init_library. state = {}",
|
||||
static_cast<int>(get_state()));
|
||||
throw std::runtime_error(message);
|
||||
}
|
||||
|
||||
tim::manager::add_metadata("ROCPROFSYS_VERSION", ROCPROFSYS_VERSION_STRING);
|
||||
@@ -256,6 +259,15 @@ configure_settings(bool _init)
|
||||
auto _system_backend =
|
||||
tim::get_env("ROCPROFSYS_PERFETTO_BACKEND_SYSTEM", false, false);
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(std::string, "ROCPROFSYS_LOG_LEVEL",
|
||||
"Rocprofiler-systems log level", "info", "debugging",
|
||||
"advanced");
|
||||
|
||||
ROCPROFSYS_CONFIG_SETTING(std::string, "ROCPROFSYS_LOG_FILE",
|
||||
"Filename for the Rocprofiler-systems log file. Leave "
|
||||
"empty to not write to a file.",
|
||||
"rocprof-sys-log.txt", "debugging", "advanced");
|
||||
|
||||
auto _rocprofsys_debug = _config->get<bool>("ROCPROFSYS_DEBUG");
|
||||
if(_rocprofsys_debug) tim::set_env("TIMEMORY_DEBUG_SETTINGS", "1", 0);
|
||||
|
||||
@@ -876,6 +888,7 @@ configure_settings(bool _init)
|
||||
_add_rocprofsys_category(_config->find("ROCPROFSYS_CONFIG_FILE"));
|
||||
_add_rocprofsys_category(_config->find("ROCPROFSYS_DEBUG"));
|
||||
_add_rocprofsys_category(_config->find("ROCPROFSYS_VERBOSE"));
|
||||
_add_rocprofsys_category(_config->find("ROCPROFSYS_LOG_LEVEL"));
|
||||
_add_rocprofsys_category(_config->find("ROCPROFSYS_TIME_OUTPUT"));
|
||||
_add_rocprofsys_category(_config->find("ROCPROFSYS_OUTPUT_PREFIX"));
|
||||
_add_rocprofsys_category(_config->find("ROCPROFSYS_OUTPUT_PATH"));
|
||||
@@ -892,7 +905,8 @@ configure_settings(bool _init)
|
||||
{
|
||||
if(_config->get<bool>("ROCPROFSYS_CI"))
|
||||
{
|
||||
ROCPROFSYS_THROW("Error! Setting '%s' not found!", _name.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! Setting '{}' not found!", _name));
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -905,6 +919,7 @@ configure_settings(bool _init)
|
||||
_add_advanced_category("ROCPROFSYS_TEXT_OUTPUT");
|
||||
_add_advanced_category("ROCPROFSYS_DIFF_OUTPUT");
|
||||
_add_advanced_category("ROCPROFSYS_DEBUG");
|
||||
_add_advanced_category("ROCPROFSYS_LOG_LEVEL");
|
||||
_add_advanced_category("ROCPROFSYS_ENABLE_SIGNAL_HANDLER");
|
||||
_add_advanced_category("ROCPROFSYS_FLAT_PROFILE");
|
||||
_add_advanced_category("ROCPROFSYS_INPUT_EXTENSIONS");
|
||||
@@ -952,14 +967,11 @@ configure_settings(bool _init)
|
||||
|
||||
if(_paranoid > 2 && !_has_cap_sys_admin)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
0,
|
||||
"/proc/sys/kernel/perf_event_paranoid has a value of %i. "
|
||||
"Disabling PAPI (requires a value <= 2)...\n",
|
||||
_paranoid);
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
0, "In order to enable PAPI support, run 'echo N | sudo tee "
|
||||
"/proc/sys/kernel/perf_event_paranoid' where N is <= 2\n");
|
||||
LOG_WARNING("/proc/sys/kernel/perf_event_paranoid has a value of {}. "
|
||||
"Disabling PAPI (requires a value <= 2)",
|
||||
_paranoid);
|
||||
LOG_WARNING("In order to enable PAPI support, run 'echo N | sudo tee "
|
||||
"/proc/sys/kernel/perf_event_paranoid' where N is <= 2");
|
||||
trait::runtime_enabled<comp::papi_config>::set(false);
|
||||
trait::runtime_enabled<comp::papi_common<void>>::set(false);
|
||||
trait::runtime_enabled<comp::papi_array_t>::set(false);
|
||||
@@ -1019,7 +1031,7 @@ configure_settings(bool _init)
|
||||
{
|
||||
if(_config->get_suppress_config()) continue;
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "Reading config file %s\n", itr.c_str());
|
||||
LOG_DEBUG("Reading config file {}", itr);
|
||||
if(_config->read(itr) && _main_proc &&
|
||||
((_config->get<bool>("ROCPROFSYS_CI") && settings::verbose() >= 0) ||
|
||||
settings::verbose() >= 1 || settings::debug()))
|
||||
@@ -1035,8 +1047,7 @@ configure_settings(bool _init)
|
||||
}
|
||||
if(!_iss.str().empty())
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "config file '%s':\n%s\n", fitr.c_str(),
|
||||
_iss.str().c_str());
|
||||
LOG_DEBUG("config file '{}': {}", fitr, _iss.str());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1106,7 +1117,7 @@ configure_settings(bool _init)
|
||||
configure_signal_handler(_config);
|
||||
configure_disabled_settings(_config);
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "configuration complete\n");
|
||||
LOG_DEBUG("Configuration complete");
|
||||
|
||||
if(auto opt = get_setting_value<int>("ROCPROFSYS_VERBOSE"); opt) verbose_value = *opt;
|
||||
if(auto opt = get_setting_value<bool>("ROCPROFSYS_DEBUG"); opt) debug_value = *opt;
|
||||
@@ -1121,18 +1132,17 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
|
||||
auto _set = [](const std::string& _name, bool _v) {
|
||||
if(!set_setting_value(_name, _v))
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
4, "[configure_mode_settings] No configuration setting named '%s'...\n",
|
||||
_name.data());
|
||||
LOG_DEBUG("[configure_mode_settings] No configuration setting named '{}'...",
|
||||
_name);
|
||||
}
|
||||
else
|
||||
{
|
||||
bool _changed = get_setting_value<bool>(_name).value_or(!_v) != _v;
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
1 && _changed,
|
||||
"[configure_mode_settings] Overriding %s to %s in %s mode...\n",
|
||||
_name.c_str(), JOIN("", std::boolalpha, _v).c_str(),
|
||||
std::to_string(get_mode()).c_str());
|
||||
if(_changed)
|
||||
{
|
||||
LOG_WARNING("[configure_mode_settings] Overriding {} to {} in {} mode...",
|
||||
_name, _v, static_cast<int>(get_mode()));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1169,8 +1179,7 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
|
||||
if(gpu::device_count() == 0)
|
||||
{
|
||||
#if ROCPROFSYS_ROCM_VERSION > 0
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
1, "No ROCm devices were found: disabling rocm and amd_smi...\n");
|
||||
LOG_WARNING("No ROCm devices were found: disabling rocm and amd_smi...");
|
||||
#endif
|
||||
_set("ROCPROFSYS_USE_ROCM", false);
|
||||
_set("ROCPROFSYS_USE_AMD_SMI", false);
|
||||
@@ -1188,10 +1197,9 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
|
||||
{
|
||||
_force = 1;
|
||||
_message =
|
||||
JOIN("", " (forced. Previous value: '", _current_kokkosp_lib, "')");
|
||||
fmt::format(" (forced. Previous value: '{}')", _current_kokkosp_lib);
|
||||
}
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(1, "Setting KOKKOS_TOOLS_LIBS=%s%s\n",
|
||||
"librocprof-sys.so", _message.c_str());
|
||||
LOG_WARNING("Setting KOKKOS_TOOLS_LIBS={}{}", "librocprof-sys.so", _message);
|
||||
tim::set_env("KOKKOS_TOOLS_LIBS", "librocprof-sys.so", _force);
|
||||
}
|
||||
}
|
||||
@@ -1236,8 +1244,8 @@ rocprofsys_exit_action(int nsig)
|
||||
{
|
||||
tim::signals::block_signals(get_sampling_signals(),
|
||||
tim::signals::sigmask_scope::process);
|
||||
ROCPROFSYS_BASIC_PRINT("Finalizing after signal %i :: %s\n", nsig,
|
||||
signal_settings::str(static_cast<sys_signal>(nsig)).c_str());
|
||||
LOG_DEBUG("Finalizing after signal {} :: {}", nsig,
|
||||
signal_settings::str(static_cast<sys_signal>(nsig)));
|
||||
auto _handler = get_signal_handler().load();
|
||||
if(_handler) (*_handler)();
|
||||
kill(process::get_id(), nsig);
|
||||
@@ -1246,16 +1254,7 @@ rocprofsys_exit_action(int nsig)
|
||||
void
|
||||
rocprofsys_trampoline_handler(int _v)
|
||||
{
|
||||
if(get_verbose_env() >= 1)
|
||||
{
|
||||
::rocprofsys::debug::flush();
|
||||
::rocprofsys::debug::lock _debug_lk{};
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(warning);
|
||||
fprintf(::rocprofsys::debug::get_file(),
|
||||
"signal %i ignored (ROCPROFSYS_IGNORE_DYNINST_TRAMPOLINE=ON)\n", _v);
|
||||
::rocprofsys::debug::flush();
|
||||
timemory_print_demangled_backtrace<64>();
|
||||
}
|
||||
LOG_DEBUG("signal {} ignored (ROCPROFSYS_IGNORE_DYNINST_TRAMPOLINE=ON)", _v);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
@@ -1357,8 +1356,8 @@ get_sampling_signals(int64_t)
|
||||
if(get_use_sampling() && !get_use_sampling_cputime() &&
|
||||
!get_use_sampling_realtime() && !get_use_sampling_overflow())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "sampling enabled by cputime/realtime/overflow not "
|
||||
"specified. defaulting to cputime...\n");
|
||||
LOG_WARNING("Sampling enabled by cputime/realtime/overflow is not "
|
||||
"specified. Defaulting to cputime...");
|
||||
set_setting_value("ROCPROFSYS_SAMPLING_CPUTIME", true);
|
||||
}
|
||||
|
||||
@@ -1380,14 +1379,12 @@ configure_disabled_settings(const std::shared_ptr<settings>& _config)
|
||||
auto _disabled = _config->disable_category(_category);
|
||||
_config->enable(_opt);
|
||||
for(auto&& itr : _disabled)
|
||||
ROCPROFSYS_BASIC_VERBOSE(3, "[%s=OFF] disabled option :: '%s'\n",
|
||||
_opt.c_str(), itr.c_str());
|
||||
LOG_DEBUG("[{}=OFF] disabled option :: '{}'", _opt, itr);
|
||||
return false;
|
||||
}
|
||||
auto _enabled = _config->enable_category(_category);
|
||||
for(auto&& itr : _enabled)
|
||||
ROCPROFSYS_BASIC_VERBOSE(3, "[%s=ON] enabled option :: '%s'\n",
|
||||
_opt.c_str(), itr.c_str());
|
||||
LOG_DEBUG("[{}=ON] enabled option :: '{}'", _opt, itr);
|
||||
return true;
|
||||
};
|
||||
|
||||
@@ -1493,8 +1490,11 @@ handle_deprecated_setting(const std::string& _old, const std::string& _new, int
|
||||
|
||||
if(_old_setting == _config->end()) return;
|
||||
|
||||
ROCPROFSYS_CI_THROW(_new_setting == _config->end(),
|
||||
"New configuration setting not found: '%s'", _new.c_str());
|
||||
if(get_is_continuous_integration() && _new_setting == _config->end())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("New configuration setting not found: '{}'", _new));
|
||||
}
|
||||
|
||||
if(_old_setting->second->get_environ_updated() ||
|
||||
_old_setting->second->get_config_updated())
|
||||
@@ -1503,13 +1503,13 @@ handle_deprecated_setting(const std::string& _old, const std::string& _new, int
|
||||
std::array<char, 79> _v = {};
|
||||
_v.fill('=');
|
||||
_v.back() = '\0';
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "#%s#\n", _v.data());
|
||||
LOG_WARNING("#{}#", _v.data());
|
||||
};
|
||||
_separator();
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "#\n");
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "# DEPRECATION NOTICE:\n");
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "# %s is deprecated!\n", _old.c_str());
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "# Use %s instead!\n", _new.c_str());
|
||||
LOG_WARNING("#");
|
||||
LOG_WARNING("# DEPRECATION NOTICE:");
|
||||
LOG_WARNING("# {} is deprecated!", _old);
|
||||
LOG_WARNING("# Use {} instead!", _new);
|
||||
|
||||
if(!_new_setting->second->get_environ_updated() &&
|
||||
!_new_setting->second->get_config_updated())
|
||||
@@ -1522,15 +1522,13 @@ handle_deprecated_setting(const std::string& _old, const std::string& _new, int
|
||||
{
|
||||
std::string _cause =
|
||||
(_old_setting->second->get_environ_updated()) ? "environ" : "config";
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "#\n");
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "# %s :: '%s' -> '%s'\n", _new.c_str(),
|
||||
_before.c_str(), _after.c_str());
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "# via %s (%s)\n", _old.c_str(),
|
||||
_cause.c_str());
|
||||
LOG_WARNING("#");
|
||||
LOG_WARNING("# {} :: '{}' -> '{}'", _new, _before, _after);
|
||||
LOG_WARNING("# via {} ({})", _old, _cause);
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE(_verbose, "#\n");
|
||||
LOG_WARNING("#");
|
||||
_separator();
|
||||
}
|
||||
}
|
||||
@@ -1579,7 +1577,8 @@ print_banner(std::ostream& _os)
|
||||
if(!_properties.empty())
|
||||
_version_info << join::join(join::array_config{ ", ", " (", ")" }, _properties);
|
||||
|
||||
tim::log::stream(_os, tim::log::color::info()) << _banner << _version_info.str();
|
||||
_os << _banner << "\n";
|
||||
_os << _version_info.str() << "\n";
|
||||
_os << std::endl;
|
||||
}
|
||||
|
||||
@@ -1588,7 +1587,7 @@ print_settings(
|
||||
std::ostream& _ros,
|
||||
std::function<bool(const std::string_view&, const std::set<std::string>&)>&& _filter)
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT(true, "configuration:\n");
|
||||
LOG_INFO("configuration:");
|
||||
|
||||
std::stringstream _os{};
|
||||
|
||||
@@ -1700,18 +1699,19 @@ print_settings(bool _include_env)
|
||||
|
||||
if(_include_env)
|
||||
{
|
||||
std::cerr << tim::log::info;
|
||||
tim::print_env(std::cerr, [_is_rocprofsys_option](const std::string& _v) {
|
||||
std::stringstream _ss1{};
|
||||
tim::print_env(_ss1, [_is_rocprofsys_option](const std::string& _v) {
|
||||
auto _is_omni_opt = _is_rocprofsys_option(_v, std::set<std::string>{});
|
||||
if(settings::verbose() >= 2 || settings::debug()) return _is_omni_opt;
|
||||
return (_is_omni_opt && _v.find("ROCPROFSYS_SIGNAL_") != 0);
|
||||
});
|
||||
std::cerr << tim::log::flush;
|
||||
|
||||
LOG_INFO("{}", _ss1.str());
|
||||
}
|
||||
|
||||
print_settings(std::cerr, _is_rocprofsys_option);
|
||||
|
||||
fprintf(stderr, "\n");
|
||||
std::stringstream _ss2{};
|
||||
print_settings(_ss2, _is_rocprofsys_option);
|
||||
LOG_INFO("{}", _ss2.str());
|
||||
}
|
||||
|
||||
std::string&
|
||||
@@ -1771,8 +1771,8 @@ get_mode()
|
||||
for(const auto& itr : _v->second->get_choices())
|
||||
_ss << ", " << itr;
|
||||
auto _msg = (_ss.str().length() > 2) ? _ss.str().substr(2) : std::string{};
|
||||
ROCPROFSYS_THROW("[%s] invalid mode %s. Choices: %s\n", __FUNCTION__,
|
||||
_mode.c_str(), _msg.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("[{}] invalid mode {}. Choices: {}", __FUNCTION__, _mode, _msg));
|
||||
}
|
||||
return Mode::Trace;
|
||||
}
|
||||
@@ -1902,8 +1902,9 @@ get_use_sampling()
|
||||
static auto _v = get_config()->find("ROCPROFSYS_USE_SAMPLING");
|
||||
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
|
||||
#else
|
||||
ROCPROFSYS_THROW("Error! sampling was enabled but rocprof-sys was not built with "
|
||||
"libunwind support");
|
||||
throw std::runtime_error(
|
||||
"Error! sampling was enabled but rocprof-sys was not built with "
|
||||
"libunwind support");
|
||||
static bool _v = false;
|
||||
return _v;
|
||||
#endif
|
||||
@@ -2110,15 +2111,20 @@ get_category_config()
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_ABORT(
|
||||
"Error! Conflicting options ROCPROFSYS_ENABLE_CATEGORIES and "
|
||||
"ROCPROFSYS_DISABLE_CATEGORIES were both provided.");
|
||||
LOG_CRITICAL("Error! Conflicting options ROCPROFSYS_ENABLE_CATEGORIES and "
|
||||
"ROCPROFSYS_DISABLE_CATEGORIES were both provided.");
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(_enabled.size() + _disabled.size() != _avail.size(),
|
||||
"Error! Internal error for categories: %zu (enabled) + %zu "
|
||||
"(disabled) != %zu (total)\n",
|
||||
_enabled.size(), _disabled.size(), _avail.size());
|
||||
if(get_is_continuous_integration() &&
|
||||
_enabled.size() + _disabled.size() != _avail.size())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! Internal error for categories: {} (enabled) + {} "
|
||||
"(disabled) != {} (total)\n",
|
||||
_enabled.size(), _disabled.size(), _avail.size()));
|
||||
}
|
||||
|
||||
return std::make_pair(_enabled, _disabled);
|
||||
}();
|
||||
@@ -2181,38 +2187,26 @@ get_perfetto_output_filename()
|
||||
_val = _val.substr(0, _pos_ext);
|
||||
}
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2, "[get_perfetto_output_filename] Parsed: dir='%s', basename='%s', ext='%s'\n",
|
||||
_dir.c_str(), _val.c_str(), _ext.c_str());
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2, "[get_perfetto_output_filename] settings::output_path()='%s'\n",
|
||||
settings::output_path().c_str());
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2, "[get_perfetto_output_filename] settings::output_prefix()='%s'\n",
|
||||
settings::output_prefix().c_str());
|
||||
LOG_DEBUG("Parsed: dir='{}', basename='{}', ext='{}'", _dir, _val, _ext);
|
||||
LOG_DEBUG("settings::output_path()='{}'", settings::output_path());
|
||||
LOG_DEBUG("settings::output_prefix()='{}'", settings::output_prefix());
|
||||
|
||||
auto _cfg = settings::compose_filename_config{ settings::use_output_suffix(),
|
||||
settings::default_process_suffix(),
|
||||
false, _dir };
|
||||
_val = settings::compose_output_filename(_val, _ext, _cfg);
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2, "[get_perfetto_output_filename] After compose_output_filename: '%s'\n",
|
||||
_val.c_str());
|
||||
LOG_DEBUG("After compose_output_filename: '{}'", _val);
|
||||
|
||||
if(!_val.empty() && _val.at(0) != '/')
|
||||
{
|
||||
auto _result =
|
||||
settings::format(JOIN('/', "%env{PWD}%", _val), get_config()->get_tag());
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2, "[get_perfetto_output_filename] Path is relative, prepending PWD: '%s'\n",
|
||||
_result.c_str());
|
||||
LOG_DEBUG("Path is relative, prepending PWD: '{}'", _result);
|
||||
return _result;
|
||||
}
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2, "[get_perfetto_output_filename] Path is absolute, returning: '%s'\n",
|
||||
_val.c_str());
|
||||
LOG_DEBUG("Path is absolute, returning: '{}'", _val);
|
||||
return _val;
|
||||
}
|
||||
|
||||
@@ -2472,18 +2466,12 @@ get_perfetto_output_filename_with_suffix(std::string_view suffix)
|
||||
static auto _v = get_config()->find("ROCPROFSYS_PERFETTO_FILE");
|
||||
auto _val = static_cast<tim::tsettings<std::string>&>(*_v->second).get();
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(2,
|
||||
"[get_perfetto_output_filename_with_suffix] Initial "
|
||||
"ROCPROFSYS_PERFETTO_FILE='%s', suffix='%s'\n",
|
||||
_val.c_str(), std::string{ suffix }.c_str());
|
||||
LOG_DEBUG("Initial ROCPROFSYS_PERFETTO_FILE='{}', suffix='{}'", _val, suffix);
|
||||
|
||||
// If absolute path is provided, return it as-is
|
||||
if(!_val.empty() && _val.at(0) == '/')
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2,
|
||||
"[get_perfetto_output_filename_with_suffix] Absolute path, returning: '%s'\n",
|
||||
_val.c_str());
|
||||
LOG_DEBUG("Absolute path, returning: '{}'", _val);
|
||||
return _val;
|
||||
}
|
||||
|
||||
@@ -2509,14 +2497,9 @@ get_perfetto_output_filename_with_suffix(std::string_view suffix)
|
||||
bool _explicitly_set =
|
||||
(_v->second->get_environ_updated() || _v->second->get_config_updated());
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2,
|
||||
"[get_perfetto_output_filename_with_suffix] Parsed: dir='%s', basename='%s', "
|
||||
"ext='%s', explicitly_set=%s\n",
|
||||
_dir.c_str(), _val.c_str(), _ext.c_str(), _explicitly_set ? "true" : "false");
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2, "[get_perfetto_output_filename_with_suffix] settings::output_path()='%s'\n",
|
||||
settings::output_path().c_str());
|
||||
LOG_DEBUG("Parsed: dir='{}', basename='{}', ext='{}', explicitly_set={}", _dir, _val,
|
||||
_ext, _explicitly_set);
|
||||
LOG_DEBUG("settings::output_path()='{}'", settings::output_path());
|
||||
|
||||
auto _cfg = settings::compose_filename_config{
|
||||
!_explicitly_set && !suffix.empty(), // use_suffix only if not explicitly set
|
||||
@@ -2527,26 +2510,17 @@ get_perfetto_output_filename_with_suffix(std::string_view suffix)
|
||||
|
||||
_val = settings::compose_output_filename(_val, _ext, _cfg);
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(2,
|
||||
"[get_perfetto_output_filename_with_suffix] After "
|
||||
"compose_output_filename: '%s'\n",
|
||||
_val.c_str());
|
||||
LOG_DEBUG("After compose_output_filename: '{}'", _val);
|
||||
|
||||
if(!_val.empty() && _val.at(0) != '/')
|
||||
{
|
||||
auto _result =
|
||||
settings::format(JOIN('/', "%env{PWD}%", _val), get_config()->get_tag());
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(2,
|
||||
"[get_perfetto_output_filename_with_suffix] Path is "
|
||||
"relative, prepending PWD: '%s'\n",
|
||||
_result.c_str());
|
||||
LOG_DEBUG("Path is relative, prepending PWD: '{}'", _result);
|
||||
return _result;
|
||||
}
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
2,
|
||||
"[get_perfetto_output_filename_with_suffix] Path is absolute, returning: '%s'\n",
|
||||
_val.c_str());
|
||||
LOG_DEBUG("Path is absolute, returning: '{}'", _val);
|
||||
return _val;
|
||||
}
|
||||
|
||||
@@ -2599,7 +2573,7 @@ tmp_file::touch() const
|
||||
bool
|
||||
tmp_file::open(int _mode, int _perms)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
|
||||
LOG_DEBUG("Opening temporary file '{}'...", filename);
|
||||
|
||||
touch();
|
||||
m_pid = getpid();
|
||||
@@ -2611,7 +2585,7 @@ tmp_file::open(int _mode, int _perms)
|
||||
bool
|
||||
tmp_file::open(std::ios::openmode _mode)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
|
||||
LOG_DEBUG("Opening temporary file '{}'...", filename);
|
||||
|
||||
touch();
|
||||
|
||||
@@ -2624,7 +2598,7 @@ tmp_file::open(std::ios::openmode _mode)
|
||||
bool
|
||||
tmp_file::fopen(const char* _mode)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
|
||||
LOG_DEBUG("Opening temporary file '{}'...", filename);
|
||||
|
||||
touch();
|
||||
|
||||
@@ -2715,8 +2689,7 @@ tmp_file::remove()
|
||||
close();
|
||||
if(filepath::exists(filename))
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "Removing temporary file '%s'...\n",
|
||||
filename.c_str());
|
||||
LOG_DEBUG("Removing temporary file '{}'...", filename);
|
||||
auto _ret = ::remove(filename.c_str());
|
||||
return (_ret == 0);
|
||||
}
|
||||
@@ -2764,9 +2737,10 @@ get_tmp_file(std::string _basename, std::string _ext)
|
||||
|
||||
if(_fname.empty() || _fname.front() != '/')
|
||||
{
|
||||
ROCPROFSYS_THROW("Error! temporary file '%s' (based on '%s.%s') is either empty "
|
||||
"or is not an absolute path",
|
||||
_fname.c_str(), _basename.c_str(), _ext.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! temporary file '{}' (based on '{}.'{}) is either empty "
|
||||
"or is not an absolute path",
|
||||
_fname, _basename, _ext));
|
||||
}
|
||||
auto itr = _existing_files.find(_fname);
|
||||
if(itr != _existing_files.end()) return itr->second;
|
||||
@@ -2792,11 +2766,10 @@ get_causal_backend()
|
||||
} catch(std::runtime_error& _e)
|
||||
{
|
||||
auto _mode = static_cast<tim::tsettings<std::string>&>(*_v->second).get();
|
||||
ROCPROFSYS_THROW(
|
||||
"[%s] invalid causal backend %s. Choices: %s\n", __FUNCTION__, _mode.c_str(),
|
||||
throw std::runtime_error(fmt::format(
|
||||
"[{}] invalid causal backend {}. Choices: {}", __FUNCTION__, _mode,
|
||||
timemory::join::join(timemory::join::array_config{ ", ", "", "" },
|
||||
_v->second->get_choices())
|
||||
.c_str());
|
||||
_v->second->get_choices())));
|
||||
}
|
||||
return CausalBackend::Auto;
|
||||
}
|
||||
@@ -2824,11 +2797,10 @@ get_causal_mode()
|
||||
} catch(std::runtime_error& _e)
|
||||
{
|
||||
auto _mode = static_cast<tim::tsettings<std::string>&>(*_v->second).get();
|
||||
ROCPROFSYS_THROW(
|
||||
"[%s] invalid causal mode %s. Choices: %s\n", __FUNCTION__, _mode.c_str(),
|
||||
throw std::runtime_error(fmt::format(
|
||||
"[{}] invalid causal mode {}. Choices: {}", __FUNCTION__, _mode,
|
||||
timemory::join::join(timemory::join::array_config{ ", ", "", "" },
|
||||
_v->second->get_choices())
|
||||
.c_str());
|
||||
_v->second->get_choices())));
|
||||
}
|
||||
return CausalMode::Function;
|
||||
}();
|
||||
|
||||
@@ -22,13 +22,14 @@
|
||||
|
||||
#include "constraint.hpp"
|
||||
#include "config.hpp"
|
||||
#include "debug.hpp"
|
||||
#include "state.hpp"
|
||||
#include "utility.hpp"
|
||||
|
||||
#include <timemory/units.hpp>
|
||||
#include <timemory/utility/delimit.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <ratio>
|
||||
@@ -101,9 +102,10 @@ find_clock_identifier(const Tp& _v)
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_THROW("Unknown clock id %s: %s. Valid choices: %s\n", _descript,
|
||||
timemory::join::join("", _v).c_str(),
|
||||
timemory::join::join("", accepted_clock_ids).c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Unknown clock id {}: {}. Valid choices: {}", _descript,
|
||||
timemory::join::join("", _v).c_str(),
|
||||
timemory::join::join("", accepted_clock_ids).c_str()));
|
||||
}
|
||||
|
||||
void
|
||||
@@ -267,12 +269,10 @@ spec::operator()(const stages& _stages) const
|
||||
return _ret;
|
||||
};
|
||||
|
||||
ROCPROFSYS_VERBOSE(2,
|
||||
"Executing constraint spec %lu of %lu :: delay: %6.3f, "
|
||||
"duration: %6.3f, clock: %s\n",
|
||||
i, _spec.repeat, _spec.delay, _spec.duration,
|
||||
_spec.clock_id.as_string().c_str());
|
||||
|
||||
LOG_DEBUG("Executing constraint spec {} of {} :: delay: {:.3f}, "
|
||||
"duration: {:.3f}, clock: {}",
|
||||
i, _spec.repeat, _spec.delay, _spec.duration,
|
||||
_spec.clock_id.as_string());
|
||||
if(_stages.init(_spec) && _wait(_stages.wait, _spec.delay) &&
|
||||
_stages.start(_spec) && _wait(_stages.collect, _spec.duration) &&
|
||||
_stages.stop(_spec))
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
|
||||
#include "core/common.hpp"
|
||||
#include "core/containers/operators.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/exception.hpp"
|
||||
|
||||
#include <array>
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
|
||||
#include "core/common.hpp"
|
||||
#include "core/containers/c_array.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/exception.hpp"
|
||||
|
||||
#include <timemory/utility/demangle.hpp>
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#include "debug.hpp"
|
||||
#include "binary/address_range.hpp"
|
||||
#include "locking.hpp"
|
||||
#include "state.hpp"
|
||||
|
||||
#include <timemory/log/color.hpp>
|
||||
#include <timemory/process/threading.hpp>
|
||||
#include <timemory/utility/filepath.hpp>
|
||||
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace debug
|
||||
{
|
||||
namespace
|
||||
{
|
||||
struct source_location_history
|
||||
{
|
||||
std::array<source_location, 10> data = {};
|
||||
size_t size = 0;
|
||||
};
|
||||
|
||||
const std::string&
|
||||
get_file_name()
|
||||
{
|
||||
static auto _fname = tim::get_env<std::string>("ROCPROFSYS_LOG_FILE", "");
|
||||
return _fname;
|
||||
}
|
||||
|
||||
std::atomic<FILE*>&
|
||||
get_file_pointer()
|
||||
{
|
||||
static auto _v = std::atomic<FILE*>{ []() {
|
||||
const auto& _fname = get_file_name();
|
||||
if(!_fname.empty()) tim::log::monochrome() = true;
|
||||
return (_fname.empty())
|
||||
? stderr
|
||||
: filepath::fopen(
|
||||
settings::format(_fname, filepath::basename(filepath::realpath(
|
||||
"/proc/self/exe", nullptr, false))),
|
||||
"w");
|
||||
}() };
|
||||
return _v;
|
||||
}
|
||||
|
||||
auto&
|
||||
get_source_location_history()
|
||||
{
|
||||
static thread_local auto _v = source_location_history{};
|
||||
return _v;
|
||||
}
|
||||
|
||||
auto _protect_lock = std::atomic<bool>{ false };
|
||||
auto _protect_unlock = std::atomic<bool>{ false };
|
||||
} // namespace
|
||||
|
||||
void
|
||||
set_source_location(source_location&& _v)
|
||||
{
|
||||
auto& _hist = get_source_location_history();
|
||||
auto _idx = _hist.size++;
|
||||
_hist.data.at(_idx % _hist.data.size()) = _v;
|
||||
}
|
||||
|
||||
lock::lock()
|
||||
: m_lk{ tim::type_mutex<decltype(std::cerr), TIMEMORY_API, 1, locking::atomic_mutex>(),
|
||||
std::defer_lock }
|
||||
{
|
||||
if(!m_lk.owns_lock() && !_protect_lock)
|
||||
{
|
||||
_protect_lock.store(true);
|
||||
push_thread_state(ThreadState::Internal);
|
||||
m_lk.lock();
|
||||
_protect_lock.store(false);
|
||||
}
|
||||
}
|
||||
|
||||
lock::~lock()
|
||||
{
|
||||
if(m_lk.owns_lock() && !_protect_unlock)
|
||||
{
|
||||
_protect_unlock.store(true);
|
||||
m_lk.unlock();
|
||||
pop_thread_state();
|
||||
_protect_unlock.store(false);
|
||||
}
|
||||
}
|
||||
|
||||
FILE*
|
||||
get_file()
|
||||
{
|
||||
return get_file_pointer();
|
||||
}
|
||||
|
||||
void
|
||||
close_file()
|
||||
{
|
||||
if(get_file() != stderr)
|
||||
{
|
||||
auto* _file = get_file_pointer().load();
|
||||
get_file_pointer().store(stderr);
|
||||
fclose(_file);
|
||||
// Write the trace into a file.
|
||||
if(get_verbose() >= 0)
|
||||
operation::file_output_message<tim::project::rocprofsys>{}(
|
||||
get_file_name(), std::string{ "debug" });
|
||||
}
|
||||
}
|
||||
|
||||
int64_t
|
||||
get_tid()
|
||||
{
|
||||
static thread_local auto _v = threading::get_id();
|
||||
return _v;
|
||||
}
|
||||
} // namespace debug
|
||||
|
||||
template <typename Tp>
|
||||
std::string
|
||||
as_hex(Tp _v, size_t _width)
|
||||
{
|
||||
std::stringstream _ss;
|
||||
_ss.fill('0');
|
||||
_ss << "0x" << std::hex << std::setw(_width) << _v;
|
||||
return _ss.str();
|
||||
}
|
||||
|
||||
template <>
|
||||
std::string
|
||||
as_hex<address_range_t>(address_range_t _v, size_t _width)
|
||||
{
|
||||
return (_v.is_range()) ? JOIN('-', as_hex(_v.low, _width), as_hex(_v.high, _width))
|
||||
: as_hex(_v.low, _width);
|
||||
}
|
||||
|
||||
template std::string as_hex<int32_t>(int32_t, size_t);
|
||||
template std::string as_hex<uint32_t>(uint32_t, size_t);
|
||||
template std::string as_hex<int64_t>(int64_t, size_t);
|
||||
template std::string as_hex<uint64_t>(uint64_t, size_t);
|
||||
template std::string
|
||||
as_hex<void*>(void*, size_t);
|
||||
} // namespace rocprofsys
|
||||
@@ -1,717 +0,0 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "defines.hpp"
|
||||
#include "exception.hpp"
|
||||
#include "locking.hpp"
|
||||
|
||||
#include <timemory/api.hpp>
|
||||
#include <timemory/backends/dmp.hpp>
|
||||
#include <timemory/backends/process.hpp>
|
||||
#include <timemory/backends/threading.hpp>
|
||||
#include <timemory/log/logger.hpp>
|
||||
#include <timemory/mpl/concepts.hpp>
|
||||
#include <timemory/signals/signal_handlers.hpp>
|
||||
#include <timemory/utility/backtrace.hpp>
|
||||
#include <timemory/utility/locking.hpp>
|
||||
#include <timemory/utility/utility.hpp>
|
||||
|
||||
#include <array>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
inline namespace config
|
||||
{
|
||||
bool
|
||||
get_debug() ROCPROFSYS_HOT;
|
||||
|
||||
int
|
||||
get_verbose() ROCPROFSYS_HOT;
|
||||
|
||||
bool
|
||||
get_debug_env() ROCPROFSYS_HOT;
|
||||
|
||||
int
|
||||
get_verbose_env() ROCPROFSYS_HOT;
|
||||
|
||||
bool
|
||||
get_is_continuous_integration() ROCPROFSYS_HOT;
|
||||
|
||||
bool
|
||||
get_debug_tid() ROCPROFSYS_HOT;
|
||||
|
||||
bool
|
||||
get_debug_pid() ROCPROFSYS_HOT;
|
||||
} // namespace config
|
||||
|
||||
namespace debug
|
||||
{
|
||||
struct source_location
|
||||
{
|
||||
std::string_view function = {};
|
||||
std::string_view file = {};
|
||||
int line = 0;
|
||||
};
|
||||
//
|
||||
void
|
||||
set_source_location(source_location&&);
|
||||
//
|
||||
FILE*
|
||||
get_file();
|
||||
//
|
||||
void
|
||||
close_file();
|
||||
//
|
||||
int64_t
|
||||
get_tid();
|
||||
//
|
||||
inline void
|
||||
flush()
|
||||
{
|
||||
fprintf(stdout, "%s", ::tim::log::color::end());
|
||||
fflush(stdout);
|
||||
std::cout << ::tim::log::color::end() << std::flush;
|
||||
fprintf(::rocprofsys::debug::get_file(), "%s", ::tim::log::color::end());
|
||||
fflush(::rocprofsys::debug::get_file());
|
||||
std::cerr << ::tim::log::color::end() << std::flush;
|
||||
}
|
||||
//
|
||||
struct lock
|
||||
{
|
||||
lock();
|
||||
~lock();
|
||||
|
||||
private:
|
||||
locking::atomic_lock m_lk;
|
||||
};
|
||||
//
|
||||
template <typename Arg, typename... Args>
|
||||
bool
|
||||
is_bracket(Arg&& _arg, Args&&...)
|
||||
{
|
||||
if constexpr(::tim::concepts::is_string_type<Arg>::value)
|
||||
return (::std::string_view{ _arg }.empty()) ? false : _arg[0] == '[';
|
||||
else
|
||||
return false;
|
||||
}
|
||||
//
|
||||
namespace
|
||||
{
|
||||
template <typename T, size_t... Idx>
|
||||
auto
|
||||
get_chars(T&& _c, std::index_sequence<Idx...>)
|
||||
{
|
||||
return std::array<const char, sizeof...(Idx) + 1>{ std::forward<T>(_c)[Idx]...,
|
||||
'\0' };
|
||||
}
|
||||
} // namespace
|
||||
} // namespace debug
|
||||
|
||||
namespace binary
|
||||
{
|
||||
struct address_range;
|
||||
}
|
||||
|
||||
using address_range_t = binary::address_range;
|
||||
|
||||
template <typename Tp>
|
||||
std::string
|
||||
as_hex(Tp, size_t _wdith = 16);
|
||||
|
||||
template <>
|
||||
std::string as_hex<address_range_t>(address_range_t, size_t);
|
||||
|
||||
extern template std::string as_hex<int32_t>(int32_t, size_t);
|
||||
extern template std::string as_hex<uint32_t>(uint32_t, size_t);
|
||||
extern template std::string as_hex<int64_t>(int64_t, size_t);
|
||||
extern template std::string as_hex<uint64_t>(uint64_t, size_t);
|
||||
extern template std::string
|
||||
as_hex<void*>(void*, size_t);
|
||||
} // namespace rocprofsys
|
||||
|
||||
#if !defined(ROCPROFSYS_DEBUG_BUFFER_LEN)
|
||||
# define ROCPROFSYS_DEBUG_BUFFER_LEN 1024
|
||||
#endif
|
||||
|
||||
#if !defined(ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER)
|
||||
# if defined(ROCPROFSYS_USE_MPI)
|
||||
# define ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER static_cast<int>(::tim::dmp::rank())
|
||||
# elif defined(ROCPROFSYS_USE_MPI_HEADERS)
|
||||
# define ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER \
|
||||
(::tim::dmp::is_initialized()) ? static_cast<int>(::tim::dmp::rank()) \
|
||||
: static_cast<int>(::tim::process::get_id())
|
||||
# else
|
||||
# define ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER \
|
||||
static_cast<int>(::tim::process::get_id())
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if !defined(ROCPROFSYS_DEBUG_THREAD_IDENTIFIER)
|
||||
# define ROCPROFSYS_DEBUG_THREAD_IDENTIFIER ::rocprofsys::debug::get_tid()
|
||||
#endif
|
||||
|
||||
#if !defined(ROCPROFSYS_SOURCE_LOCATION)
|
||||
# define ROCPROFSYS_SOURCE_LOCATION \
|
||||
::rocprofsys::debug::source_location { __PRETTY_FUNCTION__, __FILE__, __LINE__ }
|
||||
#endif
|
||||
|
||||
#if !defined(ROCPROFSYS_RECORD_SOURCE_LOCATION)
|
||||
# define ROCPROFSYS_RECORD_SOURCE_LOCATION \
|
||||
::rocprofsys::debug::set_source_location(ROCPROFSYS_SOURCE_LOCATION)
|
||||
#endif
|
||||
|
||||
#if defined(__clang__) || (__GNUC__ < 9)
|
||||
# define ROCPROFSYS_FUNCTION \
|
||||
std::string{ __FUNCTION__ } \
|
||||
.substr(0, std::string_view{ __FUNCTION__ }.find("_hidden")) \
|
||||
.c_str()
|
||||
# define ROCPROFSYS_PRETTY_FUNCTION \
|
||||
std::string{ __PRETTY_FUNCTION__ } \
|
||||
.substr(0, std::string_view{ __PRETTY_FUNCTION__ }.find("_hidden")) \
|
||||
.c_str()
|
||||
#else
|
||||
# define ROCPROFSYS_FUNCTION \
|
||||
::rocprofsys::debug::get_chars( \
|
||||
std::string_view{ __FUNCTION__ }, \
|
||||
std::make_index_sequence<std::min( \
|
||||
std::string_view{ __FUNCTION__ }.find("_hidden"), \
|
||||
std::string_view{ __FUNCTION__ }.length())>{}) \
|
||||
.data()
|
||||
# define ROCPROFSYS_PRETTY_FUNCTION \
|
||||
::rocprofsys::debug::get_chars( \
|
||||
std::string_view{ __PRETTY_FUNCTION__ }, \
|
||||
std::make_index_sequence<std::min( \
|
||||
std::string_view{ __PRETTY_FUNCTION__ }.find("_hidden"), \
|
||||
std::string_view{ __PRETTY_FUNCTION__ }.length())>{}) \
|
||||
.data()
|
||||
#endif
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_FPRINTF_STDERR_COLOR(COLOR) \
|
||||
fprintf(::rocprofsys::debug::get_file(), "%s", ::tim::log::color::COLOR())
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_PRINT_COLOR(COLOR, COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(COLOR); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_PRINT_COLOR_F(COLOR, COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(COLOR); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_PRINT_COLOR(COLOR, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_PRINT_COLOR(COLOR, true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_PRINT_COLOR_F(COLOR, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_PRINT_COLOR_F(COLOR, true, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_PRINT(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(info); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_PRINT(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(info); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_PRINT_F(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(info); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(info); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_WARN(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(warning); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_WARN(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(warning); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_WARN_F(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(warning); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_WARN_F(COND, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND) && ::rocprofsys::config::get_debug_tid() && \
|
||||
::rocprofsys::config::get_debug_pid())) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::debug::lock _debug_lk{}; \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(warning); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
}
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_THROW_E(COND, TYPE, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND))) \
|
||||
{ \
|
||||
char _msg_buffer[ROCPROFSYS_DEBUG_BUFFER_LEN]; \
|
||||
bool _print_backtrace = ::rocprofsys::get_debug() || \
|
||||
::rocprofsys::get_verbose() >= 2 || \
|
||||
::rocprofsys::get_is_continuous_integration(); \
|
||||
snprintf(_msg_buffer, ROCPROFSYS_DEBUG_BUFFER_LEN, \
|
||||
"[rocprof-sys][%i][%li][%s]%s", ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, \
|
||||
ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
auto len = strlen(_msg_buffer); \
|
||||
snprintf(_msg_buffer + len, ROCPROFSYS_DEBUG_BUFFER_LEN - len, __VA_ARGS__); \
|
||||
if(!_print_backtrace) \
|
||||
throw ::rocprofsys::exception<TYPE>( \
|
||||
::tim::log::string(::tim::log::color::fatal(), _msg_buffer), false); \
|
||||
else \
|
||||
throw ::rocprofsys::exception<TYPE>( \
|
||||
::tim::log::string(::tim::log::color::fatal(), _msg_buffer)); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_THROW_E(COND, TYPE, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND))) \
|
||||
{ \
|
||||
char _msg_buffer[ROCPROFSYS_DEBUG_BUFFER_LEN]; \
|
||||
snprintf(_msg_buffer, ROCPROFSYS_DEBUG_BUFFER_LEN, "[rocprof-sys][%i][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
auto len = strlen(_msg_buffer); \
|
||||
snprintf(_msg_buffer + len, ROCPROFSYS_DEBUG_BUFFER_LEN - len, __VA_ARGS__); \
|
||||
throw ::rocprofsys::exception<TYPE>( \
|
||||
::tim::log::string(::tim::log::color::fatal(), _msg_buffer)); \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CI_THROW_E(COND, TYPE, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_THROW_E( \
|
||||
::rocprofsys::get_is_continuous_integration() && (COND), TYPE, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_BASIC_THROW_E(COND, TYPE, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_THROW_E( \
|
||||
::rocprofsys::get_is_continuous_integration() && (COND), TYPE, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_THROW(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_THROW_E((COND), std::runtime_error, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_THROW(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_THROW_E((COND), std::runtime_error, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_THROW(COND, ...) \
|
||||
ROCPROFSYS_CI_THROW_E((COND), std::runtime_error, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_BASIC_THROW(COND, ...) \
|
||||
ROCPROFSYS_CI_BASIC_THROW_E((COND), std::runtime_error, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_FAILURE(COND, METHOD, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND))) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(fatal); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized); \
|
||||
timemory_print_demangled_backtrace<64>(); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_FAILURE(COND, METHOD, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND))) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(fatal); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized); \
|
||||
timemory_print_demangled_backtrace<64>(); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_FAILURE_F(COND, METHOD, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND))) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(fatal); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%li][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_DEBUG_THREAD_IDENTIFIER, \
|
||||
ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized); \
|
||||
timemory_print_demangled_backtrace<64>(); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_FAILURE_F(COND, METHOD, ...) \
|
||||
if(ROCPROFSYS_UNLIKELY((COND))) \
|
||||
{ \
|
||||
::rocprofsys::debug::flush(); \
|
||||
ROCPROFSYS_FPRINTF_STDERR_COLOR(fatal); \
|
||||
fprintf(::rocprofsys::debug::get_file(), "[rocprof-sys][%i][%s]%s", \
|
||||
ROCPROFSYS_DEBUG_PROCESS_IDENTIFIER, ROCPROFSYS_FUNCTION, \
|
||||
::rocprofsys::debug::is_bracket(__VA_ARGS__) ? "" : " "); \
|
||||
fprintf(::rocprofsys::debug::get_file(), __VA_ARGS__); \
|
||||
::rocprofsys::debug::flush(); \
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized); \
|
||||
timemory_print_demangled_backtrace<64>(); \
|
||||
METHOD; \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_CI_FAILURE(COND, METHOD, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_FAILURE( \
|
||||
::rocprofsys::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_BASIC_FAILURE(COND, METHOD, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_FAILURE( \
|
||||
::rocprofsys::get_is_continuous_integration() && (COND), METHOD, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_FAIL(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_FAILURE(COND, ROCPROFSYS_ESC(::std::exit(EXIT_FAILURE)), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_FAIL(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_FAILURE( \
|
||||
COND, ROCPROFSYS_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_FAIL_F(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_FAILURE_F(COND, ROCPROFSYS_ESC(::std::exit(EXIT_FAILURE)), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_FAIL_F(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_FAILURE_F( \
|
||||
COND, ROCPROFSYS_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_FAIL(COND, ...) \
|
||||
ROCPROFSYS_CI_FAILURE(COND, ROCPROFSYS_ESC(::std::exit(EXIT_FAILURE)), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_BASIC_FAIL(COND, ...) \
|
||||
ROCPROFSYS_CI_BASIC_FAILURE(COND, ROCPROFSYS_ESC(::std::exit(EXIT_FAILURE)), \
|
||||
__VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_ABORT(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_FAILURE(COND, ROCPROFSYS_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_ABORT(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_FAILURE(COND, ROCPROFSYS_ESC(::std::abort()), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_ABORT_F(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_FAILURE_F(COND, ROCPROFSYS_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CONDITIONAL_BASIC_ABORT_F(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_FAILURE_F(COND, ROCPROFSYS_ESC(::std::abort()), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_ABORT(COND, ...) \
|
||||
ROCPROFSYS_CI_FAILURE(COND, ROCPROFSYS_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_CI_BASIC_ABORT(COND, ...) \
|
||||
ROCPROFSYS_CI_BASIC_FAILURE(COND, ROCPROFSYS_ESC(::std::abort()), __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Debug macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_DEBUG(...) \
|
||||
ROCPROFSYS_CONDITIONAL_PRINT(::rocprofsys::get_debug(), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_DEBUG(...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT(::rocprofsys::get_debug_env(), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_DEBUG_F(...) \
|
||||
ROCPROFSYS_CONDITIONAL_PRINT_F(::rocprofsys::get_debug(), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_DEBUG_F(...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(::rocprofsys::get_debug_env(), __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Verbose macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_VERBOSE(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_PRINT(::rocprofsys::get_debug() || \
|
||||
(::rocprofsys::get_verbose() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_VERBOSE(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT(::rocprofsys::get_debug_env() || \
|
||||
(::rocprofsys::get_verbose_env() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_VERBOSE_F(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_PRINT_F(::rocprofsys::get_debug() || \
|
||||
(::rocprofsys::get_verbose() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_VERBOSE_F(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(::rocprofsys::get_debug_env() || \
|
||||
(::rocprofsys::get_verbose_env() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Warning macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_WARNING(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_WARN(::rocprofsys::get_debug() || \
|
||||
(::rocprofsys::get_verbose() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_WARNING(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_WARN(::rocprofsys::get_debug_env() || \
|
||||
(::rocprofsys::get_verbose_env() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_WARNING_F(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_WARN_F(::rocprofsys::get_debug() || \
|
||||
(::rocprofsys::get_verbose() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_WARNING_F(LEVEL, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_WARN_F(::rocprofsys::get_debug_env() || \
|
||||
(::rocprofsys::get_verbose_env() >= LEVEL), \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_WARNING_IF(COND, ...) ROCPROFSYS_CONDITIONAL_WARN((COND), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_WARNING_IF_F(COND, ...) \
|
||||
ROCPROFSYS_CONDITIONAL_WARN_F((COND), __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_WARNING_OR_CI_THROW(LEVEL, ...) \
|
||||
{ \
|
||||
if(ROCPROFSYS_UNLIKELY(::rocprofsys::get_is_continuous_integration())) \
|
||||
{ \
|
||||
ROCPROFSYS_CI_THROW(true, __VA_ARGS__); \
|
||||
} \
|
||||
else \
|
||||
{ \
|
||||
ROCPROFSYS_CONDITIONAL_WARN(::rocprofsys::get_debug() || \
|
||||
(::rocprofsys::get_verbose() >= LEVEL), \
|
||||
__VA_ARGS__) \
|
||||
} \
|
||||
}
|
||||
|
||||
#define ROCPROFSYS_REQUIRE(...) TIMEMORY_REQUIRE(__VA_ARGS__)
|
||||
#define ROCPROFSYS_PREFER(COND) \
|
||||
((ROCPROFSYS_LIKELY(COND)) \
|
||||
? ::tim::log::base() \
|
||||
: ((::rocprofsys::get_is_continuous_integration()) ? TIMEMORY_FATAL \
|
||||
: TIMEMORY_WARNING))
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Basic print macros (basic means it will not provide PID/RANK or TID) and will not
|
||||
// initialize the settings.
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_BASIC_PRINT(...) ROCPROFSYS_CONDITIONAL_BASIC_PRINT(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_PRINT_F(...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(true, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Print macros. Will provide PID/RANK and TID (will initialize settings)
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_PRINT(...) ROCPROFSYS_CONDITIONAL_PRINT(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_PRINT_F(...) ROCPROFSYS_CONDITIONAL_PRINT_F(true, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Throw macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_THROW(...) ROCPROFSYS_CONDITIONAL_THROW(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_THROW(...) ROCPROFSYS_CONDITIONAL_BASIC_THROW(true, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Fail macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_FAIL(...) ROCPROFSYS_CONDITIONAL_FAIL(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_FAIL_F(...) ROCPROFSYS_CONDITIONAL_FAIL_F(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_FAIL(...) ROCPROFSYS_CONDITIONAL_BASIC_FAIL(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_FAIL_F(...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_FAIL_F(true, __VA_ARGS__)
|
||||
|
||||
//--------------------------------------------------------------------------------------//
|
||||
//
|
||||
// Abort macros
|
||||
//
|
||||
//--------------------------------------------------------------------------------------//
|
||||
|
||||
#define ROCPROFSYS_ABORT(...) ROCPROFSYS_CONDITIONAL_ABORT(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_ABORT_F(...) ROCPROFSYS_CONDITIONAL_ABORT_F(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_ABORT(...) ROCPROFSYS_CONDITIONAL_BASIC_ABORT(true, __VA_ARGS__)
|
||||
|
||||
#define ROCPROFSYS_BASIC_ABORT_F(...) \
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_ABORT_F(true, __VA_ARGS__)
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace std
|
||||
{
|
||||
inline std::string
|
||||
to_string(bool _v)
|
||||
{
|
||||
return (_v) ? "true" : "false";
|
||||
}
|
||||
} // namespace std
|
||||
@@ -22,14 +22,14 @@
|
||||
|
||||
#include "dynamic_library.hpp"
|
||||
#include "common.hpp"
|
||||
#include "debug.hpp"
|
||||
#include "defines.hpp"
|
||||
|
||||
#include <timemory/environment.hpp>
|
||||
#include <timemory/utility/delimit.hpp>
|
||||
#include <timemory/utility/filepath.hpp>
|
||||
#include <timemory/utility/procfs/maps.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
@@ -100,19 +100,15 @@ dynamic_library::dynamic_library(std::string _env, std::string _fname, int _flag
|
||||
}
|
||||
else if(_env_val.find('/') == 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
1,
|
||||
"Ignoring environment variable %s=\"%s\" because the "
|
||||
"filepath does not exist. Using \"%s\" instead...\n",
|
||||
envname.c_str(), _env_val.c_str(), filename.c_str())
|
||||
LOG_WARNING("Ignoring environment variable {}=\"{}\" because the "
|
||||
"filepath does not exist. Using \"{}\" instead...",
|
||||
envname, _env_val, filename);
|
||||
}
|
||||
else if(_env_val.find('/') != 0 && filename.find('/') == 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
1,
|
||||
"Ignoring environment variable %s=\"%s\" because the "
|
||||
"filepath is relative. Using absolute path \"%s\" instead...\n",
|
||||
envname.c_str(), _env_val.c_str(), filename.c_str())
|
||||
LOG_WARNING("Ignoring environment variable {}=\"{}\" because the "
|
||||
"filepath is relative. Using absolute path \"{}\" instead...",
|
||||
envname, _env_val, filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -130,8 +126,8 @@ dynamic_library::open()
|
||||
handle = dlopen(filename.c_str(), flags);
|
||||
if(!handle)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "[dynamic_library] Error opening %s=\"%s\" :: %s.\n",
|
||||
envname.c_str(), filename.c_str(), dlerror());
|
||||
LOG_WARNING("[dynamic_library] Error opening {}=\"{}\" :: {}.", envname,
|
||||
filename, dlerror());
|
||||
}
|
||||
dlerror(); // Clear any existing error
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "defines.hpp"
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <dlfcn.h>
|
||||
#include <string>
|
||||
@@ -77,8 +77,7 @@ dynamic_library::invoke(std::string_view _name, RetT (*&_func)(Args...), Args...
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "[rocprof-sys][pid=%i]> %s :: %s\n", getpid(), _name.data(),
|
||||
dlerror());
|
||||
LOG_WARNING("[rocprof-sys][pid={}]> {} :: {}", getpid(), _name, dlerror());
|
||||
}
|
||||
}
|
||||
return RetT{};
|
||||
|
||||
@@ -35,7 +35,6 @@
|
||||
# define ROCPROFSYS_USE_ROCM 0
|
||||
#endif
|
||||
|
||||
#include "debug.hpp"
|
||||
#include "defines.hpp"
|
||||
#include "gpu.hpp"
|
||||
|
||||
@@ -53,6 +52,8 @@
|
||||
# include <rocprofiler-sdk/fwd.h>
|
||||
#endif
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace gpu
|
||||
@@ -70,12 +71,14 @@ check_amdsmi_error(amdsmi_status_t _code, const char* _file, int _line)
|
||||
const char* _msg = nullptr;
|
||||
auto _err = amdsmi_status_code_to_string(_code, &_msg);
|
||||
if(_err != AMDSMI_STATUS_SUCCESS)
|
||||
ROCPROFSYS_THROW(
|
||||
{
|
||||
throw std::runtime_error(fmt::format(
|
||||
"amdsmi_status_code_to_string failed. No error message available. "
|
||||
"Error code %i originated at %s:%i\n",
|
||||
static_cast<int>(_code), _file, _line);
|
||||
ROCPROFSYS_THROW("[%s:%i] Error code %i :: %s", _file, _line, static_cast<int>(_code),
|
||||
_msg);
|
||||
"Error code {} originated at {}:{}",
|
||||
static_cast<int>(_code), _file, _line));
|
||||
}
|
||||
throw std::runtime_error(fmt::format("[{}:{}] Error code {} :: {}", _file, _line,
|
||||
static_cast<int>(_code), _msg));
|
||||
}
|
||||
|
||||
// Ensures initialization happens only once
|
||||
@@ -114,8 +117,7 @@ amdsmi_init()
|
||||
prevent_amdsmi_library_unload();
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "Exception thrown initializing amd-smi: %s\n",
|
||||
_e.what());
|
||||
LOG_ERROR("Exception thrown initializing amd-smi: {}", _e.what());
|
||||
_amdsmi_is_initialized() = false; // Mark as not initialized
|
||||
return false;
|
||||
}
|
||||
@@ -165,9 +167,8 @@ query_rocm_agents()
|
||||
sizeof(rocprofiler_agent_v0_t), nullptr);
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
1, "Exception thrown getting the rocm agents: %s. _dev_cnt=%ld\n", _e.what(),
|
||||
_dev_cnt);
|
||||
LOG_ERROR("Exception thrown getting the rocm agents: {}. _dev_cnt={}", _e.what(),
|
||||
_dev_cnt);
|
||||
}
|
||||
_dev_cnt = get_agent_manager_instance().get_gpu_agents_count();
|
||||
#endif
|
||||
@@ -230,8 +231,7 @@ add_device_metadata(ArchiveT& ar)
|
||||
sizeof(rocprofiler_agent_v0_t), &_agents_vec);
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "Exception thrown getting the rocm agents: %s.\n",
|
||||
_e.what());
|
||||
LOG_ERROR("Exception thrown getting the rocm agents: {}", _e.what());
|
||||
}
|
||||
|
||||
ar(make_nvp("rocm_agents", _agents_vec));
|
||||
@@ -251,7 +251,7 @@ add_device_metadata()
|
||||
add_device_metadata(ar);
|
||||
} catch(std::runtime_error& _e)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "%s\n", _e.what());
|
||||
LOG_ERROR("Exception thrown adding device metadata: {}", _e.what());
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -309,8 +309,7 @@ get_processor_handles()
|
||||
ret = amdsmi_get_processor_type(processor, &processor_type);
|
||||
if(processor_type != AMDSMI_PROCESSOR_TYPE_AMD_GPU)
|
||||
{
|
||||
ROCPROFSYS_THROW("Not AMD_GPU device type!");
|
||||
return;
|
||||
throw std::runtime_error("Not AMD_GPU device type!");
|
||||
}
|
||||
processors::processors_list.push_back(processor);
|
||||
|
||||
|
||||
@@ -27,12 +27,13 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "debug.hpp"
|
||||
#include <timemory/timemory.hpp>
|
||||
|
||||
#include <timemory/environment/declaration.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
@@ -259,8 +260,9 @@ check_error(const char* _func, int err_code, comm_t _comm = mpi::comm_world_v)
|
||||
PMPI_Error_string(err_code, msg, &len);
|
||||
msg[std::min<int>(len, 1023)] = '\0';
|
||||
int _rank = rank();
|
||||
fprintf(stderr, "[rank=%i][pid=%i][tid=%i][%s]> Error code (%i): %s\n", _rank,
|
||||
(int) process::get_id(), (int) threading::get_id(), _func, err_code, msg);
|
||||
LOG_ERROR("[rank={}][pid={}][tid={}][{}]> Error code ({}): {}", _rank,
|
||||
(int) process::get_id(), (int) threading::get_id(), _func, err_code,
|
||||
msg);
|
||||
}
|
||||
if(!_success && fail_on_error()) PMPI_Abort(_comm, err_code);
|
||||
return (err_code == MPI_SUCCESS);
|
||||
@@ -342,8 +344,7 @@ initialize(int& argc, char**& argv)
|
||||
auto ret = MPI_Init_thread(&argc, &argv, itr, &_actual);
|
||||
if(_actual != itr)
|
||||
{
|
||||
fprintf(stderr, "Warning! MPI_Init_thread does not support: %s\n",
|
||||
_type.c_str());
|
||||
LOG_WARNING("MPI_Init_thread does not support: {}", _type);
|
||||
}
|
||||
return ROCPROFSYS_MPI_ERROR_CHECK(ret);
|
||||
};
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "mproc.hpp"
|
||||
#include "common.hpp"
|
||||
#include "debug.hpp"
|
||||
|
||||
#include <fstream>
|
||||
#include <set>
|
||||
@@ -32,6 +31,8 @@
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace mproc
|
||||
@@ -46,7 +47,7 @@ get_concurrent_processes(int _ppid)
|
||||
std::ifstream _ifs{ _inp };
|
||||
if(!_ifs)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(2, "Warning! File '%s' cannot be read\n", _inp.c_str());
|
||||
LOG_WARNING("File '{}' cannot be read. Returning empty set.", _inp);
|
||||
return _children;
|
||||
}
|
||||
|
||||
@@ -92,16 +93,8 @@ wait_pid(pid_t _pid, int _opts)
|
||||
}
|
||||
|
||||
int
|
||||
diagnose_status(pid_t _pid, int _status, int _verbose)
|
||||
diagnose_status(pid_t _pid, int _status, [[maybe_unused]] int _verbose)
|
||||
{
|
||||
if(_verbose >= 3)
|
||||
{
|
||||
fflush(stderr);
|
||||
fflush(stdout);
|
||||
std::cout << std::flush;
|
||||
std::cerr << std::flush;
|
||||
}
|
||||
|
||||
bool _normal_exit = (WIFEXITED(_status) > 0);
|
||||
bool _unhandled_signal = (WIFSIGNALED(_status) > 0);
|
||||
bool _core_dump = (WCOREDUMP(_status) > 0);
|
||||
@@ -110,81 +103,48 @@ diagnose_status(pid_t _pid, int _status, int _verbose)
|
||||
int _stop_signal = (_stopped) ? WSTOPSIG(_status) : 0;
|
||||
int _ec = (_unhandled_signal) ? WTERMSIG(_status) : 0;
|
||||
|
||||
if(_verbose >= 4)
|
||||
{
|
||||
TIMEMORY_PRINTF_INFO(
|
||||
stderr,
|
||||
"diagnosing status for process %i :: status: %i... normal exit: %s, "
|
||||
"unhandled signal: %s, core dump: %s, stopped: %s, exit status: %i, stop "
|
||||
"signal: %i, exit code: %i\n",
|
||||
_pid, _status, std::to_string(_normal_exit).c_str(),
|
||||
std::to_string(_unhandled_signal).c_str(), std::to_string(_core_dump).c_str(),
|
||||
std::to_string(_stopped).c_str(), _exit_status, _stop_signal, _ec);
|
||||
}
|
||||
else if(_verbose >= 3)
|
||||
{
|
||||
TIMEMORY_PRINTF_INFO(stderr,
|
||||
"diagnosing status for process %i :: status: %i ...\n", _pid,
|
||||
_status);
|
||||
}
|
||||
LOG_TRACE("diagnosing status for process {} :: status: {}... normal exit: {}, "
|
||||
"unhandled signal: {}, core dump: {}, stopped: {}, exit status: {}, stop "
|
||||
"signal: {}, exit code: {}",
|
||||
_pid, _status, std::to_string(_normal_exit),
|
||||
std::to_string(_unhandled_signal), std::to_string(_core_dump),
|
||||
std::to_string(_stopped), _exit_status, _stop_signal, _ec);
|
||||
|
||||
if(!_normal_exit)
|
||||
{
|
||||
if(_ec == 0) _ec = EXIT_FAILURE;
|
||||
if(_verbose >= 0)
|
||||
{
|
||||
TIMEMORY_PRINTF_FATAL(
|
||||
stderr, "process %i terminated abnormally. exit code: %i\n", _pid, _ec);
|
||||
}
|
||||
LOG_ERROR("process {} terminated abnormally. exit code: {}", _pid, _ec);
|
||||
}
|
||||
|
||||
if(_stopped)
|
||||
{
|
||||
if(_verbose >= 0)
|
||||
{
|
||||
TIMEMORY_PRINTF_FATAL(stderr,
|
||||
"process %i stopped with signal %i. exit code: %i\n",
|
||||
_pid, _stop_signal, _ec);
|
||||
}
|
||||
LOG_ERROR("process {} stopped with signal {}. exit code: {}", _pid, _stop_signal,
|
||||
_ec);
|
||||
}
|
||||
|
||||
if(_core_dump)
|
||||
{
|
||||
if(_verbose >= 0)
|
||||
{
|
||||
TIMEMORY_PRINTF_FATAL(
|
||||
stderr, "process %i terminated and produced a core dump. exit code: %i\n",
|
||||
_pid, _ec);
|
||||
}
|
||||
LOG_CRITICAL("process {} terminated and produced a core dump. exit code: {}",
|
||||
_pid, _ec);
|
||||
}
|
||||
|
||||
if(_unhandled_signal)
|
||||
{
|
||||
if(_verbose >= 0)
|
||||
{
|
||||
TIMEMORY_PRINTF_FATAL(stderr,
|
||||
"process %i terminated because it received a signal "
|
||||
"(%i) that was not handled. exit code: %i\n",
|
||||
_pid, _ec, _ec);
|
||||
}
|
||||
LOG_ERROR("process {} terminated because it received a signal "
|
||||
"({}) that was not handled. exit code: {}",
|
||||
_pid, _ec, _ec);
|
||||
}
|
||||
|
||||
if(!_normal_exit && _exit_status > 0)
|
||||
{
|
||||
if(_verbose >= 0)
|
||||
if(_exit_status == 127)
|
||||
{
|
||||
if(_exit_status == 127)
|
||||
{
|
||||
TIMEMORY_PRINTF_FATAL(
|
||||
stderr, "execv in process %i failed. exit code: %i\n", _pid, _ec);
|
||||
}
|
||||
else
|
||||
{
|
||||
TIMEMORY_PRINTF_FATAL(
|
||||
stderr,
|
||||
"process %i terminated with a non-zero status. exit code: %i\n", _pid,
|
||||
_ec);
|
||||
}
|
||||
LOG_CRITICAL("execv in process {} failed. exit code: {}", _pid, _ec);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_CRITICAL("process {} terminated with a non-zero status. exit code: {}",
|
||||
_pid, _ec);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,13 +21,14 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "node_info.hpp"
|
||||
#include "debug.hpp"
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <sys/utsname.h>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
|
||||
@@ -36,21 +37,21 @@ node_info::node_info()
|
||||
auto ifs = std::ifstream{ "/etc/machine-id" };
|
||||
if(!ifs.is_open())
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Error: Unable to open /etc/machine-id!");
|
||||
LOG_WARNING("Error: Unable to open /etc/machine-id!");
|
||||
return;
|
||||
}
|
||||
if(!(ifs >> machine_id) || machine_id.empty())
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Error: Unable to read machine ID from /etc/machine-id!");
|
||||
LOG_WARNING("Error: Unable to read machine ID from /etc/machine-id!");
|
||||
}
|
||||
|
||||
hash = std::hash<std::string>{}(machine_id) % std::numeric_limits<int64_t>::max();
|
||||
id = hash % std::numeric_limits<size_t>::max();
|
||||
|
||||
struct utsname _sys_info;
|
||||
if(uname(&_sys_info))
|
||||
if(uname(&_sys_info) != 0)
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Error: Unable to get system information!");
|
||||
LOG_WARNING("Error: Unable to get system information!");
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -21,10 +21,11 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "perf.hpp"
|
||||
#include "debug.hpp"
|
||||
|
||||
#include <timemory/units.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace perf
|
||||
@@ -72,7 +73,8 @@ get_hw_config(std::string_view _v)
|
||||
return hw_config::reference_cpu_cycles;
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("Unknown perf hardware config: %s", _v.data());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Unknown perf hardware config: {}", _v.data()));
|
||||
}
|
||||
|
||||
#undef HW_CONFIG_REGEX
|
||||
@@ -105,7 +107,8 @@ get_sw_config(std::string_view _v)
|
||||
return sw_config::emulation_faults;
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("Unknown perf hw cache config: %s", _v.data());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Unknown perf hw cache config: {}", _v.data()));
|
||||
}
|
||||
|
||||
#undef SW_CONFIG_REGEX
|
||||
@@ -136,7 +139,8 @@ get_hw_cache_config(std::string_view _v)
|
||||
else if(HW_CACHE_CONFIG_REGEX("NODE"))
|
||||
_value |= static_cast<int>(hw_cache_config::node);
|
||||
else
|
||||
ROCPROFSYS_THROW("Unknown perf software config: %s", _v.data());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Unknown perf software config: {}", _v.data()));
|
||||
|
||||
#undef HW_CACHE_CONFIG_REGEX
|
||||
#define HW_CACHE_OP_REGEX(KEY) \
|
||||
@@ -197,7 +201,7 @@ config_overflow_sampling(struct perf_event_attr& _pe, std::string_view _event,
|
||||
case PERF_TYPE_MAX:
|
||||
default:
|
||||
{
|
||||
ROCPROFSYS_THROW("unsupported perf type");
|
||||
throw std::runtime_error("Unsupported perf type");
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "perfetto.hpp"
|
||||
#include "config.hpp"
|
||||
#include "debug.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
#include "perfetto_fwd.hpp"
|
||||
#include "utility.hpp"
|
||||
@@ -90,8 +89,7 @@ setup()
|
||||
|
||||
for(const auto& itr : config::get_disabled_categories())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Disabling perfetto track event category: %s\n",
|
||||
itr.c_str());
|
||||
LOG_DEBUG("Disabling perfetto track event category: {}", itr);
|
||||
track_event_cfg.add_disabled_categories(itr);
|
||||
}
|
||||
|
||||
@@ -130,13 +128,12 @@ start()
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(2, "Setup perfetto...\n");
|
||||
LOG_DEBUG("Setup perfetto...");
|
||||
int _fd = (_tmp_file) ? _tmp_file->fd : -1;
|
||||
auto& cfg = get_config();
|
||||
tracing_session->SetOnErrorCallback([](::perfetto::TracingError _err) {
|
||||
if(_err.code == ::perfetto::TracingError::kTracingFailed)
|
||||
ROCPROFSYS_WARNING(0, "perfetto encountered a tracing error: %s\n",
|
||||
_err.message.c_str());
|
||||
LOG_WARNING("Perfetto encountered a tracing error: {}", _err.message);
|
||||
});
|
||||
tracing_session->Setup(cfg, _fd);
|
||||
tracing_session->StartBlocking();
|
||||
@@ -149,17 +146,19 @@ stop()
|
||||
|
||||
auto& tracing_session = get_perfetto_session();
|
||||
|
||||
ROCPROFSYS_CI_THROW(tracing_session == nullptr,
|
||||
"Null pointer to the tracing session");
|
||||
if(get_is_continuous_integration() && tracing_session == nullptr)
|
||||
{
|
||||
throw std::runtime_error("Null pointer to the tracing session");
|
||||
}
|
||||
|
||||
if(tracing_session)
|
||||
{
|
||||
// Make sure the last event is closed
|
||||
ROCPROFSYS_VERBOSE(2, "Flushing the perfetto trace data...\n");
|
||||
LOG_DEBUG("Flushing the perfetto trace data...");
|
||||
::perfetto::TrackEvent::Flush();
|
||||
tracing_session->FlushBlocking();
|
||||
|
||||
ROCPROFSYS_VERBOSE(2, "Stopping the perfetto trace session (blocking)...\n");
|
||||
LOG_DEBUG("Stopping the perfetto trace session (blocking)...");
|
||||
tracing_session->StopBlocking();
|
||||
}
|
||||
}
|
||||
@@ -184,9 +183,8 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
|
||||
|
||||
if(!_fdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
-1, "Error! perfetto temp trace file '%s' could not be read",
|
||||
_tmp_file->filename.c_str());
|
||||
LOG_ERROR("perfetto temp trace file '{}' could not be read",
|
||||
_tmp_file->filename);
|
||||
return char_vec_t{ tracing_session->ReadTraceBlocking() };
|
||||
}
|
||||
|
||||
@@ -198,10 +196,12 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
|
||||
auto _fnum_read = ::fread(_data.data(), sizeof(char), _fnum_elem, _fdata);
|
||||
::fclose(_fdata);
|
||||
|
||||
ROCPROFSYS_CI_THROW(
|
||||
_fnum_read != _fnum_elem,
|
||||
"Error! read %zu elements from perfetto trace file '%s'. Expected %zu\n",
|
||||
_fnum_read, _tmp_file->filename.c_str(), _fnum_elem);
|
||||
if(get_is_continuous_integration() && _fnum_read != _fnum_elem)
|
||||
{
|
||||
throw std::runtime_error(fmt::format(
|
||||
"read {} elements from perfetto trace file '{}'. Expected {}",
|
||||
_fnum_read, _tmp_file->filename, _fnum_elem));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -272,9 +272,8 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
|
||||
}
|
||||
else if(dmp::rank() == 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0, "perfetto trace data is empty. File '%s' will not be written...\n",
|
||||
_filename.c_str());
|
||||
LOG_ERROR("Perfetto trace data is empty. File '{}' will not be written...",
|
||||
_filename);
|
||||
}
|
||||
|
||||
// Merge the output files, if rank 0
|
||||
@@ -292,7 +291,7 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
|
||||
// Test that the script exists
|
||||
if(!filepath::exists(_script_path))
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "Script not found: %s\n", _script_path.c_str());
|
||||
LOG_WARNING("Script not found: {}", _script_path);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -303,11 +302,11 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error)
|
||||
|
||||
if(result != 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "Failed to execute: %s\n", _command.c_str());
|
||||
LOG_ERROR("Failed to execute: {}", _command);
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "Successfully executed: %s\n", _command.c_str());
|
||||
LOG_INFO("Successfully executed: {}", _command);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,8 @@
|
||||
#include "categories.hpp"
|
||||
#include "common.hpp"
|
||||
|
||||
#include "config.hpp"
|
||||
|
||||
#if defined(TIMEMORY_USE_PERFETTO)
|
||||
# include <timemory/components/perfetto/backends.hpp>
|
||||
#else
|
||||
@@ -32,8 +34,6 @@
|
||||
PERFETTO_DEFINE_CATEGORIES(ROCPROFSYS_PERFETTO_CATEGORIES);
|
||||
#endif
|
||||
|
||||
#include "debug.hpp"
|
||||
|
||||
#include <timemory/process/process.hpp>
|
||||
|
||||
#include <map>
|
||||
@@ -43,6 +43,8 @@ PERFETTO_DEFINE_CATEGORIES(ROCPROFSYS_PERFETTO_CATEGORIES);
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
std::unique_ptr<::perfetto::TracingSession>& get_perfetto_session(
|
||||
@@ -144,12 +146,11 @@ perfetto_counter_track<Tp>::emplace(size_t _idx, const std::string& _v,
|
||||
std::stringstream _css{};
|
||||
for(auto&& eitr : _curr)
|
||||
_css << " " << std::hex << std::setw(12) << std::left << eitr;
|
||||
ROCPROFSYS_THROW("perfetto_counter_track emplace method for '%s' (%p) "
|
||||
"invalidated C-string '%s' (%p).\n%8s: %s\n%8s: %s\n",
|
||||
_v.c_str(), (void*) _name->c_str(),
|
||||
std::get<0>(itr).c_str(),
|
||||
(void*) std::get<0>(itr).c_str(), "previous",
|
||||
_pss.str().c_str(), "current", _css.str().c_str());
|
||||
throw std::runtime_error(fmt::format(
|
||||
"perfetto_counter_track emplace method for '{}' ({:p}) "
|
||||
"invalidated C-string '{}' ({p}).\nprevious: {}\ncurrent: {}\n",
|
||||
_v, (void*) _name->c_str(), std::get<0>(itr),
|
||||
(void*) std::get<0>(itr).c_str(), _pss.str(), _css.str()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "core/defines.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <iostream>
|
||||
#include <ostream>
|
||||
#include <sstream>
|
||||
#include <streambuf>
|
||||
#include <string>
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
inline namespace config
|
||||
{
|
||||
bool
|
||||
get_debug() ROCPROFSYS_HOT;
|
||||
|
||||
int
|
||||
get_verbose() ROCPROFSYS_HOT;
|
||||
} // namespace config
|
||||
|
||||
struct redirect
|
||||
{
|
||||
redirect(std::ostream& _os, std::string _expected)
|
||||
: m_os{ _os }
|
||||
, m_expected{ std::move(_expected) }
|
||||
{
|
||||
if(!get_debug())
|
||||
{
|
||||
// save stream buffer
|
||||
m_strm_buffer = m_os.rdbuf();
|
||||
// redirect to stringstream
|
||||
_os.rdbuf(m_buffer.rdbuf());
|
||||
}
|
||||
}
|
||||
|
||||
~redirect()
|
||||
{
|
||||
if(!m_strm_buffer) return;
|
||||
// restore stream buffer
|
||||
m_os.rdbuf(m_strm_buffer);
|
||||
auto _v = m_buffer.str();
|
||||
_v = replace<3>(m_buffer.str(), { '\n', '\t', ' ' });
|
||||
auto _expect = replace<3>(m_expected, { '\n', '\t', ' ' });
|
||||
if(_v != _expect)
|
||||
{
|
||||
if(get_verbose() > 0)
|
||||
std::cerr
|
||||
<< "[rocprofsys::redirect] Expected:\n[rocprofsys::redirect] "
|
||||
<< _expect
|
||||
<< "\n[rocprofsys::redirect] Found:\n[rocprofsys::redirect] " << _v
|
||||
<< "\n";
|
||||
if(get_verbose() <= 0 || (&m_os != &std::cerr && &m_os != &std::cout))
|
||||
m_os << m_buffer.str() << std::flush;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
template <size_t N>
|
||||
static std::string replace(std::string _v, const std::array<char, N>& _c,
|
||||
const std::string& _s = "")
|
||||
{
|
||||
for(const auto& itr : _c)
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
auto _pos = _v.find(itr);
|
||||
if(_pos == std::string::npos) break;
|
||||
_v = _v.replace(_pos, 1, _s);
|
||||
}
|
||||
}
|
||||
return _v;
|
||||
}
|
||||
|
||||
std::ostream& m_os;
|
||||
std::string m_expected = {};
|
||||
std::stringstream m_buffer{};
|
||||
std::streambuf* m_strm_buffer = nullptr;
|
||||
};
|
||||
} // namespace rocprofsys
|
||||
@@ -23,7 +23,9 @@
|
||||
#include "data_processor.hpp"
|
||||
#include "core/rocpd/data_storage/database.hpp"
|
||||
#include "core/rocpd/data_storage/table_insert_query.hpp"
|
||||
#include "debug.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
|
||||
@@ -141,7 +143,7 @@ data_processor::insert_track(const char* track_name, size_t node_id, size_t proc
|
||||
{
|
||||
if(_tracks.find(track_name) != _tracks.end())
|
||||
{
|
||||
ROCPROFSYS_WARNING(2, "Fail to add track %s, already exist!\n", track_name);
|
||||
LOG_WARNING("Fail to add track {}, already exist!", track_name);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -169,10 +171,10 @@ data_processor::insert_pmc_description(
|
||||
auto it = _pmc_descriptor_map.find({ agent_id, name });
|
||||
if(it != _pmc_descriptor_map.end())
|
||||
{
|
||||
ROCPROFSYS_WARNING(0,
|
||||
"Insert PMC description failed! Error: PMC descriptor "
|
||||
"(name:%s) (ID:%lu) already exist!\n",
|
||||
name, agent_id);
|
||||
LOG_WARNING("Insert PMC description failed! Error: PMC descriptor "
|
||||
"Insert PMC description failed! Error: PMC descriptor "
|
||||
"(name: {}) (ID: {}) already exist!",
|
||||
name, agent_id);
|
||||
return;
|
||||
}
|
||||
data_storage::queries::table_insert_query query_builder;
|
||||
@@ -199,17 +201,15 @@ void
|
||||
data_processor::insert_pmc_event(size_t event_id, size_t agent_id, const char* pmc_name,
|
||||
double value, const char* extdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2,
|
||||
"Insert PMC event: id %ld, agent id: %ld, pmc name: %s, value: "
|
||||
"%lf, extdata: %s\n",
|
||||
event_id, agent_id, pmc_name, value, extdata);
|
||||
LOG_TRACE(
|
||||
"Insert PMC event: id {}, agent id: {}, pmc name: {}, value: {}, extdata: {}",
|
||||
event_id, agent_id, pmc_name, value, extdata);
|
||||
auto it = _pmc_descriptor_map.find({ agent_id, pmc_name });
|
||||
if(it == _pmc_descriptor_map.end())
|
||||
{
|
||||
ROCPROFSYS_WARNING(0,
|
||||
"Insert PMC event failed! Error: non-existing PMC description "
|
||||
"agent id: %ld, pmc name: %s !\n",
|
||||
agent_id, pmc_name);
|
||||
LOG_WARNING("Insert PMC event failed! Error: non-existing PMC description "
|
||||
"agent id: {}, pmc name: {} !",
|
||||
agent_id, pmc_name);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -222,14 +222,12 @@ void
|
||||
data_processor::insert_sample(const char* track, uint64_t timestamp, size_t event_id,
|
||||
const char* extdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
3, "Insert sample: track: %s, timestamp: %lu, event id: %ld, extdata: %s\n",
|
||||
track, timestamp, event_id, extdata);
|
||||
LOG_TRACE("Insert sample: track: {}, timestamp: {}, event id: {}, extdata: {}", track,
|
||||
timestamp, event_id, extdata);
|
||||
auto it = _tracks.find(track);
|
||||
if(it == _tracks.end())
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Insert sample failed! Error: Unexisting track %s!\n",
|
||||
track);
|
||||
LOG_WARNING("Insert sample failed! Error: Unexisting track {}!", track);
|
||||
return;
|
||||
}
|
||||
auto track_info = it->second;
|
||||
@@ -460,7 +458,7 @@ data_processor::insert_code_object(size_t id, size_t node_id, size_t process_id,
|
||||
uint64_t ld_size, uint64_t ld_delta,
|
||||
const char* storage_type, const char* extdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "Insert code object with ID: %ld\n", id);
|
||||
LOG_TRACE("Insert code object with ID: {}", id);
|
||||
_insert_code_object_statement(id, _upid.c_str(), node_id, process_id, agent_id, uri,
|
||||
ld_base, ld_size, ld_delta, storage_type, extdata);
|
||||
}
|
||||
@@ -473,7 +471,7 @@ data_processor::insert_kernel_symbol(
|
||||
uint32_t private_segment_size, uint32_t sgrp_count, uint32_t arch_vgrp_count,
|
||||
uint32_t accum_vgrp_count, const char* extdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "Insert kernel symbol: %s with ID: %ld\n", name, id);
|
||||
LOG_TRACE("Insert kernel symbol: {} with ID: {}", name, id);
|
||||
_insert_kernel_symbol_statement(
|
||||
id, _upid.c_str(), node_id, process_id, code_obj_id, name, display_name,
|
||||
kernel_obj, kernarg_segmnt_size, kernarg_segment_alignment, group_segment_size,
|
||||
@@ -485,7 +483,7 @@ data_processor::insert_region(size_t node_id, size_t process_id, size_t thread_i
|
||||
uint64_t start, uint64_t end, size_t name_id,
|
||||
size_t event_id, const char* extdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "Insert region for event id: %ld\n", event_id);
|
||||
LOG_TRACE("Insert region for event id: {}", event_id);
|
||||
|
||||
_insert_region_statement(_upid.c_str(), node_id, process_id, thread_id, start, end,
|
||||
name_id, event_id, extdata);
|
||||
@@ -500,7 +498,7 @@ data_processor::insert_kernel_dispatch(
|
||||
size_t grid_size_x, size_t grid_size_y, size_t grid_size_z, size_t region_name_id,
|
||||
size_t event_id, const char* extdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "Insert kernel dispatch for event id: %ld\n", event_id);
|
||||
LOG_TRACE("Insert kernel dispatch for event id: {}", event_id);
|
||||
|
||||
_insert_kernel_dispatch_statement(
|
||||
_upid.c_str(), node_id, process_id, thread_id, agent_id, kernel_id, dispatch_id,
|
||||
|
||||
@@ -22,9 +22,10 @@
|
||||
|
||||
#include "database.hpp"
|
||||
#include "common/md5sum.hpp"
|
||||
#include "debug.hpp"
|
||||
#include "node_info.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <config.hpp>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
@@ -95,13 +96,13 @@ load_schema_cb(rocpd_sql_engine_t, rocpd_sql_schema_kind_t, rocpd_sql_options_t,
|
||||
{
|
||||
if(user_data == nullptr || schema_content == nullptr)
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Invalid user data or schema content pointer");
|
||||
LOG_WARNING("Invalid user data or schema content pointer");
|
||||
return;
|
||||
}
|
||||
auto* query = static_cast<std::string*>(user_data);
|
||||
if(query == nullptr)
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Invalid query pointer");
|
||||
LOG_WARNING("Invalid query pointer");
|
||||
return;
|
||||
}
|
||||
*query = std::string(schema_content);
|
||||
@@ -121,7 +122,8 @@ get_schema_query(rocpd_sql_schema_kind_t schema_kind, const std::string& upid)
|
||||
nullptr, 0, &query);
|
||||
if(status != ROCPD_STATUS_SUCCESS)
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Unable to load rocpd schema. Error code: %d", status);
|
||||
LOG_WARNING("Unable to load rocpd schema. Error code: {0:X}",
|
||||
static_cast<int>(status));
|
||||
}
|
||||
return query;
|
||||
#else
|
||||
@@ -144,7 +146,9 @@ get_schema_query(rocpd_sql_schema_kind_t schema_kind, const std::string& upid)
|
||||
case ROCPD_SQL_SCHEMA_ROCPD_SUMMARY_VIEWS:
|
||||
schema_content = rocprofsys::rocpd::data_storage::schema::SUMMARY_VIEWS_SQL;
|
||||
break;
|
||||
default: ROCPROFSYS_WARNING(0, "Unknown schema kind: %d", schema_kind); return "";
|
||||
default:
|
||||
LOG_WARNING("Unknown schema kind: {}", static_cast<int>(schema_kind));
|
||||
return "";
|
||||
}
|
||||
|
||||
return process_schema_template(schema_content, upid);
|
||||
@@ -165,7 +169,7 @@ database::database(int pid, int ppid)
|
||||
auto db_name = std::string{ "rocpd" };
|
||||
auto abs_db_path = rocprofsys::get_database_absolute_path(db_name, _tag);
|
||||
create_directory_for_database_file(abs_db_path);
|
||||
ROCPROFSYS_VERBOSE(0, "Database: %s\r\n", abs_db_path.c_str());
|
||||
LOG_INFO("Database: {}", abs_db_path);
|
||||
|
||||
validate_sqlite3_result(sqlite3_open(":memory:", &_sqlite3_db_temp), "",
|
||||
"database open failed!");
|
||||
@@ -197,8 +201,8 @@ database::initialize_schema()
|
||||
|
||||
if(query.empty())
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Failed to get schema query for schema kind: %d",
|
||||
schema_kind);
|
||||
LOG_WARNING("Failed to get schema query for schema kind: {0:X}",
|
||||
static_cast<int>(schema_kind));
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -22,10 +22,11 @@
|
||||
|
||||
#include "core/rocprofiler-sdk.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "timemory.hpp"
|
||||
#include <regex>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#if defined(ROCPROFSYS_USE_ROCM) && ROCPROFSYS_USE_ROCM > 0
|
||||
|
||||
# include <timemory/defines.h>
|
||||
@@ -48,13 +49,9 @@
|
||||
rocprofiler_status_t CHECKSTATUS = (result); \
|
||||
if(CHECKSTATUS != ROCPROFILER_STATUS_SUCCESS) \
|
||||
{ \
|
||||
auto msg = std::stringstream{}; \
|
||||
std::string status_msg = rocprofiler_get_status_string(CHECKSTATUS); \
|
||||
msg << "[" #result "][" << __FILE__ << ":" << __LINE__ << "] " \
|
||||
<< "rocprofiler-sdk call [" << #result \
|
||||
<< "] failed with error code " << CHECKSTATUS \
|
||||
<< " :: " << status_msg; \
|
||||
ROCPROFSYS_WARNING(0, "%s\n", msg.str().c_str()); \
|
||||
LOG_WARNING("rocprofiler-sdk call [{}] failed with error code {} :: {}", \
|
||||
#result, status_msg); \
|
||||
} \
|
||||
}
|
||||
|
||||
@@ -84,8 +81,8 @@ get_setting_name(std::string _v)
|
||||
__VA_ARGS__ }); \
|
||||
if(!_ret.second) \
|
||||
{ \
|
||||
ROCPROFSYS_PRINT("Warning! Duplicate setting: %s / %s\n", \
|
||||
get_setting_name(ENV_NAME).c_str(), ENV_NAME); \
|
||||
LOG_WARNING("Duplicate setting: {} / {}", get_setting_name(ENV_NAME), \
|
||||
ENV_NAME); \
|
||||
} \
|
||||
return _config->find(ENV_NAME)->second; \
|
||||
}()
|
||||
@@ -131,7 +128,12 @@ get_operations_impl(rocprofiler_callback_tracing_kind_t kindv,
|
||||
|
||||
auto _val = get_setting_value<std::string>(optname);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(!_val, "no setting %s\n", optname.c_str());
|
||||
if(!_val)
|
||||
{
|
||||
LOG_CRITICAL("no setting {}", optname);
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
if(_val->empty()) return std::unordered_set<int32_t>{};
|
||||
|
||||
@@ -143,8 +145,7 @@ get_operations_impl(rocprofiler_callback_tracing_kind_t kindv,
|
||||
auto _re = std::regex{ itr, std::regex_constants::icase };
|
||||
if(iitr.second && std::regex_search(iitr.second->data(), _re))
|
||||
{
|
||||
ROCPROFSYS_PRINT_F("%s ('%s') matched: %s\n", optname.c_str(),
|
||||
itr.c_str(), iitr.second->data());
|
||||
LOG_DEBUG("{} ('{}') matched: {}", optname, itr, iitr.second->data());
|
||||
_ret.emplace(iitr.first);
|
||||
}
|
||||
}
|
||||
@@ -172,7 +173,12 @@ get_operations_impl(rocprofiler_buffer_tracing_kind_t kindv,
|
||||
|
||||
auto _val = get_setting_value<std::string>(optname);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(!_val, "no setting %s\n", optname.c_str());
|
||||
if(!_val)
|
||||
{
|
||||
LOG_CRITICAL("no setting {}", optname);
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
if(_val->empty()) return std::unordered_set<int32_t>{};
|
||||
|
||||
@@ -184,8 +190,7 @@ get_operations_impl(rocprofiler_buffer_tracing_kind_t kindv,
|
||||
auto _re = std::regex{ itr, std::regex_constants::icase };
|
||||
if(iitr.second && std::regex_search(iitr.second->data(), _re))
|
||||
{
|
||||
ROCPROFSYS_PRINT_F("%s ('%s') matched: %s\n", optname.c_str(),
|
||||
itr.c_str(), iitr.second->data());
|
||||
LOG_DEBUG("{} ('{}') matched: {}", optname, itr, iitr.second->data());
|
||||
_ret.emplace(iitr.first);
|
||||
}
|
||||
}
|
||||
@@ -407,8 +412,10 @@ get_callback_domains()
|
||||
};
|
||||
|
||||
auto _version = get_version();
|
||||
ROCPROFSYS_WARNING_IF(_version.formatted == 0,
|
||||
"Warning! rocprofiler-sdk version not initialized\n");
|
||||
if(_version.formatted == 0)
|
||||
{
|
||||
LOG_WARNING("rocprofiler-sdk version not initialized");
|
||||
}
|
||||
|
||||
# if(ROCPROFILER_VERSION >= 600)
|
||||
if(_version.formatted >= 600)
|
||||
@@ -459,8 +466,8 @@ get_callback_domains()
|
||||
{
|
||||
if(invalid_domain(itr))
|
||||
{
|
||||
ROCPROFSYS_THROW("unsupported ROCPROFSYS_ROCM_DOMAINS value: %s\n",
|
||||
itr.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("unsupported ROCPROFSYS_ROCM_DOMAINS value: {}", itr));
|
||||
}
|
||||
|
||||
if(itr == "hsa_api")
|
||||
@@ -532,8 +539,8 @@ get_buffered_domains()
|
||||
{
|
||||
if(invalid_domain(itr))
|
||||
{
|
||||
ROCPROFSYS_THROW("unsupported ROCPROFSYS_ROCM_DOMAINS value: %s\n",
|
||||
itr.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("unsupported ROCPROFSYS_ROCM_DOMAINS value: {}", itr));
|
||||
}
|
||||
|
||||
if(itr == "hsa_api")
|
||||
@@ -603,9 +610,13 @@ get_group_by_queue(void)
|
||||
std::vector<int32_t>
|
||||
get_operations(rocprofiler_callback_tracing_kind_t kindv)
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(
|
||||
callback_operation_option_names.count(kindv) == 0,
|
||||
"callback_operation_operation_names does not have value for %i\n", kindv);
|
||||
if(callback_operation_option_names.count(kindv) == 0)
|
||||
{
|
||||
LOG_CRITICAL("callback_operation_operation_names does not have value for {}",
|
||||
static_cast<int>(kindv));
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
auto _complete = get_operations_impl(kindv);
|
||||
auto _include = get_operations_impl(
|
||||
@@ -619,9 +630,13 @@ get_operations(rocprofiler_callback_tracing_kind_t kindv)
|
||||
std::vector<int32_t>
|
||||
get_operations(rocprofiler_buffer_tracing_kind_t kindv)
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(
|
||||
buffered_operation_option_names.count(kindv) == 0,
|
||||
"buffered_operation_option_names does not have value for %i\n", kindv);
|
||||
if(buffered_operation_option_names.count(kindv) == 0)
|
||||
{
|
||||
LOG_CRITICAL("buffered_operation_option_names does not have value for {}",
|
||||
static_cast<int>(kindv));
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
auto _complete = get_operations_impl(kindv);
|
||||
auto _include = get_operations_impl(
|
||||
@@ -635,9 +650,13 @@ get_operations(rocprofiler_buffer_tracing_kind_t kindv)
|
||||
std::unordered_set<int32_t>
|
||||
get_backtrace_operations(rocprofiler_callback_tracing_kind_t kindv)
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(
|
||||
callback_operation_option_names.count(kindv) == 0,
|
||||
"callback_operation_operation_names does not have value for %i\n", kindv);
|
||||
if(callback_operation_option_names.count(kindv) == 0)
|
||||
{
|
||||
LOG_CRITICAL("callback_operation_operation_names does not have value for {}",
|
||||
static_cast<int>(kindv));
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
auto _data = get_operations_impl(
|
||||
kindv, callback_operation_option_names.at(kindv).operations_annotate_backtrace);
|
||||
@@ -651,9 +670,13 @@ get_backtrace_operations(rocprofiler_callback_tracing_kind_t kindv)
|
||||
std::unordered_set<int32_t>
|
||||
get_backtrace_operations(rocprofiler_buffer_tracing_kind_t kindv)
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(
|
||||
buffered_operation_option_names.count(kindv) == 0,
|
||||
"buffered_operation_option_names does not have value for %i\n", kindv);
|
||||
if(buffered_operation_option_names.count(kindv) == 0)
|
||||
{
|
||||
LOG_CRITICAL("buffered_operation_option_names does not have value for {}",
|
||||
static_cast<int>(kindv));
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
auto _data = get_operations_impl(
|
||||
kindv, buffered_operation_option_names.at(kindv).operations_annotate_backtrace);
|
||||
|
||||
@@ -23,9 +23,10 @@
|
||||
#include "state.hpp"
|
||||
#include "common/static_object.hpp"
|
||||
#include "config.hpp"
|
||||
#include "debug.hpp"
|
||||
#include "utility.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
|
||||
@@ -79,13 +80,19 @@ get_thread_state()
|
||||
State
|
||||
set_state(State _n)
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_PRINT_F(get_debug_init(), "Setting state :: %s -> %s\n",
|
||||
std::to_string(get_state()).c_str(),
|
||||
std::to_string(_n).c_str());
|
||||
if(get_debug_init())
|
||||
{
|
||||
LOG_DEBUG("Setting state :: {} -> {}", std::to_string(get_state()),
|
||||
std::to_string(_n));
|
||||
}
|
||||
// state should always be increased, not decreased
|
||||
ROCPROFSYS_CI_BASIC_THROW(
|
||||
_n < get_state(), "State is being assigned to a lesser value :: %s -> %s",
|
||||
std::to_string(get_state()).c_str(), std::to_string(_n).c_str());
|
||||
if(get_is_continuous_integration() && _n < get_state())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("State is being assigned to a lesser value :: {} -> {}",
|
||||
std::to_string(get_state()), std::to_string(_n)));
|
||||
}
|
||||
|
||||
auto _v = get_state();
|
||||
get_state_value().store(_n, std::memory_order_relaxed);
|
||||
// std::swap(get_state_value(), _n);
|
||||
|
||||
@@ -22,6 +22,8 @@
|
||||
|
||||
#include "buffer_storage.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
@@ -50,15 +52,17 @@ flush_worker_t::start(const pid_t& current_pid)
|
||||
{
|
||||
if(m_worker_synchronization->is_running)
|
||||
{
|
||||
std::stringstream _ss;
|
||||
_ss << "Flush worker is already running";
|
||||
throw std::runtime_error(_ss.str());
|
||||
LOG_WARNING("Flush worker is already running for pid={}", current_pid);
|
||||
throw std::runtime_error("Flush worker is already running");
|
||||
}
|
||||
|
||||
LOG_DEBUG("Starting flush worker for pid={}, filepath={}", current_pid, m_filepath);
|
||||
|
||||
m_ofs = std::ofstream{ m_filepath, std::ios::binary | std::ios::out };
|
||||
|
||||
if(!m_ofs.good())
|
||||
{
|
||||
LOG_CRITICAL("Failed to open file for writing: {}", m_filepath);
|
||||
std::stringstream _ss;
|
||||
_ss << "Error opening file for writing: " << m_filepath;
|
||||
throw std::runtime_error(_ss.str());
|
||||
@@ -68,6 +72,8 @@ flush_worker_t::start(const pid_t& current_pid)
|
||||
m_worker_synchronization->is_running = true;
|
||||
|
||||
m_flushing_thread = std::make_unique<std::thread>([&]() {
|
||||
LOG_TRACE("Flush worker thread started for pid={}",
|
||||
m_worker_synchronization->origin_pid);
|
||||
std::mutex _shutdown_condition_mutex;
|
||||
while(m_worker_synchronization->is_running)
|
||||
{
|
||||
@@ -78,21 +84,28 @@ flush_worker_t::start(const pid_t& current_pid)
|
||||
[&]() { return !m_worker_synchronization->is_running; });
|
||||
}
|
||||
|
||||
LOG_TRACE("Flush worker thread performing final flush");
|
||||
m_worker_function(m_ofs, true);
|
||||
m_ofs.close();
|
||||
m_worker_synchronization->exit_finished = true;
|
||||
m_worker_synchronization->exit_finished_condition.notify_one();
|
||||
LOG_TRACE("Flush worker thread exiting");
|
||||
});
|
||||
|
||||
LOG_DEBUG("Flush worker started successfully for pid={}", current_pid);
|
||||
}
|
||||
void
|
||||
flush_worker_t::stop(const pid_t& current_pid)
|
||||
{
|
||||
LOG_DEBUG("Stopping flush worker for pid={}", current_pid);
|
||||
|
||||
const bool flushing_thread_exist = m_flushing_thread != nullptr;
|
||||
const bool worker_is_running =
|
||||
m_worker_synchronization != nullptr && m_worker_synchronization->is_running;
|
||||
|
||||
if(flushing_thread_exist && worker_is_running)
|
||||
{
|
||||
LOG_TRACE("Signaling flush worker to stop");
|
||||
m_worker_synchronization->is_running = false;
|
||||
m_worker_synchronization->is_running_condition.notify_all();
|
||||
|
||||
@@ -100,9 +113,11 @@ flush_worker_t::stop(const pid_t& current_pid)
|
||||
current_pid == m_worker_synchronization->origin_pid;
|
||||
if(!thread_is_created_in_this_process)
|
||||
{
|
||||
LOG_DEBUG("Flush worker was created in different process, skipping join");
|
||||
return;
|
||||
}
|
||||
|
||||
LOG_TRACE("Waiting for flush worker thread to finish");
|
||||
std::mutex _exit_mutex;
|
||||
std::unique_lock _exit_lock{ _exit_mutex };
|
||||
m_worker_synchronization->exit_finished_condition.wait(
|
||||
@@ -112,7 +127,14 @@ flush_worker_t::stop(const pid_t& current_pid)
|
||||
{
|
||||
m_flushing_thread->join();
|
||||
m_flushing_thread.reset();
|
||||
LOG_TRACE("Flush worker thread joined successfully");
|
||||
}
|
||||
|
||||
LOG_DEBUG("Flush worker stopped for pid={}", current_pid);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_TRACE("Flush worker not running or thread doesn't exist, nothing to stop");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,9 +29,9 @@
|
||||
|
||||
#include "core/agent_manager.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
|
||||
#include "library/runtime.hpp"
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
@@ -85,9 +85,9 @@ struct enabled_formats_t
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_PRINT(
|
||||
"Generating [%s] format(s) with collected data from trace cache. This may "
|
||||
"take a while..\n",
|
||||
LOG_INFO(
|
||||
"Generating [{}] format(s) with collected data from trace cache. This may "
|
||||
"take a while..",
|
||||
ss.str().c_str());
|
||||
|
||||
if(has_parallel_formats())
|
||||
@@ -103,8 +103,7 @@ struct enabled_formats_t
|
||||
first_parallel = false;
|
||||
}
|
||||
}
|
||||
ROCPROFSYS_PRINT(" - Using parallel processing for: %s\n",
|
||||
parallel_ss.str().c_str());
|
||||
LOG_INFO(" - Using parallel processing for: {}", parallel_ss.str());
|
||||
}
|
||||
|
||||
if(has_sequential_formats())
|
||||
@@ -120,8 +119,8 @@ struct enabled_formats_t
|
||||
first_sequential = false;
|
||||
}
|
||||
}
|
||||
ROCPROFSYS_PRINT(" - Using sequential processing for: %s\n",
|
||||
sequential_ss.str().c_str());
|
||||
LOG_INFO(" - Using sequential processing for: {}",
|
||||
sequential_ss.str().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,16 +220,16 @@ remove_if_exists(const std::string& fname)
|
||||
auto result = std::remove(fname.c_str());
|
||||
if(result == 0)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("Removed file: %s\n", fname.c_str());
|
||||
LOG_DEBUG("Removed file: {}", fname);
|
||||
}
|
||||
else if(errno == ENOENT)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("File does not exist: %s\n", fname.c_str());
|
||||
LOG_DEBUG("File does not exist: {}", fname);
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Failed to remove file: %s (errno: %d - %s)\n",
|
||||
fname.c_str(), errno, std::strerror(errno));
|
||||
LOG_WARNING("Failed to remove file: {} (errno: {} - {})", fname, errno,
|
||||
std::strerror(errno));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -251,7 +250,7 @@ list_dir_files(const std::string& _path)
|
||||
|
||||
if(!dir)
|
||||
{
|
||||
ROCPROFSYS_THROW("Error opening directory: %s", _path.c_str());
|
||||
throw std::runtime_error(fmt::format("Error opening directory: {}", _path));
|
||||
}
|
||||
|
||||
data::directory_files_t result{};
|
||||
@@ -317,14 +316,13 @@ get_cache_files(const pid_t& root_pid,
|
||||
void
|
||||
clear_cache_files(const data::mapped_cache_files_t& _cache_files)
|
||||
{
|
||||
ROCPROFSYS_PRINT("Removing cached temporary files...\n");
|
||||
LOG_DEBUG("Removing cached temporary files...");
|
||||
for(const auto& [_, files] : _cache_files)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("Removing cached temporary file: %s\n",
|
||||
files.buff_storage.c_str());
|
||||
LOG_DEBUG("Removing cached temporary file: {}", files.buff_storage);
|
||||
filesystem_utils::remove_if_exists(files.buff_storage);
|
||||
|
||||
ROCPROFSYS_DEBUG("Removing cached temporary file: %s\n", files.metadata.c_str());
|
||||
LOG_DEBUG("Removing cached temporary file: {}", files.metadata);
|
||||
filesystem_utils::remove_if_exists(files.metadata);
|
||||
}
|
||||
}
|
||||
@@ -335,9 +333,8 @@ merge_perfetto_files(const std::vector<std::string>& perfetto_files,
|
||||
{
|
||||
if(perfetto_files.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0, "perfetto trace data is empty. File '%s' will not be written...\n",
|
||||
_filename.c_str());
|
||||
LOG_ERROR("Perfetto trace data is empty. File '{}' will not be written...",
|
||||
_filename);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -362,7 +359,7 @@ merge_perfetto_files(const std::vector<std::string>& perfetto_files,
|
||||
std::ifstream ifs(file, std::ios::binary);
|
||||
if(!ifs)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(-1, "Error opening '%s'...\n", file.c_str());
|
||||
LOG_ERROR("Error opening '{}'...", file);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -401,9 +398,8 @@ merge_perfetto_files(const std::vector<std::string>& perfetto_files,
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0, "perfetto trace data is empty. File '%s' will not be written...\n",
|
||||
_filename.c_str());
|
||||
LOG_ERROR("Perfetto trace data is empty. File '{}' will not be written...",
|
||||
_filename);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,6 +435,9 @@ process_buffered_storage(
|
||||
const std::shared_ptr<data::processor_config_t>& _processor_config,
|
||||
const std::string& _storage_filename, const data::enabled_formats_t& _enabled_formats)
|
||||
{
|
||||
LOG_DEBUG("Processing buffered storage: {} for pid={}", _storage_filename,
|
||||
_processor_config->_pid);
|
||||
|
||||
auto _processor_coordinator = std::make_shared<sample_processor_t>();
|
||||
auto processor_storage =
|
||||
configure_processors(_processor_coordinator, _processor_config, _enabled_formats);
|
||||
@@ -448,12 +447,15 @@ process_buffered_storage(
|
||||
try
|
||||
{
|
||||
_parser.load(_processor_coordinator);
|
||||
|
||||
LOG_TRACE("Successfully loaded buffered storage: {}", _storage_filename);
|
||||
} catch(const std::runtime_error& exp)
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Error parsing buffered storage: %s\n", exp.what());
|
||||
LOG_WARNING("Error parsing buffered storage {}: {}", _storage_filename,
|
||||
exp.what());
|
||||
}
|
||||
_processor_coordinator->finalize_processing();
|
||||
|
||||
LOG_DEBUG("Finished processing buffered storage: {}", _storage_filename);
|
||||
}
|
||||
|
||||
std::vector<std::shared_ptr<data::processor_config_t>>
|
||||
@@ -489,12 +491,15 @@ multithreaded_processing(
|
||||
const std::vector<std::shared_ptr<data::processor_config_t>>& _processor_configs,
|
||||
const data::enabled_formats_t& _enabled_formats)
|
||||
{
|
||||
LOG_DEBUG("Starting multithreaded processing with {} configs",
|
||||
_processor_configs.size());
|
||||
ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
|
||||
|
||||
std::vector<std::thread> processing_threads;
|
||||
processing_threads.reserve(_processor_configs.size());
|
||||
for(const auto& processor_config : _processor_configs)
|
||||
{
|
||||
LOG_TRACE("Spawning processing thread for pid={}", processor_config->_pid);
|
||||
processing_threads.emplace_back(
|
||||
process_buffered_storage, processor_config,
|
||||
utility::get_buffered_storage_filename(processor_config->_ppid,
|
||||
@@ -502,10 +507,12 @@ multithreaded_processing(
|
||||
_enabled_formats);
|
||||
}
|
||||
|
||||
LOG_TRACE("Waiting for {} processing threads to complete", processing_threads.size());
|
||||
for(auto& thread : processing_threads)
|
||||
{
|
||||
thread.join();
|
||||
}
|
||||
LOG_DEBUG("Multithreaded processing completed");
|
||||
}
|
||||
|
||||
void
|
||||
@@ -513,13 +520,17 @@ sequential_processing(
|
||||
const std::vector<std::shared_ptr<data::processor_config_t>>& _processor_configs,
|
||||
const data::enabled_formats_t& _enabled_formats)
|
||||
{
|
||||
LOG_DEBUG("Starting sequential processing with {} configs",
|
||||
_processor_configs.size());
|
||||
for(const auto& processor_config : _processor_configs)
|
||||
{
|
||||
LOG_TRACE("Processing config for pid={}", processor_config->_pid);
|
||||
process_buffered_storage(processor_config,
|
||||
utility::get_buffered_storage_filename(
|
||||
processor_config->_ppid, processor_config->_pid),
|
||||
_enabled_formats);
|
||||
}
|
||||
LOG_DEBUG("Sequential processing completed");
|
||||
}
|
||||
|
||||
void
|
||||
@@ -551,24 +562,33 @@ cache_manager::get_instance()
|
||||
void
|
||||
cache_manager::post_process_bulk()
|
||||
{
|
||||
LOG_TRACE("Starting trace cache bulk post-processing");
|
||||
|
||||
if(!is_root_process())
|
||||
{
|
||||
LOG_DEBUG("Not root process, skipping bulk post-processing");
|
||||
return;
|
||||
}
|
||||
|
||||
if(m_storage.is_running())
|
||||
{
|
||||
ROCPROFSYS_WARNING(2, "Postprocessing called without previously shutting down "
|
||||
"cache storage. Calling shutdown explicitly..\n");
|
||||
LOG_WARNING(
|
||||
"Post-processing called without previously shutting down cache storage"
|
||||
"cache storage. Calling shutdown explicitly..");
|
||||
shutdown();
|
||||
}
|
||||
|
||||
const auto root_pid = get_root_process_id();
|
||||
LOG_DEBUG("Root process ID: {}", root_pid);
|
||||
|
||||
const auto temp_directory_content =
|
||||
filesystem_utils::list_dir_files(trace_cache::tmp_directory);
|
||||
LOG_TRACE("Found {} files in temp directory", temp_directory_content.size());
|
||||
|
||||
const auto cache_files =
|
||||
filesystem_utils::get_cache_files(root_pid, temp_directory_content);
|
||||
LOG_DEBUG("Found {} cache file pairs to process", cache_files.size());
|
||||
|
||||
const data::enabled_formats_t enabled_formats;
|
||||
enabled_formats.print();
|
||||
|
||||
@@ -579,6 +599,7 @@ cache_manager::post_process_bulk()
|
||||
getpid(), root_pid, m_metadata,
|
||||
std::make_shared<agent_manager>(get_agent_manager_instance().get_agents())));
|
||||
|
||||
LOG_INFO("Processing {} trace cache configurations", processor_configs.size());
|
||||
processing_utils::dispatch_processing(processor_configs, enabled_formats);
|
||||
|
||||
if(enabled_formats.is_perfetto_enabled())
|
||||
@@ -611,26 +632,28 @@ cache_manager::post_process_bulk()
|
||||
}
|
||||
else if(perfetto_files.size() > 1)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0,
|
||||
"Generated %zu separate perfetto trace files. "
|
||||
"Set ROCPROFSYS_PERFETTO_COMBINE_TRACES=ON to merge them.\n",
|
||||
perfetto_files.size());
|
||||
LOG_INFO("Generated {} separate perfetto trace files. "
|
||||
"Set ROCPROFSYS_PERFETTO_COMBINE_TRACES=ON to merge them.",
|
||||
perfetto_files.size());
|
||||
|
||||
for(const auto& file : perfetto_files)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(1, " - %s\n", file.c_str());
|
||||
LOG_INFO(" - {}", file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
filesystem_utils::clear_cache_files(cache_files);
|
||||
|
||||
LOG_TRACE("Trace cache bulk post-processing completed");
|
||||
}
|
||||
|
||||
void
|
||||
cache_manager::shutdown()
|
||||
{
|
||||
LOG_DEBUG("Shutting down cache manager storage");
|
||||
m_storage.shutdown();
|
||||
LOG_TRACE("Cache manager storage shutdown complete");
|
||||
}
|
||||
|
||||
} // namespace trace_cache
|
||||
|
||||
+36
-17
@@ -22,7 +22,10 @@
|
||||
|
||||
#include "metadata_registry.hpp"
|
||||
#include "agent_manager.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include "core/config.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
|
||||
@@ -467,7 +470,7 @@ from_json(metadata_registry& _registry, std::vector<std::shared_ptr<agent>>& _ag
|
||||
|
||||
if(!_agents.empty())
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Given agents vector is not empty. Clearing it..");
|
||||
LOG_WARNING("Given agents vector is not empty. Clearing it..");
|
||||
_agents.clear();
|
||||
}
|
||||
|
||||
@@ -759,21 +762,30 @@ metadata_registry::overwrite_callback_names(
|
||||
modified_ops[i] = extract_operations(i);
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(modified_ops.find(callback_kind) != modified_ops.end(),
|
||||
"Overwriting a previously overwritten entry is forbidden");
|
||||
if(get_is_continuous_integration() &&
|
||||
modified_ops.find(callback_kind) != modified_ops.end())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
"Overwriting a previously overwritten entry is forbidden");
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(!modified_ops.empty() &&
|
||||
callback_kind >= modified_ops.begin()->first,
|
||||
"Category must have a larger enum value than all previously "
|
||||
"modified_ops categories");
|
||||
if(get_is_continuous_integration() && !modified_ops.empty() &&
|
||||
callback_kind >= modified_ops.begin()->first)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
"Category must have a larger enum value than all previously "
|
||||
"modified_ops categories");
|
||||
}
|
||||
|
||||
// Overwrite desired category
|
||||
auto operation_names = extract_operations(callback_kind);
|
||||
for(const auto& [index, new_value] : category_info.second)
|
||||
{
|
||||
ROCPROFSYS_CI_THROW(index < 0 ||
|
||||
static_cast<size_t>(index) >= operation_names.size(),
|
||||
"Index is invalid");
|
||||
if(get_is_continuous_integration() &&
|
||||
(index < 0 || static_cast<size_t>(index) >= operation_names.size()))
|
||||
{
|
||||
throw std::runtime_error("Index is invalid");
|
||||
}
|
||||
operation_names[index] = new_value;
|
||||
}
|
||||
modified_ops[callback_kind] = std::move(operation_names);
|
||||
@@ -787,8 +799,10 @@ metadata_registry::overwrite_callback_names(
|
||||
{
|
||||
auto renaming_entry = modified_ops.find(i);
|
||||
|
||||
ROCPROFSYS_CI_THROW(renaming_entry == modified_ops.end(),
|
||||
"A category that needs to be emplaced is missing");
|
||||
if(get_is_continuous_integration() && renaming_entry == modified_ops.end())
|
||||
{
|
||||
throw std::runtime_error("A category that needs to be emplaced is missing");
|
||||
}
|
||||
|
||||
const auto& operations_vec = renaming_entry->second;
|
||||
m_callback_tracing_info.emplace(i, category_names.at(i).data());
|
||||
@@ -832,6 +846,7 @@ bool
|
||||
metadata_registry::save_to_file(const std::string& filepath,
|
||||
const std::vector<std::shared_ptr<agent>>& _agents) const
|
||||
{
|
||||
LOG_DEBUG("Saving metadata registry to file: {}", filepath);
|
||||
try
|
||||
{
|
||||
auto json = to_json(*this, _agents);
|
||||
@@ -840,16 +855,17 @@ metadata_registry::save_to_file(const std::string& filep
|
||||
std::ofstream file(filepath);
|
||||
if(!file.is_open())
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Error opening file for writing: %s", filepath.c_str());
|
||||
LOG_WARNING("Failed to open file for writing: {}", filepath);
|
||||
return false;
|
||||
}
|
||||
|
||||
file << json_string;
|
||||
file.close();
|
||||
LOG_INFO("Metadata registry saved successfully to: {}", filepath);
|
||||
return true;
|
||||
} catch(const std::exception& e)
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Error saving metadata to file: %s", e.what());
|
||||
LOG_ERROR("Exception while saving metadata to file {}: {}", filepath, e.what());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -858,12 +874,13 @@ bool
|
||||
metadata_registry::load_from_file(const std::string& filepath,
|
||||
std::vector<std::shared_ptr<agent>>& _agents)
|
||||
{
|
||||
LOG_DEBUG("Loading metadata registry from file: {}", filepath);
|
||||
try
|
||||
{
|
||||
std::ifstream file(filepath);
|
||||
if(!file.is_open())
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Error opening file for reading: %s", filepath.c_str());
|
||||
LOG_WARNING("Failed to open file for reading: {}", filepath);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -872,10 +889,12 @@ metadata_registry::load_from_file(const std::string& filepath,
|
||||
file.close();
|
||||
|
||||
rocprofsys::trace_cache::from_json(*this, _agents, json);
|
||||
LOG_INFO("Metadata registry loaded successfully from: {}", filepath);
|
||||
return true;
|
||||
} catch(const std::exception& e)
|
||||
{
|
||||
ROCPROFSYS_WARNING(1, "Error loading metadata from file: %s", e.what());
|
||||
LOG_ERROR("Exception while loading metadata from file {}: {}", filepath,
|
||||
e.what());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+58
-35
@@ -21,19 +21,20 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "core/trace_cache/perfetto_processor.hpp"
|
||||
#include "common.hpp"
|
||||
#include "core/agent_manager.hpp"
|
||||
#include "core/categories.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/common_types.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/gpu_metrics.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/utility.hpp"
|
||||
#include "library/tracing.hpp"
|
||||
#include "perfetto.hpp"
|
||||
#include "trace_cache/metadata_registry.hpp"
|
||||
#include "trace_cache/sample_type.hpp"
|
||||
#include "trace_cache/storage_parser.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
#include <cstdint>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
@@ -176,8 +177,7 @@ write_sampling_track_data(const struct backtrace_region_sample& _sample,
|
||||
}
|
||||
} catch(const std::exception& e)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(2, "Failed to parse call_stack JSON: %s\n",
|
||||
e.what());
|
||||
LOG_WARNING("Failed to parse call_stack JSON: {}", e.what());
|
||||
}
|
||||
}
|
||||
annotate_perfetto(ctx, annotations);
|
||||
@@ -260,24 +260,31 @@ perfetto_processor_t::initialize_perfetto()
|
||||
{
|
||||
static std::once_flag init_flag;
|
||||
std::call_once(init_flag, []() {
|
||||
LOG_DEBUG("Initializing perfetto tracing backend");
|
||||
auto args = ::perfetto::TracingInitArgs{};
|
||||
args.backends = ::perfetto::kInProcessBackend;
|
||||
args.shmem_size_hint_kb = config::get_perfetto_shmem_size_hint();
|
||||
|
||||
::perfetto::Tracing::Initialize(args);
|
||||
::perfetto::TrackEvent::Register(); // Only register once globally!
|
||||
::perfetto::TrackEvent::Register();
|
||||
LOG_TRACE("Perfetto tracing backend initialized");
|
||||
});
|
||||
}
|
||||
|
||||
void
|
||||
perfetto_processor_t::setup_perfetto()
|
||||
{
|
||||
LOG_DEBUG("Setting up perfetto configuration for pid={}", m_process_id);
|
||||
|
||||
auto track_event_cfg = ::perfetto::protos::gen::TrackEventConfig{};
|
||||
auto& cfg = m_session_config;
|
||||
|
||||
auto perfetto_buffer_size = config::get_perfetto_buffer_size();
|
||||
auto flush_period = config::get_perfetto_flush_period();
|
||||
|
||||
LOG_TRACE("Perfetto buffer size: {} KB, flush period: {} ms", perfetto_buffer_size,
|
||||
flush_period);
|
||||
|
||||
auto _policy =
|
||||
config::get_perfetto_fill_policy() == "discard"
|
||||
? ::perfetto::protos::gen::TraceConfig_BufferConfig_FillPolicy_DISCARD
|
||||
@@ -288,31 +295,36 @@ perfetto_processor_t::setup_perfetto()
|
||||
|
||||
for(const auto& itr : config::get_disabled_categories())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Disabling perfetto track event category: %s\n",
|
||||
itr.c_str());
|
||||
LOG_TRACE("Disabling perfetto track event category: {}", itr);
|
||||
track_event_cfg.add_disabled_categories(itr);
|
||||
}
|
||||
|
||||
cfg.set_flush_period_ms(flush_period);
|
||||
|
||||
auto* ds_cfg = cfg.add_data_sources()->mutable_config();
|
||||
ds_cfg->set_name("track_event"); // this MUST be track_event
|
||||
ds_cfg->set_name("track_event");
|
||||
ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString());
|
||||
|
||||
LOG_TRACE("Perfetto configuration setup complete");
|
||||
}
|
||||
|
||||
void
|
||||
perfetto_processor_t::start_session()
|
||||
{
|
||||
if(config::get_perfetto_backend() != "inprocess") return;
|
||||
if(config::get_perfetto_backend() != "inprocess")
|
||||
{
|
||||
LOG_TRACE("Perfetto backend is not 'inprocess', skipping session start");
|
||||
return;
|
||||
}
|
||||
|
||||
LOG_DEBUG("Starting perfetto tracing session for pid={}", m_process_id);
|
||||
|
||||
if(!m_tracing_session)
|
||||
{
|
||||
m_tracing_session = ::perfetto::Tracing::NewTrace();
|
||||
LOG_TRACE("Created new perfetto trace");
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(2,
|
||||
"Starting perfetto post-processing session with cached data...\n");
|
||||
|
||||
int temp_fd = -1;
|
||||
if(config::get_use_tmp_files())
|
||||
{
|
||||
@@ -320,20 +332,28 @@ perfetto_processor_t::start_session()
|
||||
m_tmp_file = config::get_tmp_file(_base, "proto");
|
||||
m_tmp_file->open(O_RDWR | O_CREAT | O_TRUNC, 0600);
|
||||
temp_fd = m_tmp_file->fd;
|
||||
LOG_TRACE("Using temp file for perfetto trace: {}", m_tmp_file->filename);
|
||||
}
|
||||
m_tracing_session->Setup(m_session_config, temp_fd);
|
||||
m_tracing_session->StartBlocking();
|
||||
|
||||
LOG_TRACE("Perfetto tracing session started for pid={}", m_process_id);
|
||||
}
|
||||
|
||||
void
|
||||
perfetto_processor_t::stop_session()
|
||||
{
|
||||
if(!m_tracing_session) return;
|
||||
if(!m_tracing_session)
|
||||
{
|
||||
LOG_TRACE("No active perfetto session to stop");
|
||||
return;
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(2, "Stopping perfetto post-processing session...\n");
|
||||
LOG_DEBUG("Stopping perfetto tracing session for pid={}", m_process_id);
|
||||
::perfetto::TrackEvent::Flush();
|
||||
m_tracing_session->FlushBlocking();
|
||||
m_tracing_session->StopBlocking();
|
||||
LOG_TRACE("Perfetto tracing session stopped");
|
||||
}
|
||||
|
||||
char_vec_t
|
||||
@@ -347,9 +367,8 @@ perfetto_processor_t::get_session_data()
|
||||
|
||||
if(!_fdata)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(-1,
|
||||
"Error! perfetto temp trace file '%s' could not be read",
|
||||
m_tmp_file->filename.c_str());
|
||||
LOG_ERROR("Perfetto temp trace file '{}' could not be read",
|
||||
m_tmp_file->filename);
|
||||
return char_vec_t{ m_tracing_session->ReadTraceBlocking() };
|
||||
}
|
||||
|
||||
@@ -361,10 +380,12 @@ perfetto_processor_t::get_session_data()
|
||||
auto _fnum_read = ::fread(_data.data(), sizeof(char), _fnum_elem, _fdata);
|
||||
::fclose(_fdata);
|
||||
|
||||
ROCPROFSYS_CI_THROW(
|
||||
_fnum_read != _fnum_elem,
|
||||
"Error! read %zu elements from perfetto trace file '%s'. Expected %zu\n",
|
||||
_fnum_read, m_tmp_file->filename.c_str(), _fnum_elem);
|
||||
if(get_is_continuous_integration() && _fnum_read != _fnum_elem)
|
||||
{
|
||||
throw std::runtime_error(fmt::format(
|
||||
"Error! read {} elements from perfetto trace file '{}'. Expected {}",
|
||||
_fnum_read, m_tmp_file->filename, _fnum_elem));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -417,9 +438,8 @@ perfetto_processor_t::flush(bool& _perfetto_output_error)
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0, "perfetto trace data is empty. File '%s' will not be written...\n",
|
||||
_filename.c_str());
|
||||
LOG_ERROR("Perfetto trace data is empty. File '{}' will not be written...",
|
||||
_filename.c_str());
|
||||
}
|
||||
|
||||
if(m_tmp_file)
|
||||
@@ -435,21 +455,27 @@ perfetto_processor_t::flush(bool& _perfetto_output_error)
|
||||
void
|
||||
perfetto_processor_t::prepare_for_processing()
|
||||
{
|
||||
LOG_DEBUG("Preparing perfetto processor for pid={}", m_process_id);
|
||||
initialize_perfetto();
|
||||
setup_perfetto();
|
||||
start_session();
|
||||
LOG_TRACE("Perfetto processor prepared for processing");
|
||||
}
|
||||
|
||||
void
|
||||
perfetto_processor_t::finalize_processing()
|
||||
{
|
||||
LOG_DEBUG("Finalizing perfetto processor for pid={}", m_process_id);
|
||||
bool _perfetto_output_error = false;
|
||||
flush(_perfetto_output_error);
|
||||
|
||||
if(_perfetto_output_error)
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "Perfetto trace generation failed for process: %lu\n",
|
||||
m_process_id);
|
||||
LOG_ERROR("Perfetto trace generation failed for pid={}", m_process_id);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_DEBUG("Perfetto processing finalized successfully for pid={}", m_process_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -660,8 +686,7 @@ perfetto_processor_t::handle(const region_sample& _rs)
|
||||
}
|
||||
} catch(const std::exception& e)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(2, "Failed to parse call_stack JSON: %s\n",
|
||||
e.what());
|
||||
LOG_ERROR("Failed to parse call_stack JSON: {}", e.what());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -916,9 +941,8 @@ perfetto_processor_t::handle([[maybe_unused]] const pmc_event_with_sample& _pmc)
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(2,
|
||||
"Unknown PMC event category_enum_id: %zu for track '%s'\n",
|
||||
_pmc.category_enum_id, _track_name.c_str());
|
||||
LOG_WARNING("Unknown PMC event category_enum_id: {} for track '{}'",
|
||||
_pmc.category_enum_id, _track_name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1202,9 +1226,8 @@ perfetto_processor_t::handle([[maybe_unused]] const in_time_sample& _sample)
|
||||
// Dispatch based on category_enum_id using the category type mapping
|
||||
if(!dispatch_in_time_sample(_sample.category_enum_id, _sample, m_use_annotations))
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
2, "Unknown in_time_sample category_enum_id: %zu, using user category\n",
|
||||
_sample.category_enum_id);
|
||||
LOG_DEBUG("Unknown in_time_sample category_enum_id: {}, using user category",
|
||||
_sample.category_enum_id);
|
||||
write_in_time_sample_data(category::user{}, _sample, m_use_annotations);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
#include "core/agent_manager.hpp"
|
||||
#include "core/common_types.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/gpu_metrics.hpp"
|
||||
#include "core/node_info.hpp"
|
||||
@@ -33,6 +32,7 @@
|
||||
#include "core/trace_cache/metadata_registry.hpp"
|
||||
#include "core/trace_cache/sample_type.hpp"
|
||||
#include "library/thread_info.hpp"
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
@@ -607,13 +607,17 @@ rocpd_processor_t::rocpd_processor_t(const std::shared_ptr<metadata_registry>& m
|
||||
void
|
||||
rocpd_processor_t::prepare_for_processing()
|
||||
{
|
||||
LOG_DEBUG("Preparing rocpd processor for processing");
|
||||
post_process_metadata();
|
||||
LOG_TRACE("Rocpd processor prepared for processing");
|
||||
}
|
||||
|
||||
void
|
||||
rocpd_processor_t::finalize_processing()
|
||||
{
|
||||
LOG_DEBUG("Finalizing rocpd processor");
|
||||
m_data_processor->flush();
|
||||
LOG_INFO("Rocpd processor finalized successfully");
|
||||
}
|
||||
|
||||
void
|
||||
@@ -622,9 +626,10 @@ rocpd_processor_t::post_process_metadata()
|
||||
#if ROCPROFSYS_USE_ROCM > 0
|
||||
if(!get_use_rocpd())
|
||||
{
|
||||
LOG_TRACE("Rocpd not enabled, skipping metadata post-processing");
|
||||
return;
|
||||
}
|
||||
ROCPROFSYS_DEBUG("Post processing metadata..\n");
|
||||
LOG_DEBUG("Post-processing metadata for rocpd");
|
||||
auto n_info = node_info::get_instance();
|
||||
|
||||
m_data_processor->insert_node_info(
|
||||
|
||||
@@ -27,13 +27,14 @@
|
||||
#include "core/trace_cache/cacheable.hpp"
|
||||
#include "core/trace_cache/type_registry.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace rocprofsys
|
||||
@@ -67,12 +68,13 @@ public:
|
||||
throw std::runtime_error("TypeProcessing is nullptr");
|
||||
}
|
||||
|
||||
LOG_DEBUG("Loading storage from file: {}", m_filename);
|
||||
|
||||
std::ifstream ifs(m_filename, std::ios::binary);
|
||||
if(!ifs.good())
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << "Error opening file for reading: " << m_filename << "\n";
|
||||
throw std::runtime_error(ss.str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error opening file for reading: {}", m_filename));
|
||||
}
|
||||
|
||||
struct __attribute__((packed)) sample_header
|
||||
@@ -91,9 +93,8 @@ public:
|
||||
{
|
||||
if(!ifs.good())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
std::string("Stream not in good state, stopping parse. File: ") +
|
||||
m_filename + "\n");
|
||||
throw std::runtime_error(fmt::format(
|
||||
"Stream not in good state, stopping parse. File: {}", m_filename));
|
||||
}
|
||||
|
||||
ifs.read(reinterpret_cast<char*>(&header), sizeof(header));
|
||||
@@ -115,13 +116,14 @@ public:
|
||||
if(ifs.fail())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
std::string("Bad read while consuming buffered storage. Filename: ") +
|
||||
m_filename + " Bytes read: " +
|
||||
std::to_string(static_cast<int>(ifs.tellg())) + "\n");
|
||||
fmt::format("Bad read while consuming buffered storage. Filename: {} "
|
||||
"Bytes read: {}",
|
||||
m_filename, static_cast<int>(ifs.tellg())));
|
||||
}
|
||||
|
||||
if(header.type == TypeIdentifierEnum::fragmented_space)
|
||||
{
|
||||
LOG_TRACE("Skipping fragmented space in storage");
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -139,11 +141,13 @@ public:
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG_TRACE("Unknown sample type encountered, skipping");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
ifs.close();
|
||||
LOG_DEBUG("Storage parsing complete from {}", m_filename);
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@@ -34,7 +34,10 @@ add_library(trace-cache-tests OBJECT ${trace_cache_tests_sources})
|
||||
|
||||
target_link_libraries(
|
||||
trace-cache-tests
|
||||
PRIVATE rocprofiler-systems-googletest-library rocprofiler-systems-core-library
|
||||
PRIVATE
|
||||
rocprofiler-systems-googletest-library
|
||||
rocprofiler-systems-core-library
|
||||
rocprofiler-systems-logger
|
||||
)
|
||||
|
||||
target_include_directories(
|
||||
|
||||
@@ -21,7 +21,8 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "utility.hpp"
|
||||
#include "debug.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
@@ -71,12 +72,11 @@ parse_numeric_range(std::string _input_string, const std::string& _label, Up _in
|
||||
{
|
||||
if(_v.find_first_not_of("0123456789-:") != std::string::npos)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
0,
|
||||
"Invalid %s specification. Only numerical values (e.g., 0), ranges "
|
||||
LOG_WARNING(
|
||||
"Invalid {} specification. Only numerical values (e.g., 0), ranges "
|
||||
"(e.g., 0-7), and ranges with increments (e.g. 20-40:10) are permitted. "
|
||||
"Ignoring %s...",
|
||||
_label.c_str(), _v.c_str());
|
||||
"Ignoring {}...",
|
||||
_label, _v);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -92,10 +92,13 @@ parse_numeric_range(std::string _input_string, const std::string& _label, Up _in
|
||||
if(_v.find('-') != std::string::npos)
|
||||
{
|
||||
auto _vv = tim::delimit(_v, "-");
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_vv.size() != 2,
|
||||
"Invalid %s range specification: %s. Required format N-M, e.g. 0-4",
|
||||
_label.c_str(), _v.c_str());
|
||||
if(_vv.size() != 2)
|
||||
{
|
||||
throw std::runtime_error(fmt::format(
|
||||
"Invalid {} range specification: {}. Required format N-M, e.g. 0-4",
|
||||
_label, _v));
|
||||
}
|
||||
|
||||
Tp _vn = _get_value(_vv.at(0));
|
||||
Tp _vN = _get_value(_vv.at(1));
|
||||
do
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# ------------------------------------------------------------------------------#
|
||||
#
|
||||
# rocprofiler-systems logger library
|
||||
#
|
||||
# ------------------------------------------------------------------------------#
|
||||
|
||||
add_library(rocprofiler-systems-logger INTERFACE)
|
||||
add_library(rocprofsys-logger ALIAS rocprofiler-systems-logger)
|
||||
add_library(
|
||||
rocprofiler-systems::rocprofiler-systems-logger
|
||||
ALIAS rocprofiler-systems-logger
|
||||
)
|
||||
|
||||
target_sources(
|
||||
rocprofiler-systems-logger
|
||||
INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/logger.hpp ${CMAKE_CURRENT_SOURCE_DIR}/debug.hpp
|
||||
)
|
||||
|
||||
get_filename_component(LOGGER_SOURCE_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" DIRECTORY)
|
||||
|
||||
target_include_directories(
|
||||
rocprofiler-systems-logger
|
||||
INTERFACE $<BUILD_INTERFACE:${LOGGER_SOURCE_INCLUDE_DIR}>
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
rocprofiler-systems-logger
|
||||
INTERFACE $<BUILD_INTERFACE:rocprofiler-systems::rocprofiler-systems-spdlog>
|
||||
)
|
||||
|
||||
if(ROCPROFSYS_BUILD_TESTING)
|
||||
add_subdirectory(tests)
|
||||
endif()
|
||||
@@ -0,0 +1,40 @@
|
||||
// Copyright (c) Advanced Micro Devices, Inc.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "logger.hpp"
|
||||
|
||||
#define LOG_CRITICAL(...) \
|
||||
do \
|
||||
{ \
|
||||
rocprofsys::logger_t::instance().log( \
|
||||
spdlog::source_loc{ __FILE__, __LINE__, __func__ }, spdlog::level::critical, \
|
||||
__VA_ARGS__); \
|
||||
rocprofsys::logger_t::instance().flush(); \
|
||||
} while(0)
|
||||
|
||||
#define LOG_ERROR(...) \
|
||||
rocprofsys::logger_t::instance().log( \
|
||||
spdlog::source_loc{ __FILE__, __LINE__, __func__ }, spdlog::level::err, \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define LOG_WARNING(...) \
|
||||
rocprofsys::logger_t::instance().log( \
|
||||
spdlog::source_loc{ __FILE__, __LINE__, __func__ }, spdlog::level::warn, \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define LOG_INFO(...) \
|
||||
rocprofsys::logger_t::instance().log( \
|
||||
spdlog::source_loc{ __FILE__, __LINE__, __func__ }, spdlog::level::info, \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define LOG_DEBUG(...) \
|
||||
rocprofsys::logger_t::instance().log( \
|
||||
spdlog::source_loc{ __FILE__, __LINE__, __func__ }, spdlog::level::debug, \
|
||||
__VA_ARGS__)
|
||||
|
||||
#define LOG_TRACE(...) \
|
||||
rocprofsys::logger_t::instance().log( \
|
||||
spdlog::source_loc{ __FILE__, __LINE__, __func__ }, spdlog::level::trace, \
|
||||
__VA_ARGS__)
|
||||
@@ -0,0 +1,215 @@
|
||||
// Copyright (c) Advanced Micro Devices, Inc.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <spdlog/sinks/basic_file_sink.h>
|
||||
#include <spdlog/sinks/stdout_color_sinks.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <pthread.h>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <sys/cdefs.h>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
inline __attribute__((always_inline)) auto
|
||||
to_lower(std::string_view s)
|
||||
{
|
||||
std::string result;
|
||||
result.reserve(s.size());
|
||||
for(char c : s)
|
||||
{
|
||||
result += static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string
|
||||
include_process_id_in_filename(std::string_view filename)
|
||||
{
|
||||
if(filename.empty())
|
||||
{
|
||||
return std::string{};
|
||||
}
|
||||
|
||||
auto last_sep = filename.find_last_of('/');
|
||||
auto filename_start = (last_sep == std::string_view::npos) ? 0 : last_sep + 1;
|
||||
auto dot_pos = filename.find_last_of('.');
|
||||
|
||||
bool has_extension =
|
||||
(dot_pos != std::string_view::npos) && (dot_pos > filename_start);
|
||||
|
||||
std::string pid_suffix = "_" + std::to_string(getpid());
|
||||
|
||||
if(!has_extension)
|
||||
{
|
||||
return std::string(filename) + pid_suffix;
|
||||
}
|
||||
|
||||
return std::string(filename.substr(0, dot_pos)) + pid_suffix +
|
||||
std::string(filename.substr(dot_pos));
|
||||
}
|
||||
|
||||
inline bool
|
||||
parse_boolean_env(const char* env)
|
||||
{
|
||||
if(!env)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
constexpr std::array<const char*, 4> true_values = { "1", "on", "true", "yes" };
|
||||
|
||||
auto lower = to_lower(env);
|
||||
return std::any_of(true_values.begin(), true_values.end(),
|
||||
[&](const std::string& value) { return value == lower; });
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct logger_settings_t
|
||||
{
|
||||
logger_settings_t()
|
||||
: m_log_level(log_level_from_env(std::getenv("ROCPROFSYS_LOG_LEVEL")))
|
||||
, m_log_file(std::getenv("ROCPROFSYS_LOG_FILE"))
|
||||
{
|
||||
const char* rocprofsys_monochrome_env = std::getenv("ROCPROFSYS_MONOCHROME");
|
||||
const char* monochrome_env = std::getenv("MONOCHROME");
|
||||
if(rocprofsys_monochrome_env || monochrome_env)
|
||||
{
|
||||
m_monochrome = parse_boolean_env(rocprofsys_monochrome_env) ||
|
||||
parse_boolean_env(monochrome_env);
|
||||
}
|
||||
}
|
||||
|
||||
spdlog::level::level_enum log_level_from_env(const char* env)
|
||||
{
|
||||
if(!env)
|
||||
{
|
||||
return m_default_level;
|
||||
}
|
||||
|
||||
return parse_level(env);
|
||||
}
|
||||
|
||||
spdlog::level::level_enum parse_level(std::string_view level)
|
||||
{
|
||||
const auto lower = to_lower(level);
|
||||
|
||||
if(lower == "trace") return spdlog::level::trace;
|
||||
if(lower == "debug") return spdlog::level::debug;
|
||||
if(lower == "info") return spdlog::level::info;
|
||||
if(lower == "warn" || lower == "warning") return spdlog::level::warn;
|
||||
if(lower == "error" || lower == "err") return spdlog::level::err;
|
||||
if(lower == "critical") return spdlog::level::critical;
|
||||
if(lower == "off") return spdlog::level::off;
|
||||
|
||||
return m_default_level;
|
||||
}
|
||||
|
||||
spdlog::level::level_enum get_log_level() const { return m_log_level; }
|
||||
|
||||
std::string get_log_file() const
|
||||
{
|
||||
if(m_log_file == nullptr)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
return { m_log_file };
|
||||
}
|
||||
|
||||
const char* get_log_pattern() const
|
||||
{
|
||||
return m_monochrome ? m_log_pattern_monochrome : m_log_pattern;
|
||||
}
|
||||
|
||||
protected:
|
||||
const spdlog::level::level_enum m_default_level{ spdlog::level::info };
|
||||
const spdlog::level::level_enum m_log_level;
|
||||
const char* m_log_file;
|
||||
|
||||
bool m_monochrome{ false };
|
||||
|
||||
// Pattern:
|
||||
// [TIME][P:PID T:THREAD_ID][LOG_LEVEL][FILE:LINE FUNCTION] MESSAGE
|
||||
const char* m_log_pattern{ "%^[%H:%M:%S.%e][P:%P T:%t][%s:%# %!][%l] %v %$" };
|
||||
const char* m_log_pattern_monochrome{ "[%H:%M:%S.%e][P:%P T:%t][%s:%# %!][%l] %v" };
|
||||
};
|
||||
|
||||
class logger_t
|
||||
{
|
||||
public:
|
||||
static spdlog::logger& instance()
|
||||
{
|
||||
static std::shared_ptr<spdlog::logger> _instance;
|
||||
static std::atomic<bool> _initialized{ false };
|
||||
static std::mutex _init_mutex;
|
||||
|
||||
static std::once_flag _atfork_flag;
|
||||
std::call_once(_atfork_flag, [] {
|
||||
pthread_atfork(nullptr, nullptr, [] {
|
||||
spdlog::drop(s_logger_name);
|
||||
_instance.reset();
|
||||
_initialized.store(false, std::memory_order_release);
|
||||
});
|
||||
});
|
||||
|
||||
if(!_initialized.load(std::memory_order_acquire))
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_init_mutex);
|
||||
|
||||
if(!_initialized.load(std::memory_order_relaxed))
|
||||
{
|
||||
_instance = create_logger();
|
||||
_initialized.store(true, std::memory_order_release);
|
||||
}
|
||||
}
|
||||
|
||||
return *_instance;
|
||||
}
|
||||
|
||||
logger_t() = delete;
|
||||
|
||||
private:
|
||||
static std::shared_ptr<spdlog::logger> create_logger()
|
||||
{
|
||||
logger_settings_t logger_settings;
|
||||
|
||||
std::vector<spdlog::sink_ptr> sinks;
|
||||
|
||||
sinks.push_back(std::make_shared<spdlog::sinks::stdout_color_sink_mt>());
|
||||
|
||||
auto log_file = logger_settings.get_log_file();
|
||||
if(!log_file.empty())
|
||||
{
|
||||
log_file = include_process_id_in_filename(log_file);
|
||||
|
||||
sinks.push_back(
|
||||
std::make_shared<spdlog::sinks::basic_file_sink_mt>(log_file, true));
|
||||
}
|
||||
|
||||
auto log =
|
||||
std::make_shared<spdlog::logger>(s_logger_name, sinks.begin(), sinks.end());
|
||||
|
||||
log->set_pattern(logger_settings.get_log_pattern());
|
||||
log->set_level(logger_settings.get_log_level());
|
||||
|
||||
spdlog::register_logger(log);
|
||||
return log;
|
||||
}
|
||||
|
||||
static constexpr const char* s_logger_name = "rocprofiler-systems";
|
||||
};
|
||||
|
||||
} // namespace rocprofsys
|
||||
@@ -0,0 +1,33 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
set(logger_tests_sources test_logger.cpp)
|
||||
|
||||
add_library(logger-tests OBJECT ${logger_tests_sources})
|
||||
|
||||
target_link_libraries(
|
||||
logger-tests
|
||||
PRIVATE
|
||||
rocprofiler-systems-googletest-library
|
||||
rocprofiler-systems-logger
|
||||
rocprofiler-systems-spdlog
|
||||
)
|
||||
@@ -0,0 +1,324 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2025 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#include "logger/logger.hpp"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
class logger_test : public ::testing::Test
|
||||
{
|
||||
protected:
|
||||
};
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_with_extension)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result = rocprofsys::include_process_id_in_filename("logfile.log");
|
||||
auto expected = "logfile_" + pid + ".log";
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_without_extension)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result = rocprofsys::include_process_id_in_filename("logfile");
|
||||
auto expected = "logfile_" + pid;
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_with_path)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result = rocprofsys::include_process_id_in_filename("/var/log/myapp.log");
|
||||
auto expected = "/var/log/myapp_" + pid + ".log";
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_with_path_no_extension)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result = rocprofsys::include_process_id_in_filename("/var/log/myapp");
|
||||
auto expected = "/var/log/myapp_" + pid;
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_empty)
|
||||
{
|
||||
auto result = rocprofsys::include_process_id_in_filename("");
|
||||
EXPECT_TRUE(result.empty());
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_multiple_dots)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result = rocprofsys::include_process_id_in_filename("file.name.with.dots.txt");
|
||||
auto expected = "file.name.with.dots_" + pid + ".txt";
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_dot_in_directory)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result =
|
||||
rocprofsys::include_process_id_in_filename("/path.with.dots/logfile.log");
|
||||
auto expected = "/path.with.dots/logfile_" + pid + ".log";
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_hidden_file)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result = rocprofsys::include_process_id_in_filename(".hidden");
|
||||
auto expected = ".hidden_" + pid;
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, include_process_id_in_filename_hidden_file_with_extension)
|
||||
{
|
||||
auto pid = std::to_string(getpid());
|
||||
auto result = rocprofsys::include_process_id_in_filename(".hidden.log");
|
||||
auto expected = ".hidden_" + pid + ".log";
|
||||
|
||||
EXPECT_EQ(result, expected);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, parse_boolean_env_true_values)
|
||||
{
|
||||
EXPECT_TRUE(rocprofsys::parse_boolean_env("1"));
|
||||
EXPECT_TRUE(rocprofsys::parse_boolean_env("on"));
|
||||
EXPECT_TRUE(rocprofsys::parse_boolean_env("true"));
|
||||
EXPECT_TRUE(rocprofsys::parse_boolean_env("yes"));
|
||||
}
|
||||
|
||||
TEST_F(logger_test, parse_boolean_env_false_values)
|
||||
{
|
||||
EXPECT_FALSE(rocprofsys::parse_boolean_env("0"));
|
||||
EXPECT_FALSE(rocprofsys::parse_boolean_env("off"));
|
||||
EXPECT_FALSE(rocprofsys::parse_boolean_env("false"));
|
||||
EXPECT_FALSE(rocprofsys::parse_boolean_env("no"));
|
||||
EXPECT_FALSE(rocprofsys::parse_boolean_env("random"));
|
||||
EXPECT_FALSE(rocprofsys::parse_boolean_env(nullptr));
|
||||
}
|
||||
|
||||
TEST_F(logger_test, to_lower_conversion)
|
||||
{
|
||||
EXPECT_EQ(rocprofsys::to_lower("HELLO"), "hello");
|
||||
EXPECT_EQ(rocprofsys::to_lower("Hello World"), "hello world");
|
||||
EXPECT_EQ(rocprofsys::to_lower("already_lower"), "already_lower");
|
||||
EXPECT_EQ(rocprofsys::to_lower("MiXeD123"), "mixed123");
|
||||
EXPECT_EQ(rocprofsys::to_lower(""), "");
|
||||
}
|
||||
|
||||
TEST_F(logger_test, logger_settings_parse_level)
|
||||
{
|
||||
rocprofsys::logger_settings_t settings;
|
||||
|
||||
EXPECT_EQ(settings.parse_level("trace"), spdlog::level::trace);
|
||||
EXPECT_EQ(settings.parse_level("TRACE"), spdlog::level::trace);
|
||||
EXPECT_EQ(settings.parse_level("debug"), spdlog::level::debug);
|
||||
EXPECT_EQ(settings.parse_level("DEBUG"), spdlog::level::debug);
|
||||
EXPECT_EQ(settings.parse_level("info"), spdlog::level::info);
|
||||
EXPECT_EQ(settings.parse_level("INFO"), spdlog::level::info);
|
||||
EXPECT_EQ(settings.parse_level("warn"), spdlog::level::warn);
|
||||
EXPECT_EQ(settings.parse_level("warning"), spdlog::level::warn);
|
||||
EXPECT_EQ(settings.parse_level("WARNING"), spdlog::level::warn);
|
||||
EXPECT_EQ(settings.parse_level("error"), spdlog::level::err);
|
||||
EXPECT_EQ(settings.parse_level("err"), spdlog::level::err);
|
||||
EXPECT_EQ(settings.parse_level("ERROR"), spdlog::level::err);
|
||||
EXPECT_EQ(settings.parse_level("critical"), spdlog::level::critical);
|
||||
EXPECT_EQ(settings.parse_level("CRITICAL"), spdlog::level::critical);
|
||||
EXPECT_EQ(settings.parse_level("off"), spdlog::level::off);
|
||||
EXPECT_EQ(settings.parse_level("OFF"), spdlog::level::off);
|
||||
EXPECT_EQ(settings.parse_level("invalid"), spdlog::level::info);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, logger_instance_returns_valid_logger)
|
||||
{
|
||||
auto& logger = rocprofsys::logger_t::instance();
|
||||
EXPECT_NE(logger.name(), "");
|
||||
EXPECT_EQ(logger.name(), "rocprofiler-systems");
|
||||
}
|
||||
|
||||
TEST_F(logger_test, logger_instance_is_singleton)
|
||||
{
|
||||
auto& logger1 = rocprofsys::logger_t::instance();
|
||||
auto& logger2 = rocprofsys::logger_t::instance();
|
||||
|
||||
EXPECT_EQ(&logger1, &logger2);
|
||||
}
|
||||
|
||||
TEST_F(logger_test, fork_child_gets_different_pid_in_filename)
|
||||
{
|
||||
pid_t parent_pid = getpid();
|
||||
|
||||
pid_t child_pid = fork();
|
||||
if(child_pid == 0)
|
||||
{
|
||||
pid_t current_pid = getpid();
|
||||
|
||||
auto child_filename = rocprofsys::include_process_id_in_filename("test.log");
|
||||
auto expected_filename = "test_" + std::to_string(current_pid) + ".log";
|
||||
|
||||
bool pid_differs = (current_pid != parent_pid);
|
||||
bool filename_correct = (child_filename == expected_filename);
|
||||
|
||||
exit((pid_differs && filename_correct) ? 0 : 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
int status;
|
||||
waitpid(child_pid, &status, 0);
|
||||
int child_exit_code = WEXITSTATUS(status);
|
||||
|
||||
EXPECT_EQ(child_exit_code, 0) << "Child should have different PID in filename";
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(logger_test, fork_resets_logger_in_child)
|
||||
{
|
||||
auto& parent_logger = rocprofsys::logger_t::instance();
|
||||
auto* parent_logger_ptr = &parent_logger;
|
||||
|
||||
pid_t child_pid = fork();
|
||||
if(child_pid == 0)
|
||||
{
|
||||
auto& child_logger = rocprofsys::logger_t::instance();
|
||||
auto* child_logger_ptr = &child_logger;
|
||||
|
||||
bool logger_is_valid = (child_logger_ptr != nullptr);
|
||||
bool logger_has_name = (child_logger.name() == "rocprofiler-systems");
|
||||
|
||||
exit((logger_is_valid && logger_has_name) ? 0 : 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
int status;
|
||||
waitpid(child_pid, &status, 0);
|
||||
int child_exit_code = WEXITSTATUS(status);
|
||||
|
||||
EXPECT_EQ(child_exit_code, 0) << "Child logger should be valid after fork reset";
|
||||
|
||||
auto& post_fork_parent_logger = rocprofsys::logger_t::instance();
|
||||
EXPECT_EQ(&post_fork_parent_logger, parent_logger_ptr)
|
||||
<< "Parent logger should remain the same after fork";
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(logger_test, logger_settings_default_values)
|
||||
{
|
||||
unsetenv("ROCPROFSYS_LOG_LEVEL");
|
||||
unsetenv("ROCPROFSYS_LOG_FILE");
|
||||
|
||||
rocprofsys::logger_settings_t settings;
|
||||
|
||||
EXPECT_EQ(settings.get_log_level(), spdlog::level::info);
|
||||
EXPECT_TRUE(settings.get_log_file().empty());
|
||||
}
|
||||
|
||||
TEST_F(logger_test, logger_settings_with_env_vars)
|
||||
{
|
||||
setenv("ROCPROFSYS_LOG_LEVEL", "debug", 1);
|
||||
setenv("ROCPROFSYS_LOG_FILE", "/tmp/test.log", 1);
|
||||
|
||||
rocprofsys::logger_settings_t settings;
|
||||
|
||||
EXPECT_EQ(settings.get_log_level(), spdlog::level::debug);
|
||||
EXPECT_EQ(settings.get_log_file(), "/tmp/test.log");
|
||||
|
||||
unsetenv("ROCPROFSYS_LOG_LEVEL");
|
||||
unsetenv("ROCPROFSYS_LOG_FILE");
|
||||
}
|
||||
|
||||
TEST_F(logger_test, logger_settings_monochrome)
|
||||
{
|
||||
setenv("ROCPROFSYS_MONOCHROME", "1", 1);
|
||||
|
||||
rocprofsys::logger_settings_t settings;
|
||||
const auto* pattern = settings.get_log_pattern();
|
||||
|
||||
EXPECT_TRUE(std::string(pattern).find("%^") == std::string::npos);
|
||||
EXPECT_TRUE(std::string(pattern).find("%$") == std::string::npos);
|
||||
|
||||
unsetenv("ROCPROFSYS_MONOCHROME");
|
||||
}
|
||||
|
||||
TEST_F(logger_test, fork_child_creates_log_file_with_child_pid)
|
||||
{
|
||||
std::string test_log_base = "/tmp/test_fork_logger";
|
||||
std::string test_log_ext = ".log";
|
||||
std::string test_log_file = test_log_base + test_log_ext;
|
||||
|
||||
pid_t child_pid = fork();
|
||||
if(child_pid == 0)
|
||||
{
|
||||
setenv("ROCPROFSYS_LOG_FILE", test_log_file.c_str(), 1);
|
||||
|
||||
pid_t current_pid = getpid();
|
||||
|
||||
auto& child_logger = rocprofsys::logger_t::instance();
|
||||
|
||||
std::string expected_child_log_file =
|
||||
test_log_base + "_" + std::to_string(current_pid) + test_log_ext;
|
||||
|
||||
child_logger.info("Child process log entry");
|
||||
child_logger.flush();
|
||||
|
||||
std::ifstream child_log(expected_child_log_file);
|
||||
bool child_log_exists = child_log.good();
|
||||
|
||||
std::string content;
|
||||
bool has_content = false;
|
||||
if(child_log_exists)
|
||||
{
|
||||
std::getline(child_log, content);
|
||||
has_content = content.find("Child process log entry") != std::string::npos;
|
||||
}
|
||||
|
||||
std::remove(expected_child_log_file.c_str());
|
||||
|
||||
exit(child_log_exists && has_content ? 0 : 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
int status;
|
||||
waitpid(child_pid, &status, 0);
|
||||
int child_exit_code = WEXITSTATUS(status);
|
||||
|
||||
EXPECT_EQ(child_exit_code, 0)
|
||||
<< "Child process failed to create its own log file with child PID";
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,8 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "api.hpp"
|
||||
#include "core/debug.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
#include <exception>
|
||||
#include <stdexcept>
|
||||
|
||||
@@ -46,7 +46,7 @@ rocprofsys_push_region(const char* _name)
|
||||
rocprofsys_push_region_hidden(_name);
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(1, "Exception caught: %s\n", _e.what());
|
||||
LOG_WARNING("Exception caught: {}", _e.what());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
@@ -60,7 +60,7 @@ rocprofsys_pop_region(const char* _name)
|
||||
rocprofsys_pop_region_hidden(_name);
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(1, "Exception caught: %s\n", _e.what());
|
||||
LOG_WARNING("Exception caught: {}", _e.what());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
@@ -77,7 +77,7 @@ rocprofsys_push_category_region(rocprofsys_category_t _category, const char* _na
|
||||
_annotation_count);
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(1, "Exception caught: %s\n", _e.what());
|
||||
LOG_WARNING("Exception caught: {}", _e.what());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
@@ -94,7 +94,7 @@ rocprofsys_pop_category_region(rocprofsys_category_t _category, const char* _nam
|
||||
_annotation_count);
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(1, "Exception caught: %s\n", _e.what());
|
||||
LOG_WARNING("Exception caught: {}", _e.what());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
|
||||
@@ -35,7 +35,6 @@
|
||||
#include "core/config.hpp"
|
||||
#include "core/constraint.hpp"
|
||||
#include "core/cpu.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/dynamic_library.hpp"
|
||||
#include "core/gpu.hpp"
|
||||
@@ -89,6 +88,8 @@
|
||||
# include <rocprofiler-sdk/registration.h>
|
||||
#endif
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
@@ -147,12 +148,10 @@ ensure_initialization(bool _offset, int64_t _glob_n, int64_t _offset_n)
|
||||
if(_tid > 0 && _tid < _peak_num_threads)
|
||||
{
|
||||
const auto& _info = thread_info::get();
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(3,
|
||||
"thread info: %s, offset: %s, global counter: %li, "
|
||||
"offset counter: %li, max threads: %li\n",
|
||||
std::to_string(static_cast<bool>(_info)).c_str(),
|
||||
std::to_string(_offset).c_str(), _glob_n, _offset_n,
|
||||
_peak_num_threads);
|
||||
LOG_DEBUG("thread info: {}, offset: {}, global counter: {}, offset counter: {}, "
|
||||
"max threads: {}",
|
||||
static_cast<bool>(_info), _offset, _glob_n, _offset_n,
|
||||
_peak_num_threads);
|
||||
}
|
||||
|
||||
return _offset;
|
||||
@@ -177,22 +176,32 @@ ensure_finalization(bool _static_init = false)
|
||||
throw exception<std::runtime_error>("failure adding threading callback");
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_BASIC_THROW(
|
||||
config::set_signal_handler(nullptr) != &finalization_handler,
|
||||
"Assignment of signal handler failed. signal handler is %s, expected %s\n",
|
||||
as_hex(reinterpret_cast<void*>(config::set_signal_handler(nullptr))).c_str(),
|
||||
as_hex(reinterpret_cast<void*>(&finalization_handler)).c_str());
|
||||
if(config::set_signal_handler(nullptr) != &finalization_handler)
|
||||
|
||||
throw std::runtime_error(fmt::format(
|
||||
"Assignment of signal handler failed. signal handler is {:P}, expected "
|
||||
"{:P}",
|
||||
fmt::format("0x{:X}",
|
||||
reinterpret_cast<void*>(config::set_signal_handler(nullptr))),
|
||||
fmt::format("0x{:X}", reinterpret_cast<void*>(&finalization_handler))));
|
||||
|
||||
const auto& _info = thread_info::init();
|
||||
const auto& _tid = _info->index_data;
|
||||
if(_tid)
|
||||
{
|
||||
ROCPROFSYS_CI_THROW(_tid->sequent_value != threading::get_id(),
|
||||
"Error! internal tid != %li :: %li", threading::get_id(),
|
||||
_tid->sequent_value);
|
||||
ROCPROFSYS_CI_THROW(_tid->system_value != threading::get_sys_tid(),
|
||||
"Error! system tid != %li :: %li", threading::get_sys_tid(),
|
||||
_tid->system_value);
|
||||
if(get_is_continuous_integration() && _tid->sequent_value != threading::get_id())
|
||||
{
|
||||
throw std::runtime_error(fmt::format("Error! internal tid != {} :: {}",
|
||||
threading::get_id(),
|
||||
_tid->sequent_value));
|
||||
}
|
||||
if(get_is_continuous_integration() &&
|
||||
_tid->system_value != threading::get_sys_tid())
|
||||
{
|
||||
throw std::runtime_error(fmt::format("Error! system tid != {} :: {}",
|
||||
threading::get_sys_tid(),
|
||||
_tid->system_value));
|
||||
}
|
||||
}
|
||||
|
||||
if(common::get_env("ROCPROFSYS_MONOCHROME", false)) tim::log::monochrome() = true;
|
||||
@@ -212,7 +221,6 @@ ensure_finalization(bool _static_init = false)
|
||||
|
||||
if(_static_init)
|
||||
{
|
||||
ROCPROFSYS_BASIC_DEBUG_F("\n");
|
||||
auto _verbose =
|
||||
get_verbose_env() + ((get_debug_env() || get_debug_init()) ? 16 : 0);
|
||||
auto _search_paths = JOIN(':', tim::get_env<std::string>("ROCPROFSYS_PATH", ""),
|
||||
@@ -222,10 +230,6 @@ ensure_finalization(bool _static_init = false)
|
||||
tim::get_env<std::string>("PATH", ""));
|
||||
common::setup_environ(_verbose, _search_paths);
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_DEBUG_F("\n");
|
||||
}
|
||||
|
||||
if(_timemory_manager) _timemory_manager->set_write_metadata(-1);
|
||||
|
||||
@@ -246,14 +250,14 @@ struct fini_bundle
|
||||
template <typename... Args>
|
||||
void start(Args&&... _args)
|
||||
{
|
||||
TIMEMORY_FOLD_EXPRESSION(tim::operation::start<Tp>{}(
|
||||
ROCPROFSYS_FOLD_EXPRESSION(tim::operation::start<Tp>{}(
|
||||
std::get<Tp>(m_data), std::forward<Args>(_args)...));
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
void stop(Args&&... _args)
|
||||
{
|
||||
TIMEMORY_FOLD_EXPRESSION(tim::operation::stop<Tp>{}(
|
||||
ROCPROFSYS_FOLD_EXPRESSION(tim::operation::stop<Tp>{}(
|
||||
std::get<Tp>(m_data), std::forward<Args>(_args)...));
|
||||
}
|
||||
|
||||
@@ -299,19 +303,19 @@ rocprofsys_set_env_hidden(const char* env_name, const char* env_val)
|
||||
bool _success = _set_envs.emplace(env_name).second;
|
||||
|
||||
// just search env to avoid initializing the settings
|
||||
ROCPROFSYS_CONDITIONAL_PRINT_F(get_debug_init() || get_verbose_env() > 2,
|
||||
"Setting env: %s=%s\n", env_name, env_val);
|
||||
if(get_debug_init())
|
||||
{
|
||||
LOG_DEBUG("Setting env: {} = {}", env_name, env_val);
|
||||
}
|
||||
|
||||
tim::set_env(env_name, env_val, 0);
|
||||
|
||||
if(_success && get_state() >= State::Init)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
0,
|
||||
"rocprofsys_set_env(\"%s\", \"%s\") called after rocprof-sys was "
|
||||
"initialized. "
|
||||
"state = %s. This environment variable will have no effect\n",
|
||||
env_name, env_val, std::to_string(get_state()).c_str());
|
||||
LOG_WARNING(
|
||||
"rocprofsys_set_env(\"{}\", \"{}\") called after rocprof-sys was "
|
||||
"initialized. state = {}. This environment variable will have no effect",
|
||||
env_name, env_val, static_cast<int>(get_state()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -387,9 +391,10 @@ rocprofsys_set_mpi_hidden(bool use, bool attached)
|
||||
_once = true;
|
||||
|
||||
// just search env to avoid initializing the settings
|
||||
ROCPROFSYS_CONDITIONAL_PRINT_F(get_debug_init() || get_verbose_env() > 2,
|
||||
"use: %s, attached: %s\n", (use) ? "y" : "n",
|
||||
(attached) ? "y" : "n");
|
||||
if(get_debug_init())
|
||||
{
|
||||
LOG_DEBUG("use: {}, attached: {}", (use) ? "y" : "n", (attached) ? "y" : "n");
|
||||
}
|
||||
|
||||
_set_mpi_called = true;
|
||||
config::is_attached() = attached;
|
||||
@@ -405,13 +410,11 @@ rocprofsys_set_mpi_hidden(bool use, bool attached)
|
||||
|
||||
if(get_state() >= State::Init)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
0,
|
||||
"rocprofsys_set_mpi(use=%s, attached=%s) called after rocprof-sys was "
|
||||
"initialized. state = %s. MPI support may not be properly initialized. Use "
|
||||
"ROCPROFSYS_USE_MPIP=ON and ROCPROFSYS_USE_PID=ON to ensure full support\n",
|
||||
std::to_string(use).c_str(), std::to_string(attached).c_str(),
|
||||
std::to_string(get_state()).c_str());
|
||||
LOG_WARNING(
|
||||
"rocprofsys_set_mpi(use={}, attached={}) called after rocprof-sys was "
|
||||
"initialized. state = {}. MPI support may not be properly initialized. Use "
|
||||
"ROCPROFSYS_USE_MPIP=ON and ROCPROFSYS_USE_PID=ON to ensure full support",
|
||||
use, attached, static_cast<int>(get_state()));
|
||||
}
|
||||
|
||||
rocprofsys_preinit_hidden();
|
||||
@@ -437,31 +440,36 @@ rocprofsys_init_library_hidden()
|
||||
|
||||
if(_selinux_mode == 1)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "/sys/fs/selinux/enforce has a value of %i. \n",
|
||||
_selinux_mode);
|
||||
std::cerr << "SELinux enforcing mode detected. Consider disabling SELinux "
|
||||
<< "or configure permissive mode with 'sudo setenforce 0'. Aborting.\n";
|
||||
LOG_DEBUG("/sys/fs/selinux/enforce has a value of {}.", _selinux_mode);
|
||||
LOG_CRITICAL("SELinux enforcing mode detected. Consider disabling SELinux "
|
||||
"or configure permissive mode with 'sudo setenforce 0'. Aborting.");
|
||||
std::exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "State is %s...\n",
|
||||
std::to_string(get_state()).c_str());
|
||||
if(_debug_init)
|
||||
{
|
||||
LOG_DEBUG("State is {}...", std::to_string(get_state()));
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(get_state() != State::PreInit, "State is not PreInit :: %s",
|
||||
std::to_string(get_state()).c_str());
|
||||
if(get_is_continuous_integration() && get_state() != State::PreInit)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("State is not PreInit :: {}", std::to_string(get_state())));
|
||||
}
|
||||
|
||||
if(get_state() != State::PreInit || get_state() == State::Init || _once) return;
|
||||
_once = true;
|
||||
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "State is %s. Setting to %s...\n",
|
||||
std::to_string(get_state()).c_str(),
|
||||
std::to_string(State::Init).c_str());
|
||||
if(_debug_init)
|
||||
{
|
||||
LOG_DEBUG("State is {}. Setting to {}...", std::to_string(get_state()),
|
||||
std::to_string(State::Init));
|
||||
LOG_DEBUG("Calling backtrace once so that the one-time call of malloc in "
|
||||
"glibc's backtrace() occurs...");
|
||||
}
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(
|
||||
_debug_init, "Calling backtrace once so that the one-time call of malloc in "
|
||||
"glibc's backtrace() occurs...\n");
|
||||
{
|
||||
std::stringstream _ss{};
|
||||
timemory_print_backtrace<16>(_ss);
|
||||
@@ -470,11 +478,16 @@ rocprofsys_init_library_hidden()
|
||||
|
||||
set_state(State::Init);
|
||||
|
||||
ROCPROFSYS_CI_THROW(get_state() != State::Init,
|
||||
"set_state(State::Init) failed. state is %s",
|
||||
std::to_string(get_state()).c_str());
|
||||
if(get_is_continuous_integration() && get_state() != State::Init)
|
||||
{
|
||||
throw std::runtime_error(fmt::format("set_state(State::Init) failed. state is {}",
|
||||
std::to_string(get_state())));
|
||||
}
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "Configuring settings...\n");
|
||||
if(_debug_init)
|
||||
{
|
||||
LOG_DEBUG("Configuring settings...");
|
||||
}
|
||||
|
||||
// configure the settings
|
||||
configure_settings();
|
||||
@@ -484,8 +497,6 @@ rocprofsys_init_library_hidden()
|
||||
scope::destructor _debug_dtor{ [_debug_value, _debug_init]() {
|
||||
if(_debug_init) config::set_setting_value("ROCPROFSYS_DEBUG", _debug_value);
|
||||
} };
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "\n");
|
||||
}
|
||||
|
||||
//======================================================================================//
|
||||
@@ -511,8 +522,10 @@ rocprofsys_init_tooling_hidden(void)
|
||||
static pid_t _once = 0;
|
||||
static auto _debug_init = get_debug_init();
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "State is %s...\n",
|
||||
std::to_string(get_state()).c_str());
|
||||
if(_debug_init)
|
||||
{
|
||||
LOG_DEBUG("State is {}...", std::to_string(get_state()));
|
||||
}
|
||||
|
||||
if(get_state() != State::PreInit || get_state() == State::Init || _once == getpid())
|
||||
{
|
||||
@@ -522,26 +535,29 @@ rocprofsys_init_tooling_hidden(void)
|
||||
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
get_state() == State::Init,
|
||||
"%s called after rocprofsys_init_library() was explicitly called",
|
||||
ROCPROFSYS_FUNCTION);
|
||||
if(get_state() == State::Init)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("{} called after rocprofsys_init_library() was explicitly called",
|
||||
__FUNCTION__));
|
||||
}
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(get_verbose_env() >= 0,
|
||||
"Instrumentation mode: %s\n",
|
||||
std::to_string(config::get_mode()).c_str());
|
||||
LOG_DEBUG("Instrumentation mode: {}", std::to_string(config::get_mode()));
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init, "Printing banner...\n");
|
||||
if(_debug_init)
|
||||
{
|
||||
LOG_DEBUG("Printing banner...");
|
||||
}
|
||||
|
||||
if(get_verbose_env() >= 0) print_banner();
|
||||
print_banner();
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(_debug_init,
|
||||
"Calling rocprofsys_init_library()...\n");
|
||||
if(_debug_init)
|
||||
{
|
||||
LOG_DEBUG("Calling rocprofsys_init_library()...");
|
||||
}
|
||||
|
||||
rocprofsys_init_library_hidden();
|
||||
|
||||
ROCPROFSYS_DEBUG_F("\n");
|
||||
|
||||
auto _dtor = scope::destructor{ []() {
|
||||
// if set to finalized, don't continue
|
||||
if(get_state() > State::Active) return;
|
||||
@@ -575,8 +591,7 @@ rocprofsys_init_tooling_hidden(void)
|
||||
sampling::unblock_signals();
|
||||
}
|
||||
get_main_bundle()->start();
|
||||
ROCPROFSYS_DEBUG_F("State: %s -> State::Active\n",
|
||||
std::to_string(get_state()).c_str());
|
||||
LOG_DEBUG("State: {} -> State::Active", std::to_string(get_state()));
|
||||
|
||||
{
|
||||
ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
|
||||
@@ -596,7 +611,7 @@ rocprofsys_init_tooling_hidden(void)
|
||||
|
||||
if(get_use_vaapi_tracing())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Setting up VA-API traces...\n");
|
||||
LOG_DEBUG("Setting up VA-API traces...");
|
||||
component::vaapi_gotcha::start();
|
||||
}
|
||||
|
||||
@@ -605,7 +620,7 @@ rocprofsys_init_tooling_hidden(void)
|
||||
// perfetto initialization
|
||||
if(get_use_perfetto())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Setting up Perfetto...\n");
|
||||
LOG_DEBUG("Setting up Perfetto...");
|
||||
rocprofsys::perfetto::setup();
|
||||
}
|
||||
|
||||
@@ -644,7 +659,7 @@ rocprofsys_init_tooling_hidden(void)
|
||||
|
||||
if(get_use_perfetto())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Starting Perfetto...\n");
|
||||
LOG_DEBUG("Starting Perfetto...");
|
||||
rocprofsys::perfetto::start();
|
||||
}
|
||||
|
||||
@@ -675,15 +690,17 @@ rocprofsys_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _
|
||||
std::tie(_args.first, _args.second) == std::tie(_mode_sv, _is_binary_rewrite))
|
||||
return;
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_count > 0 &&
|
||||
std::tie(_args.first, _args.second) != std::tie(_mode_sv, _is_binary_rewrite),
|
||||
"\nrocprofsys_init(...) called multiple times with different arguments for mode "
|
||||
"and/or is_binary_rewrite:"
|
||||
"\n Invocation #1: rocprofsys_init(mode=%-8s, is_binary_rewrite=%-5s, ...)"
|
||||
"\n Invocation #%i: rocprofsys_init(mode=%-8s, is_binary_rewrite=%-5s, ...)",
|
||||
_args.first.data(), std::to_string(_args.second).c_str(), _count + 1, _mode,
|
||||
std::to_string(_is_binary_rewrite).c_str());
|
||||
if(_count > 0 &&
|
||||
std::tie(_args.first, _args.second) != std::tie(_mode_sv, _is_binary_rewrite))
|
||||
{
|
||||
throw std::runtime_error(fmt::format(
|
||||
"rocprofsys_init(...) called multiple times with different arguments for "
|
||||
"mode and/or is_binary_rewrite:"
|
||||
"\n Invocation #1: rocprofsys_init(mode={}, is_binary_rewrite={}, ...)"
|
||||
"\n Invocation #%i: rocprofsys_init(mode={}, is_binary_rewrite={}, ...)",
|
||||
_args.first.data(), std::to_string(_args.second).c_str(), _count + 1, _mode,
|
||||
std::to_string(_is_binary_rewrite)));
|
||||
}
|
||||
|
||||
// always the first
|
||||
(void) get_state();
|
||||
@@ -694,28 +711,29 @@ rocprofsys_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _
|
||||
{
|
||||
if(std::string_view{ _mode } != "trace" && std::string_view{ _mode } != "Trace")
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
0,
|
||||
"rocprofsys_init(mode=%s, is_binary_rewrite=%s, argv0=%s) "
|
||||
"called after rocprof-sys was initialized. state = %s. Mode-based "
|
||||
LOG_WARNING(
|
||||
"rocprofsys_init(mode={}, is_binary_rewrite={}, argv0={}) "
|
||||
"called after rocprof-sys was initialized. state = {}. Mode-based "
|
||||
"settings (via -M <MODE> passed to rocprof-sys exe) may not be "
|
||||
"properly configured.\n",
|
||||
_mode, std::to_string(_is_binary_rewrite).c_str(), _argv0.c_str(),
|
||||
std::to_string(get_state()).c_str());
|
||||
"properly configured.",
|
||||
_mode, std::to_string(_is_binary_rewrite), _argv0,
|
||||
std::to_string(get_state()));
|
||||
}
|
||||
}
|
||||
|
||||
tracing::get_finalization_functions().emplace_back([_argv0_c]() {
|
||||
ROCPROFSYS_CI_THROW(get_state() != State::Active,
|
||||
"Finalizer function for popping main invoked in non-active "
|
||||
"state :: state = %s\n",
|
||||
std::to_string(get_state()).c_str());
|
||||
if(get_is_continuous_integration() && get_state() != State::Active)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Finalizer function for popping main invoked in non-active "
|
||||
"state :: state = {}",
|
||||
std::to_string(get_state())));
|
||||
}
|
||||
if(get_state() == State::Active)
|
||||
{
|
||||
auto _name = (_argv0_c) ? std::string{ _argv0_c } : config::get_exe_name();
|
||||
// if main hasn't been popped yet, pop it
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "Running rocprofsys_pop_trace(%s)...\n",
|
||||
_name.c_str());
|
||||
LOG_DEBUG("Running rocprofsys_pop_trace({})...", _name);
|
||||
rocprofsys_pop_trace_hidden(_name.c_str());
|
||||
}
|
||||
});
|
||||
@@ -727,10 +745,11 @@ rocprofsys_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _
|
||||
|
||||
set_metadata_process_start_timestamp(comp::wall_clock::record());
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT_F(
|
||||
get_debug_env() || get_verbose_env() > 2,
|
||||
"mode: %s | is binary rewrite: %s | command: %s\n", _mode,
|
||||
(_is_binary_rewrite) ? "y" : "n", _argv0.c_str());
|
||||
if(get_debug_env() || get_verbose_env() > 2)
|
||||
{
|
||||
LOG_DEBUG("mode: {} | is binary rewrite: {} | command: {}", _mode,
|
||||
(_is_binary_rewrite) ? "y" : "n", _argv0);
|
||||
}
|
||||
|
||||
tim::set_env("ROCPROFSYS_MODE", _mode, 0);
|
||||
config::is_binary_rewrite() = _is_binary_rewrite;
|
||||
@@ -779,8 +798,7 @@ rocprofsys_finalize_hidden(void)
|
||||
// return if not active
|
||||
if(get_state() != State::Active)
|
||||
{
|
||||
ROCPROFSYS_BASIC_DEBUG_F("State = %s. Finalization skipped\n",
|
||||
std::to_string(get_state()).c_str());
|
||||
LOG_DEBUG("State = {}. Finalization skipped", std::to_string(get_state()));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -794,7 +812,7 @@ rocprofsys_finalize_hidden(void)
|
||||
// Flush buffered traces in case of child process
|
||||
if(get_use_rocm())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down ROCm...\n");
|
||||
LOG_DEBUG("Shutting down ROCm...");
|
||||
rocprofiler_sdk::shutdown();
|
||||
}
|
||||
#endif
|
||||
@@ -809,8 +827,7 @@ rocprofsys_finalize_hidden(void)
|
||||
return;
|
||||
}
|
||||
|
||||
if(get_verbose() >= 0 || get_debug()) fprintf(stderr, "\n");
|
||||
ROCPROFSYS_VERBOSE_F(0, "finalizing...\n");
|
||||
LOG_INFO("Finalizing rocprof-sys...");
|
||||
|
||||
sampling::block_samples();
|
||||
|
||||
@@ -859,25 +876,24 @@ rocprofsys_finalize_hidden(void)
|
||||
{
|
||||
if(dmp::rank() == 0)
|
||||
{
|
||||
ROCPROFSYS_PRINT_F("\n");
|
||||
config::print_settings(
|
||||
tim::get_env<bool>("ROCPROFSYS_PRINT_ENV", get_debug()));
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(1, "rocprofsys_push_trace :: called %zux\n", _push_count);
|
||||
ROCPROFSYS_VERBOSE_F(1, "rocprofsys_pop_trace :: called %zux\n", _pop_count);
|
||||
LOG_DEBUG("rocprofsys_push_trace :: called {}", _push_count);
|
||||
LOG_DEBUG("rocprofsys_pop_trace :: called {}", _pop_count);
|
||||
|
||||
tim::signals::enable_signal_detection({ tim::signals::sys_signal::Interrupt },
|
||||
[](int) {});
|
||||
|
||||
ROCPROFSYS_DEBUG_F("Copying over all timemory hash information to main thread...\n");
|
||||
LOG_DEBUG("Copying over all timemory hash information to main thread...");
|
||||
tracing::copy_timemory_hash_ids();
|
||||
|
||||
// stop the main bundle which has stats for run
|
||||
if(get_main_bundle())
|
||||
{
|
||||
ROCPROFSYS_DEBUG_F("Stopping main bundle...\n");
|
||||
LOG_DEBUG("Stopping main bundle...");
|
||||
get_main_bundle()->stop();
|
||||
}
|
||||
|
||||
@@ -886,19 +902,19 @@ rocprofsys_finalize_hidden(void)
|
||||
|
||||
if(get_use_vaapi_tracing())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down VA-API tracing...\n");
|
||||
LOG_DEBUG("Shutting down VA-API tracing...");
|
||||
component::vaapi_gotcha::shutdown();
|
||||
}
|
||||
|
||||
#if defined(ROCPROFSYS_USE_ROCM) && ROCPROFSYS_USE_ROCM > 0
|
||||
if(get_use_rocm())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down ROCm...\n");
|
||||
LOG_DEBUG("Shutting down ROCm...");
|
||||
rocprofiler_sdk::shutdown();
|
||||
}
|
||||
#endif
|
||||
|
||||
ROCPROFSYS_DEBUG_F("Stopping and destroying instrumentation bundles...\n");
|
||||
LOG_DEBUG("Stopping and destroying instrumentation bundles...");
|
||||
auto* _bundles = instrumentation_bundles::get();
|
||||
for(size_t i = 0; _bundles && i < thread_info::get_peak_num_threads(); ++i)
|
||||
{
|
||||
@@ -907,17 +923,13 @@ rocprofsys_finalize_hidden(void)
|
||||
auto& itr = _bundles->at(i);
|
||||
while(itr != nullptr && !itr->empty())
|
||||
{
|
||||
int _lvl = 1;
|
||||
if(_info->is_offset)
|
||||
{
|
||||
++_pop_count;
|
||||
_lvl = 4;
|
||||
}
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
_lvl,
|
||||
"Warning! instrumentation bundle on thread %zu (TID=%li) "
|
||||
"with label '%s' was not stopped.\n",
|
||||
i, itr->back()->tid(), itr->back()->key().c_str());
|
||||
LOG_WARNING("Instrumentation bundle on thread {} (TID={}) "
|
||||
"with label '{}' was not stopped.",
|
||||
i, itr->back()->tid(), itr->back()->key());
|
||||
|
||||
itr->back()->stop();
|
||||
itr->back()->pop();
|
||||
@@ -928,7 +940,7 @@ rocprofsys_finalize_hidden(void)
|
||||
// stop the main gotcha which shuts down the pthread gotchas
|
||||
if(get_init_bundle())
|
||||
{
|
||||
ROCPROFSYS_DEBUG_F("Stopping main gotcha...\n");
|
||||
LOG_DEBUG("Stopping main gotcha...");
|
||||
get_init_bundle()->stop();
|
||||
|
||||
pthread_gotcha::shutdown();
|
||||
@@ -938,39 +950,38 @@ rocprofsys_finalize_hidden(void)
|
||||
// stop the gotcha bundle
|
||||
if(get_preinit_bundle())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down miscellaneous gotchas...\n");
|
||||
LOG_DEBUG("Shutting down miscellaneous gotchas...");
|
||||
get_preinit_bundle()->stop();
|
||||
component::mpi_gotcha::shutdown();
|
||||
}
|
||||
|
||||
if(get_use_process_sampling())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down background sampler...\n");
|
||||
LOG_DEBUG("Shutting down background sampler...");
|
||||
process_sampler::shutdown();
|
||||
}
|
||||
|
||||
if(get_use_causal())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down causal sampling...\n");
|
||||
LOG_DEBUG("Shutting down causal sampling...");
|
||||
causal::sampling::shutdown();
|
||||
}
|
||||
|
||||
if(get_use_sampling())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down sampling...\n");
|
||||
LOG_DEBUG("Shutting down sampling...");
|
||||
sampling::shutdown();
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(3, "Reporting the process- and thread-level metrics...\n");
|
||||
LOG_TRACE("Reporting the process- and thread-level metrics...");
|
||||
// report the high-level metrics for the process
|
||||
if(get_main_bundle())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(0, "\n");
|
||||
std::string _msg = JOIN("", *get_main_bundle());
|
||||
auto _pos = _msg.find(">>> ");
|
||||
if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
|
||||
ROCPROFSYS_VERBOSE_F(0, "%s\n", _msg.c_str());
|
||||
ROCPROFSYS_DEBUG_F("Resetting main bundle...\n");
|
||||
LOG_INFO("{}", _msg);
|
||||
LOG_DEBUG("Resetting main bundle...");
|
||||
get_main_bundle()->reset();
|
||||
}
|
||||
|
||||
@@ -989,39 +1000,40 @@ rocprofsys_finalize_hidden(void)
|
||||
std::string _msg = JOIN("", *itr);
|
||||
auto _pos = _msg.find(">>> ");
|
||||
if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
|
||||
ROCPROFSYS_VERBOSE_F(_thr_verbose, "%s\n", _msg.c_str());
|
||||
if(_thr_verbose >= 0)
|
||||
{
|
||||
LOG_INFO("{}", _msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(0, "\n");
|
||||
|
||||
// ensure that all the MT instances are flushed
|
||||
if(get_use_sampling())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Post-processing the sampling backtraces...\n");
|
||||
LOG_DEBUG("Post-processing the sampling backtraces...");
|
||||
sampling::post_process();
|
||||
}
|
||||
|
||||
if(get_use_causal())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Finishing the causal experiments...\n");
|
||||
LOG_DEBUG("Finishing the causal experiments...");
|
||||
causal::finish_experimenting();
|
||||
}
|
||||
|
||||
if(get_use_process_sampling())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Post-processing the system-level samples...\n");
|
||||
LOG_DEBUG("Post-processing the system-level samples...");
|
||||
process_sampler::post_process();
|
||||
}
|
||||
|
||||
// shutdown tasking before timemory is finalized
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down thread-pools...\n");
|
||||
LOG_DEBUG("Shutting down thread-pools...");
|
||||
tasking::shutdown();
|
||||
|
||||
if(get_use_code_coverage())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Post-processing the code coverage...\n");
|
||||
LOG_DEBUG("Post-processing the code coverage...");
|
||||
coverage::post_process();
|
||||
}
|
||||
|
||||
@@ -1029,13 +1041,13 @@ rocprofsys_finalize_hidden(void)
|
||||
|
||||
// Flush any pending region cache entries (e.g., main entry point that wasn't
|
||||
// explicitly stopped before finalization)
|
||||
ROCPROFSYS_DEBUG_F("Flushing pending region cache entries...\n");
|
||||
LOG_DEBUG("Flushing pending region cache entries...");
|
||||
rocprofsys_flush_pending_region_cache_hidden();
|
||||
|
||||
bool _perfetto_output_error = false;
|
||||
if(get_use_perfetto())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(0, "Finalizing perfetto...\n");
|
||||
LOG_DEBUG("Finalizing perfetto...");
|
||||
rocprofsys::perfetto::post_process(_timemory_manager.get(),
|
||||
_perfetto_output_error);
|
||||
}
|
||||
@@ -1061,7 +1073,7 @@ rocprofsys_finalize_hidden(void)
|
||||
tim::cereal::make_nvp("memory_maps", _maps));
|
||||
});
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(1, "Finalizing timemory...\n");
|
||||
LOG_DEBUG("Finalizing timemory...");
|
||||
tim::timemory_finalize(_timemory_manager.get());
|
||||
|
||||
auto _cfg = settings::compose_filename_config{};
|
||||
@@ -1077,25 +1089,24 @@ rocprofsys_finalize_hidden(void)
|
||||
|
||||
if(_perfetto_output_error)
|
||||
{
|
||||
ROCPROFSYS_THROW("Error opening perfetto output file: %s",
|
||||
get_perfetto_output_filename().c_str());
|
||||
throw std::runtime_error(fmt::format("Error opening perfetto output file: {}",
|
||||
get_perfetto_output_filename()));
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(
|
||||
_push_count > _pop_count &&
|
||||
!get_env("ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK", false, false),
|
||||
"%s",
|
||||
TIMEMORY_JOIN(" ",
|
||||
"rocprofsys_push_trace was called more times than "
|
||||
"rocprofsys_pop_trace. The inverse is fine but the current state "
|
||||
"means not every measurement was ended :: pushed:",
|
||||
_push_count, "vs. popped:", _pop_count)
|
||||
.c_str());
|
||||
if(get_is_continuous_integration() && _push_count > _pop_count &&
|
||||
!get_env<bool>("ROCPROFSYS_CI_SKIP_PUSH_POP_CHECK", false, false))
|
||||
{
|
||||
throw std::runtime_error(fmt::format(
|
||||
"rocprofsys_push_trace was called more times than "
|
||||
"rocprofsys_pop_trace. The inverse is fine but the current state "
|
||||
"means not every measurement was ended :: pushed: {} vs. popped: {}",
|
||||
_push_count, _pop_count));
|
||||
}
|
||||
|
||||
debug::close_file();
|
||||
// debug::close_file();
|
||||
config::finalize();
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(0, "Finalized: %s\n", _finalization.as_string().c_str());
|
||||
LOG_DEBUG("Finalized: {}", _finalization.as_string());
|
||||
|
||||
tim::signals::enable_signal_detection(
|
||||
{ tim::signals::sys_signal::SegFault, tim::signals::sys_signal::Stop },
|
||||
|
||||
@@ -39,7 +39,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/gpu.hpp"
|
||||
#include "core/gpu_metrics.hpp"
|
||||
#include "core/node_info.hpp"
|
||||
@@ -57,6 +56,8 @@
|
||||
#include <timemory/utility/delimit.hpp>
|
||||
#include <timemory/utility/locking.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cassert>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
@@ -411,8 +412,10 @@ get_version()
|
||||
{
|
||||
auto _err = amdsmi_get_lib_version(&_v);
|
||||
if(_err != AMDSMI_STATUS_SUCCESS)
|
||||
ROCPROFSYS_THROW(
|
||||
{
|
||||
throw std::runtime_error(
|
||||
"amdsmi_get_version failed. No version information available.");
|
||||
}
|
||||
}
|
||||
|
||||
return _v;
|
||||
@@ -429,15 +432,16 @@ check_error(const char* _file, int _line, amdsmi_status_t _code, bool* _option =
|
||||
return;
|
||||
}
|
||||
|
||||
constexpr const char* _unknown_error_message =
|
||||
"amdsmi_status_code_to_string failed. No error message available.";
|
||||
|
||||
const char* _msg = nullptr;
|
||||
auto _err = amdsmi_status_code_to_string(_code, &_msg);
|
||||
if(_err != AMDSMI_STATUS_SUCCESS)
|
||||
ROCPROFSYS_THROW(
|
||||
"amdsmi_status_code_to_string failed. No error message available. "
|
||||
"Error code %i originated at %s:%i\n",
|
||||
static_cast<int>(_code), _file, _line);
|
||||
ROCPROFSYS_THROW("[%s:%i] Error code %i :: %s", _file, _line, static_cast<int>(_code),
|
||||
_msg);
|
||||
auto _error_code_is_known =
|
||||
amdsmi_status_code_to_string(_code, &_msg) == AMDSMI_STATUS_SUCCESS;
|
||||
|
||||
throw std::runtime_error(
|
||||
fmt::format("[{}:{}] Error code {} :: {}", _file, _line, static_cast<int>(_code),
|
||||
_error_code_is_known ? _msg : _unknown_error_message));
|
||||
}
|
||||
|
||||
std::atomic<State>&
|
||||
@@ -533,9 +537,8 @@ data::sample(uint32_t _device_id)
|
||||
ROCPROFSYS_AMD_SMI_CALL(FUNCTION(__VA_ARGS__), &OPTION); \
|
||||
} catch(std::runtime_error & _e) \
|
||||
{ \
|
||||
ROCPROFSYS_VERBOSE_F( \
|
||||
0, "[%s] Exception: %s. Disabling future samples from amd-smi...\n", \
|
||||
#FUNCTION, _e.what()); \
|
||||
LOG_ERROR("Exception: {}. Disabling future samples from amd-smi...", \
|
||||
_e.what()); \
|
||||
get_state().store(State::Disabled); \
|
||||
} \
|
||||
}
|
||||
@@ -763,11 +766,10 @@ sample()
|
||||
for(auto itr : data::device_list)
|
||||
{
|
||||
if(amd_smi::get_state() != State::Active) continue;
|
||||
ROCPROFSYS_DEBUG_F("Polling amd-smi for device %u...\n", itr);
|
||||
LOG_TRACE("Polling amd-smi for device {}", itr);
|
||||
auto& _data = *_bundle_data.at(itr);
|
||||
if(!_data) continue;
|
||||
_data->emplace_back(data{ itr });
|
||||
ROCPROFSYS_DEBUG_F(" %s\n", TIMEMORY_JOIN("", _data->back()).c_str());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -828,10 +830,15 @@ data::post_process(uint32_t _dev_id)
|
||||
auto _amd_smi = (_amd_smi_v) ? *_amd_smi_v : std::deque<amd_smi::data>{};
|
||||
const auto& _thread_info = thread_info::get(0, InternalTID);
|
||||
|
||||
ROCPROFSYS_VERBOSE(1, "Post-processing %zu amd-smi samples from device %u\n",
|
||||
_amd_smi.size(), _dev_id);
|
||||
LOG_DEBUG("Post-processing {} amd-smi samples from device {}", _amd_smi.size(),
|
||||
_dev_id);
|
||||
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error("Missing thread info for thread 0");
|
||||
return;
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "Missing thread info for thread 0");
|
||||
if(!_thread_info) return;
|
||||
|
||||
auto _settings = get_settings(_dev_id);
|
||||
@@ -898,8 +905,7 @@ data::post_process(uint32_t _dev_id)
|
||||
{
|
||||
if(itr.m_gpu_metrics.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1, "No VCN activity data collected from device %u\n", _dev_id);
|
||||
LOG_DEBUG("No VCN activity data collected from device {}", _dev_id);
|
||||
}
|
||||
else if(gpu::vcn_is_device_level_only(_dev_id))
|
||||
{
|
||||
@@ -929,8 +935,7 @@ data::post_process(uint32_t _dev_id)
|
||||
{
|
||||
if(itr.m_gpu_metrics.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1, "No JPEG activity data collected from device %u\n", _dev_id);
|
||||
LOG_DEBUG("No JPEG activity data collected from device {}", _dev_id);
|
||||
}
|
||||
else if(gpu::jpeg_is_device_level_only(_dev_id))
|
||||
{
|
||||
@@ -961,8 +966,7 @@ data::post_process(uint32_t _dev_id)
|
||||
{
|
||||
if(itr.m_gpu_metrics.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1, "No XGMI activity data collected from device %u\n", _dev_id);
|
||||
LOG_DEBUG("No XGMI activity data collected from device {}", _dev_id);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -982,8 +986,7 @@ data::post_process(uint32_t _dev_id)
|
||||
{
|
||||
if(itr.m_gpu_metrics.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1, "No PCIe activity data collected from device %u\n", _dev_id);
|
||||
LOG_DEBUG("No PCIe activity data collected from device {}", _dev_id);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -1139,14 +1142,13 @@ setup()
|
||||
|
||||
if(!gpu::initialize_amdsmi())
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(0,
|
||||
"AMD SMI is not available. Disabling AMD SMI sampling...");
|
||||
LOG_WARNING("AMD SMI is not available. Disabling AMD SMI sampling...");
|
||||
return;
|
||||
}
|
||||
|
||||
amdsmi_version_t _version = get_version();
|
||||
ROCPROFSYS_VERBOSE_F(0, "AMD SMI version: %u.%u.%u - str: %s.\n", _version.major,
|
||||
_version.minor, _version.release, _version.build);
|
||||
LOG_INFO("AMD SMI version: {} - str: {}.", _version.major, _version.minor,
|
||||
_version.release, _version.build);
|
||||
|
||||
data::device_count = gpu::device_count();
|
||||
|
||||
@@ -1177,18 +1179,22 @@ setup()
|
||||
{
|
||||
if(itr.find_first_not_of("0123456789-") != std::string::npos)
|
||||
{
|
||||
ROCPROFSYS_THROW("Invalid GPU specification: '%s'. Only numerical values "
|
||||
"(e.g., 0) or ranges (e.g., 0-7) are permitted.",
|
||||
itr.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Invalid GPU specification: '{}'. Only numerical values "
|
||||
"(e.g., 0) or ranges (e.g., 0-7) are permitted.",
|
||||
itr));
|
||||
}
|
||||
|
||||
if(itr.find('-') != std::string::npos)
|
||||
{
|
||||
auto _v = tim::delimit(itr, "-");
|
||||
ROCPROFSYS_CONDITIONAL_THROW(_v.size() != 2,
|
||||
"Invalid GPU range specification: '%s'. "
|
||||
"Required format N-M, e.g. 0-4",
|
||||
itr.c_str());
|
||||
if(_v.size() != 2)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Invalid GPU range specification: '{}'. "
|
||||
"Required format N-M, e.g. 0-4",
|
||||
itr));
|
||||
}
|
||||
for(auto i = std::stoul(_v.at(0)); i < std::stoul(_v.at(1)); ++i)
|
||||
_emplace(i);
|
||||
}
|
||||
@@ -1233,11 +1239,13 @@ setup()
|
||||
{
|
||||
auto iitr = supported.find(metric);
|
||||
if(iitr == supported.end())
|
||||
ROCPROFSYS_FAIL_F("unsupported amd-smi metric: %s\n",
|
||||
metric.c_str());
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
1, "Enabling amd-smi metric '%s' on device [%u]\n",
|
||||
metric.c_str(), itr);
|
||||
{
|
||||
LOG_CRITICAL("Unsupported amd-smi metric: {}", metric);
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
std::exit(1);
|
||||
}
|
||||
LOG_DEBUG("Enabling amd-smi metric '{}' on device [{}]", metric,
|
||||
itr);
|
||||
iitr->second = true;
|
||||
}
|
||||
}
|
||||
@@ -1249,8 +1257,7 @@ setup()
|
||||
|
||||
} catch(std::runtime_error& _e)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "Exception thrown when initializing amd-smi: %s\n",
|
||||
_e.what());
|
||||
LOG_WARNING("Exception thrown when initializing amd-smi: {}", _e.what());
|
||||
data::device_list = {};
|
||||
}
|
||||
}
|
||||
@@ -1261,15 +1268,14 @@ shutdown()
|
||||
auto_lock_t _lk{ type_mutex<category::amd_smi>() };
|
||||
|
||||
if(!is_initialized()) return;
|
||||
ROCPROFSYS_VERBOSE_F(1, "Shutting down amd-smi...\n");
|
||||
LOG_DEBUG("Shutting down amd-smi...");
|
||||
|
||||
try
|
||||
{
|
||||
data::shutdown();
|
||||
} catch(std::runtime_error& _e)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "Exception thrown when shutting down amd-smi: %s\n",
|
||||
_e.what());
|
||||
LOG_WARNING("Exception thrown when shutting down amd-smi: {}", _e.what());
|
||||
}
|
||||
|
||||
is_initialized() = false;
|
||||
@@ -1280,7 +1286,7 @@ post_process()
|
||||
{
|
||||
for(auto itr : data::device_list)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "Post-processing amd-smi data for device: %d", itr);
|
||||
LOG_DEBUG("Post-processing amd-smi data for device: {}", itr);
|
||||
data::post_process(itr);
|
||||
}
|
||||
}
|
||||
@@ -1295,7 +1301,7 @@ void
|
||||
postfork_child_cleanup()
|
||||
{
|
||||
// In child process, disable AMD SMI to prevent shutdown errors
|
||||
ROCPROFSYS_VERBOSE_F(2, "Disabling AMD SMI in child process after fork...\n");
|
||||
LOG_DEBUG("Disabling AMD SMI in child process after fork...");
|
||||
|
||||
// Set to Finalized to prevent any sampling attempts (though is_child_process() check
|
||||
// in sample() already handles this)
|
||||
@@ -1313,7 +1319,7 @@ postfork_parent_reinit()
|
||||
{
|
||||
// In parent process, AMD SMI device handles may be corrupted after fork
|
||||
// Reinitialize AMD SMI to get fresh handles
|
||||
ROCPROFSYS_VERBOSE_F(2, "Reinitializing AMD SMI in parent process after fork...\n");
|
||||
LOG_DEBUG("Reinitializing AMD SMI in parent process after fork...");
|
||||
|
||||
// Shutdown and reinitialize to get fresh device handles
|
||||
shutdown();
|
||||
|
||||
+3
-2
@@ -23,7 +23,6 @@
|
||||
#include "library/causal/components/backtrace.hpp"
|
||||
#include "core/concepts.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/utility.hpp"
|
||||
#include "library/causal/data.hpp"
|
||||
@@ -44,6 +43,8 @@
|
||||
#include <timemory/units.hpp>
|
||||
#include <timemory/utility/backtrace.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <ctime>
|
||||
#include <execinfo.h>
|
||||
@@ -218,7 +219,7 @@ backtrace::sample(int _sig)
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("unhandled signal %i\n", _sig);
|
||||
throw std::runtime_error(fmt::format("Unhandled signal {}", _sig));
|
||||
}
|
||||
|
||||
++_protect_flag;
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "library/causal/components/blocking_gotcha.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "library/causal/components/causal_gotcha.hpp"
|
||||
#include "library/causal/delay.hpp"
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include "library/causal/components/progress_point.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/concepts.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
#include "library/causal/experiment.hpp"
|
||||
#include "library/thread_data.hpp"
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "library/causal/components/unblocking_gotcha.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "library/causal/components/causal_gotcha.hpp"
|
||||
#include "library/causal/delay.hpp"
|
||||
|
||||
@@ -29,7 +29,6 @@
|
||||
#include "core/binary/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/containers/c_array.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/utility.hpp"
|
||||
@@ -54,6 +53,8 @@
|
||||
#include <timemory/utility/procfs/maps.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
@@ -88,8 +89,12 @@ auto speedup_dist = []() {
|
||||
size_t _nzero = std::ceil(_v.size() / 4.0);
|
||||
_v.resize(_v.size() + _nzero, 0);
|
||||
std::sort(_v.begin(), _v.end());
|
||||
ROCPROFSYS_CI_THROW(_v.back() > 100, "Error! last value is too large: %i\n",
|
||||
(int) _v.back());
|
||||
if(get_is_continuous_integration() && _v.back() > 100)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! last value is too large: {}", _v.back()));
|
||||
}
|
||||
|
||||
return _v;
|
||||
}();
|
||||
|
||||
@@ -192,14 +197,11 @@ get_filters(const std::set<binary::scope_filter::filter_scope>& _scopes = {
|
||||
if(_former_include != _current_include)
|
||||
{
|
||||
if(!_binary_include.empty())
|
||||
ROCPROFSYS_VERBOSE(0, "[causal] binary scope : %s\n",
|
||||
_binary_include.c_str());
|
||||
LOG_DEBUG("[causal] binary scope : {}", _binary_include);
|
||||
if(!_source_include.empty())
|
||||
ROCPROFSYS_VERBOSE(0, "[causal] source scope : %s\n",
|
||||
_source_include.c_str());
|
||||
LOG_DEBUG("[causal] source scope : {}", _source_include);
|
||||
if(!_function_include.empty())
|
||||
ROCPROFSYS_VERBOSE(0, "[causal] function scope : %s\n",
|
||||
_function_include.c_str());
|
||||
LOG_DEBUG("[causal] function scope : {}", _function_include);
|
||||
_former_include = _current_include;
|
||||
}
|
||||
|
||||
@@ -230,14 +232,11 @@ get_filters(const std::set<binary::scope_filter::filter_scope>& _scopes = {
|
||||
if(_former_exclude != _current_exclude)
|
||||
{
|
||||
if(!_binary_exclude.empty())
|
||||
ROCPROFSYS_VERBOSE(0, "[causal] binary exclude : %s\n",
|
||||
_binary_exclude.c_str());
|
||||
LOG_DEBUG("[causal] binary exclude : {}", _binary_exclude);
|
||||
if(!_source_exclude.empty())
|
||||
ROCPROFSYS_VERBOSE(0, "[causal] source exclude : %s\n",
|
||||
_source_exclude.c_str());
|
||||
LOG_DEBUG("[causal] source exclude : {}", _source_exclude);
|
||||
if(!_function_exclude.empty())
|
||||
ROCPROFSYS_VERBOSE(0, "[causal] function exclude : %s\n",
|
||||
_function_exclude.c_str());
|
||||
LOG_DEBUG("[causal] function exclude : {}", _function_exclude);
|
||||
_former_exclude = _current_exclude;
|
||||
}
|
||||
|
||||
@@ -342,10 +341,9 @@ compute_eligible_lines_impl()
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0, "[causal] eligible address ranges: %zu, coarse address range: %zu [%s]\n",
|
||||
_eligible_ar.size(), _eligible_ar.range_size(),
|
||||
_eligible_ar.get_coarse_range().as_string().c_str());
|
||||
LOG_DEBUG("[causal] eligible address ranges: {}, coarse address range: {} [{}]",
|
||||
_eligible_ar.size(), _eligible_ar.range_size(),
|
||||
_eligible_ar.get_coarse_range().as_string());
|
||||
|
||||
if(_eligible_ar.empty())
|
||||
{
|
||||
@@ -355,10 +353,12 @@ compute_eligible_lines_impl()
|
||||
save_line_info(_cfg, config::get_verbose());
|
||||
}
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_eligible_ar.empty(),
|
||||
"Error! binary analysis (after filters) resulted in zero eligible instruction "
|
||||
"pointer addresses for causal experimentation");
|
||||
if(_eligible_ar.empty())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
"Error! binary analysis (after filters) resulted in "
|
||||
"zero eligible instruction pointer addresses for causal experimentation");
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
@@ -388,8 +388,8 @@ save_line_info_impl(std::ostream& _ofs,
|
||||
auto _write_impl = [&_ofs, &_info](const binary::binary_info& _data) {
|
||||
for(const auto& itr : _data.mappings)
|
||||
{
|
||||
_ofs << itr.pathname << " [" << as_hex(itr.load_address) << " - "
|
||||
<< as_hex(itr.last_address) << "]\n";
|
||||
_ofs << itr.pathname << " [" << fmt::format("0x{:X}", itr.load_address)
|
||||
<< " - " << fmt::format("0x{:X}", itr.last_address) << "]\n";
|
||||
}
|
||||
|
||||
auto _emitted_dwarf_addresses = std::set<uintptr_t>{};
|
||||
@@ -397,7 +397,7 @@ save_line_info_impl(std::ostream& _ofs,
|
||||
{
|
||||
auto _addr = itr.address;
|
||||
auto _addr_off = itr.address + itr.load_address;
|
||||
_ofs << " " << as_hex(_addr_off) << " [" << as_hex(_addr)
|
||||
_ofs << " " << _addr_off.as_hex() << " [" << _addr.as_hex()
|
||||
<< "] :: " << itr.file;
|
||||
if(itr.line > 0) _ofs << ":" << itr.line;
|
||||
if(!itr.func.empty())
|
||||
@@ -419,7 +419,7 @@ save_line_info_impl(std::ostream& _ofs,
|
||||
{
|
||||
for(const auto& ditr : itr.dwarf_info)
|
||||
{
|
||||
_ofs << " " << as_hex(ditr.address) << " :: " << ditr.file
|
||||
_ofs << " " << ditr.address.as_hex() << " :: " << ditr.file
|
||||
<< ":" << ditr.line;
|
||||
_ofs << "\n";
|
||||
_emitted_dwarf_addresses.emplace(ditr.address.low);
|
||||
@@ -432,7 +432,7 @@ save_line_info_impl(std::ostream& _ofs,
|
||||
for(const auto& itr : _data.debug_info)
|
||||
{
|
||||
if(_emitted_dwarf_addresses.count(itr.address.low) > 0) continue;
|
||||
_ofs << " " << as_hex(itr.address) << " :: " << itr.file << ":"
|
||||
_ofs << " " << itr.address.as_hex() << " :: " << itr.file << ":"
|
||||
<< itr.line;
|
||||
_ofs << "\n";
|
||||
}
|
||||
@@ -471,8 +471,10 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
|
||||
const auto& _thr_info = thread_info::init(true);
|
||||
set_thread_state(ThreadState::Disabled);
|
||||
ROCPROFSYS_CONDITIONAL_THROW(!_thr_info->is_offset,
|
||||
"Error! causal profiling thread should be offset");
|
||||
if(!_thr_info->is_offset)
|
||||
{
|
||||
throw std::runtime_error("Error! causal profiling thread should be offset");
|
||||
}
|
||||
|
||||
if(!perform_experiment_impl_completed)
|
||||
perform_experiment_impl_completed = std::make_unique<std::promise<void>>();
|
||||
@@ -506,8 +508,7 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
|
||||
if(_delay_sec > 0.0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(1, "[causal] delaying experimentation for %.2f seconds...\n",
|
||||
_delay_sec);
|
||||
LOG_DEBUG("[causal] delaying experimentation for {} seconds...", _delay_sec);
|
||||
uint64_t _delay_nsec = _delay_sec * units::sec;
|
||||
std::this_thread::yield();
|
||||
std::this_thread::sleep_for(std::chrono::nanoseconds{ _delay_nsec });
|
||||
@@ -521,12 +522,10 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
auto _elapsed = clock_type::now() - _start_time;
|
||||
if(_elapsed >= _duration_nsec)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1,
|
||||
"[causal] stopping experimentation after %.2f seconds "
|
||||
"(elapsed: %.2f seconds)...\n",
|
||||
_duration_sec,
|
||||
std::chrono::duration_cast<duration_sec_t>(_elapsed).count());
|
||||
LOG_DEBUG("[causal] stopping experimentation after {} seconds "
|
||||
"(elapsed: {} seconds)...",
|
||||
_duration_sec,
|
||||
std::chrono::duration_cast<duration_sec_t>(_elapsed).count());
|
||||
causal::sampling::post_process();
|
||||
return true;
|
||||
}
|
||||
@@ -546,9 +545,8 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
{
|
||||
if(_impl_no > 0) return;
|
||||
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0,
|
||||
"[causal] experiment failed to start. Number of PC candidates: %zu\n",
|
||||
LOG_DEBUG(
|
||||
"[causal] experiment failed to start. Number of PC candidates: {}",
|
||||
eligible_pc_candidates.load());
|
||||
|
||||
auto _memory = std::stringstream{};
|
||||
@@ -584,7 +582,7 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
for(const auto& itr : _eligible_pc_hist)
|
||||
{
|
||||
_eligible << " " << std::setw(8) << itr.second
|
||||
<< " :: " << as_hex(itr.first) << "\n";
|
||||
<< " :: " << fmt::format("0x{:X}", itr.first) << "\n";
|
||||
}
|
||||
|
||||
auto _samples = std::vector<std::pair<uintptr_t, size_t>>{};
|
||||
@@ -608,7 +606,8 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
{
|
||||
_sample << " " << std::setw(8) << itr.second
|
||||
<< " :: " << std::setw(5) << std::boolalpha
|
||||
<< _is_eligible << " :: " << as_hex(itr.first) << " "
|
||||
<< _is_eligible
|
||||
<< " :: " << fmt::format("0x{:X}", itr.first) << " "
|
||||
<< _linfo->location << ":" << _linfo->lineno << " ["
|
||||
<< rocprofsys::utility::demangle(_linfo->name)
|
||||
<< "]\n";
|
||||
@@ -617,7 +616,8 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
_sample
|
||||
<< " " << std::setw(8) << itr.second
|
||||
<< " :: " << std::setw(5) << std::boolalpha
|
||||
<< _is_eligible << " :: " << as_hex(itr.first) << " "
|
||||
<< _is_eligible
|
||||
<< " :: " << fmt::format("0x{:X}", itr.first) << " "
|
||||
<< iitr.location << ":" << iitr.line << " ["
|
||||
<< rocprofsys::utility::demangle(iitr.name) << "]\n";
|
||||
}
|
||||
@@ -625,19 +625,14 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_PRINT_COLOR(fatal, "causal experiment never started\n");
|
||||
LOG_CRITICAL("Causal experiment never started");
|
||||
|
||||
std::cerr << std::flush;
|
||||
auto _cerr = tim::log::warning_stream(std::cerr);
|
||||
_cerr << "\npc samples:\n\n" << _sample.str() << "\n";
|
||||
_cerr << "\neligible pcs:\n\n" << _eligible.str() << "\n";
|
||||
_cerr << "\nscoped pcs:\n\n" << _scoped.str() << "\n";
|
||||
LOG_WARNING("pc samples: {} eligible pcs: {} scoped pcs: {}",
|
||||
_sample.str(), _eligible.str(), _scoped.str());
|
||||
if(get_verbose() >= 1)
|
||||
{
|
||||
_cerr << "\nbinary pcs:\n\n" << _binary.str() << "\n";
|
||||
_cerr << "\nmaps:\n\n" << _memory.str() << "\n";
|
||||
LOG_WARNING("binary pcs: {} maps: {}", _binary.str(), _memory.str());
|
||||
}
|
||||
std::cerr << std::flush;
|
||||
|
||||
// if launched via rocprof-sys-causal, allow end-to-end runs that do not
|
||||
// start experiments
|
||||
@@ -647,24 +642,25 @@ perform_experiment_impl(std::shared_ptr<std::promise<void>> _started) // NOLINT
|
||||
|
||||
if(!(get_causal_end_to_end() && _omni_causal_launcher))
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_THROW(_impl_no == 0,
|
||||
"causal experiment never started");
|
||||
if(_impl_no == 0)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
"Error! Causal experiment never started");
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1,
|
||||
"[causal] experiment failed to start. Number of PC candidates: %zu\n",
|
||||
LOG_DEBUG(
|
||||
"[causal] experiment failed to start. Number of PC candidates: {}",
|
||||
eligible_pc_candidates.load());
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(3,
|
||||
"[causal] experiment started. Number of PC candidates: %zu\n",
|
||||
eligible_pc_candidates.load());
|
||||
LOG_TRACE("[causal] experiment started. Number of PC candidates: {}",
|
||||
eligible_pc_candidates.load());
|
||||
|
||||
reset_sample_selection();
|
||||
|
||||
@@ -729,7 +725,7 @@ save_line_info(const settings::compose_filename_config& _cfg, int _verbose)
|
||||
}
|
||||
else
|
||||
{
|
||||
throw ::rocprofsys::exception<std::runtime_error>("Error opening " + ofname);
|
||||
throw std::runtime_error(fmt::format("Error opening {}", ofname));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -796,7 +792,7 @@ sample_selection(size_t _nitr, size_t _wait_ns)
|
||||
// kept because of size() - 1 in distribution range
|
||||
if(ROCPROFSYS_UNLIKELY(_address_vec.empty()))
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "no addresses for sample selection...\n");
|
||||
LOG_WARNING("No addresses for sample selection...");
|
||||
return selected_entry{};
|
||||
}
|
||||
|
||||
@@ -838,14 +834,14 @@ sample_selection(size_t _nitr, size_t _wait_ns)
|
||||
{
|
||||
if(ROCPROFSYS_UNLIKELY(config::get_debug()))
|
||||
{
|
||||
ROCPROFSYS_WARNING(
|
||||
0, "[%s][%s][%s][%s] %s [%s:%i][%s][%zu]\n",
|
||||
as_hex(_lookup_addr).c_str(), as_hex(_addr).c_str(),
|
||||
as_hex(_sym_addr).c_str(),
|
||||
(_location.empty()) ? "" : _location.data(),
|
||||
rocprofsys::utility::demangle(itr.func).c_str(),
|
||||
itr.file.c_str(), itr.line, itr.address.as_string().c_str(),
|
||||
itr.address.size());
|
||||
LOG_WARNING("[{}][{}][{}] [{}] {} [{}:{}][{}][{}]",
|
||||
fmt::format("0x{:X}", _lookup_addr),
|
||||
fmt::format("0x{:X}", _addr),
|
||||
fmt::format("0x{:X}", _sym_addr),
|
||||
(_location.empty()) ? "" : _location,
|
||||
rocprofsys::utility::demangle(itr.func), itr.file,
|
||||
itr.line, itr.address.as_string(),
|
||||
itr.address.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -871,7 +867,7 @@ sample_selection(size_t _nitr, size_t _wait_ns)
|
||||
{
|
||||
if(ROCPROFSYS_UNLIKELY(!aitr))
|
||||
{
|
||||
ROCPROFSYS_WARNING(0, "invalid atomic pc...\n");
|
||||
LOG_WARNING("Invalid atomic pc...");
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -903,12 +899,13 @@ get_line_info(uintptr_t _addr, bool _include_discarded)
|
||||
|
||||
// make sure the address is in the coarse grained mapped regions
|
||||
// before performing an exhaustive search
|
||||
bool _is_mapped = std::find_if(litr.mappings.begin(), litr.mappings.end(),
|
||||
[_addr](const auto& mitr) {
|
||||
return address_range_t{ mitr.load_address,
|
||||
mitr.last_address }
|
||||
.contains(_addr);
|
||||
}) != litr.mappings.end();
|
||||
bool _is_mapped =
|
||||
std::find_if(litr.mappings.begin(), litr.mappings.end(),
|
||||
[_addr](const auto& mitr) {
|
||||
return binary::address_range{ mitr.load_address,
|
||||
mitr.last_address }
|
||||
.contains(_addr);
|
||||
}) != litr.mappings.end();
|
||||
|
||||
if(!_is_mapped) return;
|
||||
|
||||
@@ -942,10 +939,10 @@ get_line_info(uintptr_t _addr, bool _include_discarded)
|
||||
for(const auto& itr : ditr.get_debug_line_info(_filters))
|
||||
{
|
||||
if(!_ipaddr.contains(itr.ipaddr()))
|
||||
ROCPROFSYS_THROW(
|
||||
"Error! debug line info ipaddr (%s) is not contained in "
|
||||
"symbol ipaddr (%s)",
|
||||
as_hex(itr.ipaddr()).c_str(), as_hex(_ipaddr).c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! debug line info ipaddr ({}) is not "
|
||||
"contained in symbol ipaddr ({})",
|
||||
itr.ipaddr().as_hex(), _ipaddr.as_hex()));
|
||||
if(itr.ipaddr().contains(_addr)) _debug_data.emplace_back(itr);
|
||||
}
|
||||
utility::combine(_local_data, _debug_data);
|
||||
@@ -1064,10 +1061,12 @@ start_experimenting()
|
||||
speedup_dist.clear();
|
||||
for(auto itr : _user_speedup_dist)
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(itr > 100,
|
||||
"Virtual speedups must be in range [0, 100]. "
|
||||
"Invalid virtual speedup: %lu\n",
|
||||
itr);
|
||||
if(itr > 100)
|
||||
{
|
||||
LOG_DEBUG("Virtual speedups must be in range [0, 100]. "
|
||||
"Invalid virtual speedup: {}",
|
||||
itr);
|
||||
}
|
||||
speedup_dist.emplace_back(static_cast<uint16_t>(itr));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,6 +31,8 @@
|
||||
#include "library/thread_info.hpp"
|
||||
#include "library/tracing.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <timemory/components/macros.hpp>
|
||||
#include <timemory/mpl/concepts.hpp>
|
||||
#include <timemory/mpl/types.hpp>
|
||||
@@ -72,16 +74,17 @@ compute_sleep_for_overhead()
|
||||
int64_t _end = tracing::now();
|
||||
if(i < _nwarm) continue;
|
||||
auto _diff = (_end - _beg);
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_diff < _val, "Error! sleep_for(%zu) [nanoseconds] >= %zu", _val, _diff);
|
||||
if(_diff < _val)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! sleep_for({}) [nanoseconds] >= {}", _val, _diff));
|
||||
}
|
||||
_stats += (_diff - _val);
|
||||
}
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE(2,
|
||||
"[causal] overhead of std::this_thread::sleep_for(...) "
|
||||
"invocation = %6.3f usec +/- %e\n",
|
||||
_stats.get_mean() / units::usec,
|
||||
_stats.get_stddev() / units::usec);
|
||||
LOG_TRACE("[causal] overhead of std::this_thread::sleep_for(...) "
|
||||
"invocation = {} usec +/- {} usec",
|
||||
_stats.get_mean() / units::usec, _stats.get_stddev() / units::usec);
|
||||
|
||||
tim::manager::instance()->add_metadata([_stats](auto& ar) {
|
||||
ar(tim::cereal::make_nvp("causal thread sleep overhead [nsec]", _stats));
|
||||
|
||||
+34
-34
@@ -26,7 +26,6 @@
|
||||
#include "binary/symbol.hpp"
|
||||
#include "common/defines.h"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "library/causal/components/backtrace.hpp"
|
||||
@@ -48,6 +47,8 @@
|
||||
#include <timemory/units.hpp>
|
||||
#include <timemory/unwind/dlinfo.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <ratio>
|
||||
#include <regex>
|
||||
@@ -252,8 +253,7 @@ experiment::start()
|
||||
init_progress = component::progress_point::get_progress_points();
|
||||
start_time = tracing::now();
|
||||
|
||||
ROCPROFSYS_VERBOSE(0, "Starting causal experiment #%-3u: %s\n", index,
|
||||
as_string().c_str());
|
||||
LOG_INFO("Starting causal experiment #{}: {}", index, as_string());
|
||||
|
||||
if(get_state() < State::Finalized)
|
||||
{
|
||||
@@ -321,34 +321,30 @@ experiment::stop()
|
||||
|
||||
if(_lowv <= 3 && (_mean < 5 || _medi < 5))
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2,
|
||||
"[progress points] increasing experiment time :: low: %6.3f, "
|
||||
"high: %6.3f, mean: %6.3f, median: %zi\n",
|
||||
_lowv, _high, _mean, _medi);
|
||||
LOG_DEBUG("[progress points] increasing experiment time :: low: {}, high: {}, "
|
||||
"mean: {}, median: {}",
|
||||
_lowv, _high, _mean, _medi);
|
||||
global_scaling *= 2;
|
||||
++global_scaling_increments; // keep track of how many successive increments have
|
||||
// been performed
|
||||
}
|
||||
else if(_mean > 10 && _lowv >= 8 && global_scaling > 1)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2,
|
||||
"[progress points] decreasing experiment time :: low: %6.3f, "
|
||||
"high: %6.3f, mean: %6.3f, median: %zi\n",
|
||||
_lowv, _high, _mean, _medi);
|
||||
LOG_DEBUG("[progress points] decreasing experiment time :: low: {}, high: {}, "
|
||||
"mean: {}, median: {}",
|
||||
_lowv, _high, _mean, _medi);
|
||||
global_scaling /= 2;
|
||||
global_scaling_increments = 0;
|
||||
}
|
||||
|
||||
if(ROCPROFSYS_UNLIKELY(global_scaling_increments >= 5))
|
||||
{
|
||||
ROCPROFSYS_WARNING(
|
||||
0,
|
||||
"Warning! causal experimentation hasn't seen at least 5 progress points "
|
||||
"in the last %li experiments. Progress points are necessary for measuring "
|
||||
LOG_WARNING(
|
||||
"causal experimentation hasn't seen at least 5 progress points "
|
||||
"in the last {} experiments. Progress points are necessary for measuring "
|
||||
"the effect of the virtual speed-up. Please visit "
|
||||
"https://rocm.docs.amd.com/projects/rocprofiler-systems/en/latest/ for "
|
||||
"documentation on progress "
|
||||
"points and how to add them\n",
|
||||
"documentation on progress points and how to add them",
|
||||
global_scaling_increments);
|
||||
}
|
||||
|
||||
@@ -371,9 +367,9 @@ experiment::as_string() const
|
||||
if(!config::get_causal_end_to_end())
|
||||
_ss << ", duration: " << std::setw(5) << std::fixed << std::setprecision(3)
|
||||
<< _dur << " sec";
|
||||
_ss << " :: experiment: " << as_hex(selection.address) << " ";
|
||||
_ss << " :: experiment: " << fmt::format("0x{:X}", selection.address) << " ";
|
||||
if(selection.symbol_address > 0 && selection.address != selection.symbol_address)
|
||||
_ss << "(symbol@" << as_hex(selection.symbol_address) << ") ";
|
||||
_ss << "(symbol@" << fmt::format("0x{:X}", selection.symbol_address) << ") ";
|
||||
if(!selection.symbol.file.empty() && selection.symbol.line > 0)
|
||||
_ss << "[" << filepath::basename(selection.symbol.file) << ":"
|
||||
<< selection.symbol.line << "]";
|
||||
@@ -523,8 +519,8 @@ experiment::save_experiments(std::string _fname_base, const filename_config_t& _
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(1, "Processing line info for %zu sampled addresses...\n",
|
||||
_total_samples.size());
|
||||
LOG_DEBUG("Processing line info for {} sampled addresses...",
|
||||
_total_samples.size());
|
||||
|
||||
for(const auto& itr : _total_samples)
|
||||
{
|
||||
@@ -571,8 +567,8 @@ experiment::save_experiments(std::string _fname_base, const filename_config_t& _
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("Error opening causal experiments output file: %s",
|
||||
_fname.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error opening causal experiments output file: %s", _fname));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -614,12 +610,15 @@ experiment::save_experiments(std::string _fname_base, const filename_config_t& _
|
||||
? _line_info.func
|
||||
: join(":", _line_info.file, _line_info.line);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_name.empty(),
|
||||
"Error! causal experiment selection has no name: address=%s, file=%s, "
|
||||
"line=%u, func=%s",
|
||||
as_hex(_line_info.address).c_str(), _line_info.file.c_str(),
|
||||
_line_info.line, _line_info.func.c_str());
|
||||
if(_name.empty())
|
||||
{
|
||||
throw std::runtime_error(fmt::format("Error! causal experiment selection "
|
||||
"has no name: address={}, file={}, "
|
||||
"line={}, func={}",
|
||||
_line_info.address.as_hex(),
|
||||
_line_info.file, _line_info.line,
|
||||
_line_info.func));
|
||||
}
|
||||
|
||||
ofs << "experiment\tselected=" << rocprofsys::utility::demangle(_name)
|
||||
<< "\tspeedup=" << std::setprecision(2)
|
||||
@@ -663,14 +662,15 @@ experiment::save_experiments(std::string _fname_base, const filename_config_t& _
|
||||
{
|
||||
ofs << "samples\tlocation=" << itr.get_identifier()
|
||||
<< "\tcount=" << itr.count;
|
||||
if(config::get_debug()) ofs << "\taddress=" << as_hex(itr.address);
|
||||
if(config::get_debug())
|
||||
ofs << "\taddress=" << fmt::format("0x{:X}", itr.address);
|
||||
ofs << "\n";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("Error opening causal experiments output file: %s",
|
||||
_fname.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error opening causal experiments output file: {}", _fname));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -707,8 +707,8 @@ experiment::load_experiments(std::string _fname, const filename_config_t& _cfg,
|
||||
{
|
||||
if(_throw_on_error)
|
||||
{
|
||||
ROCPROFSYS_THROW("Error opening causal experiments input file: %s",
|
||||
_fname.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error opening causal experiments input file: %s", _fname));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+38
-20
@@ -25,7 +25,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/concepts.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/locking.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/utility.hpp"
|
||||
@@ -49,6 +48,8 @@
|
||||
#include <timemory/utility/backtrace.hpp>
|
||||
#include <timemory/variadic.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <csignal>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
@@ -219,9 +220,11 @@ configure(bool _setup, int64_t _tid)
|
||||
auto& _running = get_causal_sampler_running(_tid);
|
||||
auto& _signal_types = get_causal_sampler_signals(_tid);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(get_use_sampling(),
|
||||
"Internal error! configuring causal profiling not "
|
||||
if(get_use_sampling())
|
||||
{
|
||||
throw std::runtime_error("Internal error! configuring causal profiling not "
|
||||
"permitted when sampling is enabled");
|
||||
}
|
||||
|
||||
ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
|
||||
|
||||
@@ -287,7 +290,7 @@ configure(bool _setup, int64_t _tid)
|
||||
return perf::get_instance(_idx)->stop();
|
||||
},
|
||||
_tid, threading::get_sys_tid() });
|
||||
if(_tid == 0) ROCPROFSYS_VERBOSE(1, "causal profiling backend: perf\n");
|
||||
if(_tid == 0) LOG_DEBUG("Causal profiling backend: perf");
|
||||
}
|
||||
|
||||
return _open_error;
|
||||
@@ -301,11 +304,16 @@ configure(bool _setup, int64_t _tid)
|
||||
_causal->configure(timer{ get_sampling_realtime_signal(), CLOCK_REALTIME,
|
||||
SIGEV_THREAD_ID, 1000.0, 1.0e-6, _tid,
|
||||
threading::get_sys_tid() });
|
||||
if(_tid == 0) ROCPROFSYS_VERBOSE(1, "causal profiling backend: timer\n");
|
||||
if(_tid == 0) LOG_DEBUG("Causal profiling backend: timer");
|
||||
return true;
|
||||
};
|
||||
|
||||
TIMEMORY_REQUIRE(_causal) << "nullptr to causal profiling instance";
|
||||
if(!_causal)
|
||||
{
|
||||
LOG_CRITICAL("nullptr to causal profiling instance");
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
_causal->set_flags(SA_RESTART);
|
||||
_causal->set_verbose(_verbose);
|
||||
@@ -314,14 +322,20 @@ configure(bool _setup, int64_t _tid)
|
||||
if(get_causal_backend() == CausalBackend::Perf)
|
||||
{
|
||||
auto _perf_error = _activate_perf_backend();
|
||||
ROCPROFSYS_REQUIRE(!_perf_error)
|
||||
<< "perf backend for causal profiling failed to activate: "
|
||||
<< *_perf_error << "\n";
|
||||
if(_perf_error)
|
||||
{
|
||||
LOG_ERROR("Perf backend for causal profiling failed to activate: {}",
|
||||
*_perf_error);
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
else if(get_causal_backend() == CausalBackend::Timer)
|
||||
{
|
||||
ROCPROFSYS_REQUIRE(_activate_timer_backend())
|
||||
<< "timer backend for causal profiling failed to activate\n";
|
||||
if(!_activate_timer_backend())
|
||||
{
|
||||
LOG_ERROR("Timer backend for causal profiling failed to activate");
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
else if(get_causal_backend() == CausalBackend::Auto)
|
||||
{
|
||||
@@ -333,12 +347,14 @@ configure(bool _setup, int64_t _tid)
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
0, "perf backend for causal profiling failed to activate: %s\n",
|
||||
_perf_error->c_str());
|
||||
LOG_WARNING("Perf backend for causal profiling failed to activate: {}",
|
||||
_perf_error->c_str());
|
||||
|
||||
ROCPROFSYS_REQUIRE(_activate_timer_backend())
|
||||
<< "timer backend for causal profiling failed to activate\n";
|
||||
if(!_activate_timer_backend())
|
||||
{
|
||||
LOG_ERROR("Timer backend for causal profiling failed to activate");
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
config::set_setting_value("ROCPROFSYS_CAUSAL_BACKEND",
|
||||
std::string{ "timer" });
|
||||
@@ -354,7 +370,7 @@ configure(bool _setup, int64_t _tid)
|
||||
}
|
||||
else if(!_setup && _causal && _running)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("Destroying causal sampler for thread %lu...\n", _tid);
|
||||
LOG_DEBUG("Destroying causal sampler for thread {}...", _tid);
|
||||
_running = false;
|
||||
|
||||
if(_tid == threading::get_id() && !_signal_types.empty())
|
||||
@@ -390,7 +406,7 @@ configure(bool _setup, int64_t _tid)
|
||||
_causal_perf.reset();
|
||||
}
|
||||
|
||||
ROCPROFSYS_DEBUG("Causal sampler destroyed for thread %lu\n", _tid);
|
||||
LOG_DEBUG("Causal sampler destroyed for thread {}...", _tid);
|
||||
}
|
||||
|
||||
return _signal_types;
|
||||
@@ -570,8 +586,10 @@ post_process()
|
||||
{
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
|
||||
ROCPROFSYS_VERBOSE(2 || get_debug_sampling(),
|
||||
"Stopping causal sampling components...\n");
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Stopping causal sampling components...");
|
||||
}
|
||||
|
||||
block_samples();
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
#include "binary/dwarf_entry.hpp"
|
||||
#include "binary/symbol.hpp"
|
||||
#include "core/binary/fwd.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "library/causal/fwd.hpp"
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/state.hpp"
|
||||
|
||||
+21
-8
@@ -24,7 +24,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/trace_cache/cache_manager.hpp"
|
||||
#include "core/trace_cache/cacheable.hpp"
|
||||
@@ -61,6 +60,8 @@
|
||||
#include <timemory/utility/types.hpp>
|
||||
#include <timemory/variadic.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
@@ -194,7 +195,11 @@ apply_for_all_thread_names(int64_t _tid, std::function<void(const std::string&)>
|
||||
{
|
||||
std::string _desc = tim::papi::get_event_info(itr).short_descr;
|
||||
if(_desc.empty()) _desc = itr;
|
||||
ROCPROFSYS_CI_THROW(_desc.empty(), "Empty description for %s\n", itr.c_str());
|
||||
if(get_is_continuous_integration() && _desc.empty())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Empty description for {}", itr.c_str()));
|
||||
}
|
||||
|
||||
std::stringstream track_name_ss;
|
||||
track_name_ss << "Thread " << _desc << " [" << _tid << "] (S)";
|
||||
@@ -344,7 +349,7 @@ backtrace_metrics::configure(bool _setup, int64_t _tid)
|
||||
if constexpr(tim::trait::is_available<hw_counters>::value)
|
||||
{
|
||||
perfetto_counter_track<hw_counters>::init();
|
||||
ROCPROFSYS_DEBUG("HW COUNTER: starting...\n");
|
||||
LOG_DEBUG("HW COUNTER: starting...");
|
||||
if(get_papi_vector(_tid))
|
||||
{
|
||||
get_papi_vector(_tid)->start();
|
||||
@@ -354,7 +359,7 @@ backtrace_metrics::configure(bool _setup, int64_t _tid)
|
||||
}
|
||||
else if(!_setup && _is_running)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("Destroying sampler for thread %lu...\n", _tid);
|
||||
LOG_DEBUG("Destroying sampler for thread {}...", _tid);
|
||||
*_running = false;
|
||||
|
||||
if constexpr(tim::trait::is_available<hw_counters>::value)
|
||||
@@ -362,10 +367,10 @@ backtrace_metrics::configure(bool _setup, int64_t _tid)
|
||||
if(_tid == threading::get_id())
|
||||
{
|
||||
if(get_papi_vector(_tid)) get_papi_vector(_tid)->stop();
|
||||
ROCPROFSYS_DEBUG("HW COUNTER: stopped...\n");
|
||||
LOG_DEBUG("HW COUNTER: stopped...");
|
||||
}
|
||||
}
|
||||
ROCPROFSYS_DEBUG("Sampler destroyed for thread %lu\n", _tid);
|
||||
LOG_DEBUG("Sampler destroyed for thread {}...", _tid);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -399,7 +404,11 @@ backtrace_metrics::init_perfetto(int64_t _tid, valid_array_t _valid)
|
||||
{
|
||||
std::string _desc = tim::papi::get_event_info(itr).short_descr;
|
||||
if(_desc.empty()) _desc = itr;
|
||||
ROCPROFSYS_CI_THROW(_desc.empty(), "Empty description for %s\n", itr.c_str());
|
||||
if(get_is_continuous_integration() && _desc.empty())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Empty description for {}", itr.c_str()));
|
||||
}
|
||||
perfetto_counter_track<hw_counters>::emplace(
|
||||
_tid, JOIN(' ', "Thread", _desc, _tid_name, "(S)"));
|
||||
}
|
||||
@@ -412,7 +421,11 @@ backtrace_metrics::fini_perfetto(int64_t _tid, valid_array_t _valid)
|
||||
auto _hw_cnt_labels = *get_papi_labels(_tid);
|
||||
const auto& _thread_info = thread_info::get(_tid, SequentTID);
|
||||
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "Error! missing thread info for tid=%li\n", _tid);
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! missing thread info for tid={}", _tid));
|
||||
}
|
||||
if(!_thread_info) return;
|
||||
|
||||
uint64_t _ts = _thread_info->get_stop();
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "library/components/ensure_storage.hpp"
|
||||
|
||||
+22
-19
@@ -43,6 +43,8 @@
|
||||
#include <timemory/mpl/types.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
|
||||
@@ -267,11 +269,12 @@ category_region<CategoryT>::start(std::string_view name, Args&&... args)
|
||||
constexpr bool _ct_use_causal =
|
||||
(sizeof...(OptsT) == 0 || is_one_of<quirk::causal, type_list<OptsT...>>::value);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_PRINT(
|
||||
tracing::debug_push,
|
||||
"[%s][PID=%i][state=%s][thread_state=%s] rocprofsys_push_region(%s)\n",
|
||||
category_name, process::get_id(), std::to_string(get_state()).c_str(),
|
||||
std::to_string(get_thread_state()).c_str(), name.data());
|
||||
if(tracing::debug_push)
|
||||
{
|
||||
LOG_DEBUG("[{}][PID={}][state={}][thread_state={}] rocprofsys_push_region({})",
|
||||
category_name, process::get_id(), std::to_string(get_state()),
|
||||
std::to_string(get_thread_state()), name.data());
|
||||
}
|
||||
|
||||
if constexpr(is_one_of<CategoryT, tracing_count_categories_t>::value)
|
||||
{
|
||||
@@ -329,11 +332,12 @@ category_region<CategoryT>::stop(std::string_view name, Args&&... args)
|
||||
constexpr bool _ct_use_causal =
|
||||
(sizeof...(OptsT) == 0 || is_one_of<quirk::causal, type_list<OptsT...>>::value);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_PRINT(
|
||||
tracing::debug_pop,
|
||||
"[%s][PID=%i][state=%s][thread_state=%s] rocprofsys_pop_region(%s)\n",
|
||||
category_name, process::get_id(), std::to_string(get_state()).c_str(),
|
||||
std::to_string(get_thread_state()).c_str(), name.data());
|
||||
if(tracing::debug_pop)
|
||||
{
|
||||
LOG_DEBUG("[{}][PID={}][state={}][thread_state={}] rocprofsys_pop_region({})",
|
||||
category_name, process::get_id(), std::to_string(get_state()),
|
||||
std::to_string(get_thread_state()), name.data());
|
||||
}
|
||||
|
||||
// only execute when active
|
||||
if(get_state() == State::Active)
|
||||
@@ -376,10 +380,8 @@ category_region<CategoryT>::stop(std::string_view name, Args&&... args)
|
||||
}
|
||||
else
|
||||
{
|
||||
static auto _debug = get_debug_env();
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT(
|
||||
_debug, "[%s] rocprofsys_pop_region(%s) ignored :: state = %s\n",
|
||||
category_name, name.data(), std::to_string(get_state()).c_str());
|
||||
LOG_DEBUG("[{}] rocprofsys_pop_region({}) ignored :: state = {}", category_name,
|
||||
name.data(), std::to_string(get_state()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -408,11 +410,12 @@ category_region<CategoryT>::mark(std::string_view name, Args&&...)
|
||||
|
||||
if(get_use_causal())
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_PRINT(
|
||||
tracing::debug_mark,
|
||||
"[%s][PID=%i][state=%s][thread_state=%s] rocprofsys_progress(%s)\n",
|
||||
category_name, process::get_id(), std::to_string(get_state()).c_str(),
|
||||
std::to_string(get_thread_state()).c_str(), name.data());
|
||||
if(tracing::debug_mark)
|
||||
{
|
||||
LOG_DEBUG("[{}][PID={}][state={}][thread_state={}] rocprofsys_progress({})",
|
||||
category_name, process::get_id(), std::to_string(get_state()),
|
||||
std::to_string(get_thread_state()), name.data());
|
||||
}
|
||||
|
||||
causal::mark_progress_point(name);
|
||||
}
|
||||
|
||||
+22
-17
@@ -24,7 +24,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
@@ -36,6 +35,8 @@
|
||||
#include <timemory/utility/procfs/cpuinfo.hpp>
|
||||
#include <timemory/utility/type_list.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace cpuinfo = tim::procfs::cpuinfo;
|
||||
|
||||
namespace rocprofsys
|
||||
@@ -98,20 +99,21 @@ cpu_freq::configure()
|
||||
{
|
||||
if(_v.find_first_not_of("0123456789-") != std::string::npos)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
0,
|
||||
"Invalid CPU specification. Only numerical values (e.g., 0) or "
|
||||
"ranges (e.g., 0-7) are permitted. Ignoring %s...",
|
||||
_v.c_str());
|
||||
LOG_DEBUG("Invalid CPU specification. Only numerical values (e.g., 0) or "
|
||||
"ranges (e.g., 0-7) are permitted. Ignoring {}...",
|
||||
_v.c_str());
|
||||
continue;
|
||||
}
|
||||
if(_v.find('-') != std::string::npos)
|
||||
{
|
||||
auto _vv = tim::delimit(_v, "-");
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_vv.size() != 2,
|
||||
"Invalid CPU range specification: %s. Required format N-M, e.g. 0-4",
|
||||
_v.c_str());
|
||||
if(_vv.size() != 2)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Invalid CPU range specification: {}. Required "
|
||||
"format N-M, e.g. 0-4",
|
||||
_v));
|
||||
}
|
||||
for(size_t i = std::stoull(_vv.at(0)); i <= std::stoull(_vv.at(1)); ++i)
|
||||
_enabled_freqs.emplace(i);
|
||||
}
|
||||
@@ -137,21 +139,24 @@ cpu_freq::configure()
|
||||
_enabled_freqs.emplace(itr);
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0, "[cpu_freq::config] Warning! Removing invalid cpu %zu...\n", itr);
|
||||
LOG_DEBUG("[cpu_freq::config] Removing invalid cpu {}...", itr);
|
||||
}
|
||||
}
|
||||
|
||||
if(!cpuinfo::freq{})
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "[cpu_freq::config] Warning! CPU frequencies are disabled "
|
||||
":: unable to open /proc/cpuinfo");
|
||||
LOG_WARNING("[cpu_freq::config] CPU frequencies are disabled "
|
||||
":: unable to open /proc/cpuinfo");
|
||||
_enabled_freqs.clear();
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_FAIL(!cpuinfo::freq{},
|
||||
"[cpu_freq::config] CPU frequencies are disabled "
|
||||
":: unable to open /proc/cpuinfo");
|
||||
if(get_is_continuous_integration() && !cpuinfo::freq{})
|
||||
{
|
||||
LOG_CRITICAL("[cpu_freq::config] CPU frequencies are disabled :: unable to open "
|
||||
"/proc/cpuinfo");
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
get_enabled_cpus() = _enabled_freqs;
|
||||
}
|
||||
|
||||
+12
-42
@@ -23,7 +23,6 @@
|
||||
#include "library/components/exit_gotcha.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
|
||||
@@ -31,6 +30,8 @@
|
||||
#include <timemory/process/threading.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
@@ -63,37 +64,19 @@ invoke_exit_gotcha(const exit_gotcha::gotcha_data& _data, FuncT _func, Args... _
|
||||
|
||||
if(get_state() < State::Finalized && !is_child_process())
|
||||
{
|
||||
if(config::settings_are_configured())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "finalizing %s before calling %s(%s)...\n",
|
||||
get_exe_name().c_str(), _data.tool_id.c_str(),
|
||||
JOIN(", ", _args...).c_str());
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "finalizing %s before calling %s(%s)...\n",
|
||||
get_exe_name().c_str(), _data.tool_id.c_str(),
|
||||
JOIN(", ", _args...).c_str());
|
||||
}
|
||||
LOG_DEBUG("Finalizing {} before calling {}({})...", get_exe_name(), _data.tool_id,
|
||||
JOIN(", ", _args...).c_str());
|
||||
|
||||
rocprofsys_finalize();
|
||||
}
|
||||
|
||||
if(config::settings_are_configured())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "calling %s(%s) in %s...\n", _data.tool_id.c_str(),
|
||||
JOIN(", ", _args...).c_str(), get_exe_name().c_str());
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "calling %s(%s) in %s...\n", _data.tool_id.c_str(),
|
||||
JOIN(", ", _args...).c_str(), get_exe_name().c_str());
|
||||
}
|
||||
LOG_DEBUG("Calling {}({}) in {}...", _data.tool_id, JOIN(", ", _args...),
|
||||
get_exe_name());
|
||||
|
||||
if(_exit_info.is_known && _exit_info.exit_code != 0)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "%s exiting with non-zero exit code: %i...\n",
|
||||
get_exe_name().c_str(), _exit_info.exit_code);
|
||||
LOG_DEBUG("{} exiting with non-zero exit code: {}...", get_exe_name(),
|
||||
_exit_info.exit_code);
|
||||
}
|
||||
|
||||
(*_func)(_args...);
|
||||
@@ -113,27 +96,14 @@ exit_gotcha::operator()(const gotcha_data& _data, exit_func_t _func, int _ec) co
|
||||
|
||||
if(get_state() < ::rocprofsys::State::Finalized && !is_child_process())
|
||||
{
|
||||
if(config::settings_are_configured())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(0, "finalizing %s before calling %s(%i)...\n",
|
||||
get_exe_name().c_str(), _data.tool_id.c_str(), _ec);
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "finalizing %s before calling %s(%i)...\n",
|
||||
get_exe_name().c_str(), _data.tool_id.c_str(),
|
||||
_ec);
|
||||
}
|
||||
LOG_DEBUG("Finalizing {} before calling {}({})...", get_exe_name(),
|
||||
_data.tool_id, _ec);
|
||||
|
||||
rocprofsys_finalize();
|
||||
}
|
||||
|
||||
if(config::settings_are_configured())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
0, "calling _exit(%i) in %s to avoid AMD SMI cleanup issues...\n", _ec,
|
||||
get_exe_name().c_str());
|
||||
}
|
||||
LOG_DEBUG("Calling _exit({}) in {} to avoid AMD SMI cleanup issues...", _ec,
|
||||
get_exe_name().c_str());
|
||||
|
||||
std::fflush(nullptr);
|
||||
_exit(_ec);
|
||||
|
||||
+16
-15
@@ -23,7 +23,6 @@
|
||||
#include "api.hpp"
|
||||
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/perfetto_fwd.hpp"
|
||||
#include "core/state.hpp"
|
||||
@@ -36,6 +35,8 @@
|
||||
#include <timemory/mpl/types.hpp>
|
||||
#include <timemory/process/process.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <memory>
|
||||
#include <pthread.h>
|
||||
@@ -75,12 +76,11 @@ prefork_setup()
|
||||
tim::set_env("ROCPROFSYS_PRELOAD", "0", 1);
|
||||
tim::set_env("ROCPROFSYS_ROOT_PROCESS", process::get_id(), 0);
|
||||
rocprofsys_reset_preload_hidden();
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "fork() called on PID %i (rank: %i), TID %li\n",
|
||||
process::get_id(), dmp::rank(), threading::get_id());
|
||||
ROCPROFSYS_BASIC_DEBUG(
|
||||
"Warning! Calling fork() within an OpenMPI application using libfabric "
|
||||
"may result is segmentation fault\n");
|
||||
TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(get_debug_env(), 16);
|
||||
LOG_INFO("fork() called on PID {} (rank: {}), TID {}", process::get_id(), dmp::rank(),
|
||||
threading::get_id());
|
||||
LOG_WARNING("Calling fork() within an OpenMPI application using libfabric "
|
||||
"may result is segmentation fault");
|
||||
// TIMEMORY_CONDITIONAL_DEMANGLED_BACKTRACE(get_debug_env(), 16);
|
||||
|
||||
if(config::get_use_sampling()) sampling::block_samples();
|
||||
|
||||
@@ -116,9 +116,12 @@ postfork_child()
|
||||
{
|
||||
if(postfork_child_lock) return;
|
||||
|
||||
ROCPROFSYS_REQUIRE(is_child_process())
|
||||
<< "Error! child process " << process::get_id()
|
||||
<< " believes it is the root process " << get_root_process_id() << "\n";
|
||||
if(!is_child_process())
|
||||
{
|
||||
LOG_ERROR("Child process {} believes it is the root process {}",
|
||||
process::get_id(), get_root_process_id());
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
set_state(State::Finalized);
|
||||
|
||||
@@ -165,8 +168,7 @@ fork_gotcha::operator()(const gotcha_data_t&, pid_t (*_real_fork)()) const
|
||||
|
||||
if(_pid != 0)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "fork() called on PID %i created PID %i\n", getpid(),
|
||||
_pid);
|
||||
LOG_INFO("fork() called on PID {} created PID {}", getpid(), _pid);
|
||||
|
||||
postfork_parent();
|
||||
}
|
||||
@@ -177,9 +179,8 @@ fork_gotcha::operator()(const gotcha_data_t&, pid_t (*_real_fork)()) const
|
||||
|
||||
if(!settings::use_output_suffix())
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
0, "Application which make calls to fork() should enable using an process "
|
||||
"identifier output suffix (i.e. set ROCPROFSYS_USE_PID=ON)\n");
|
||||
LOG_DEBUG("Application which make calls to fork() should enable using an process "
|
||||
"identifier output suffix (i.e. set ROCPROFSYS_USE_PID=ON)");
|
||||
}
|
||||
|
||||
return _pid;
|
||||
|
||||
+5
-4
@@ -22,7 +22,8 @@
|
||||
|
||||
#include "kill_gotcha.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <unistd.h>
|
||||
@@ -57,9 +58,9 @@ kill_gotcha::operator()(const gotcha_data& _data, kill_func_t _func, pid_t _pid,
|
||||
|
||||
if(_sig == SIGKILL && _pid != _self_pid && _pid > 0)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("[kill_gotcha] Intercepted '%s(%d, SIGKILL)' triggered from "
|
||||
"process with id: %d. Sleeping for %d seconds...\n",
|
||||
_data.tool_id.c_str(), _pid, _self_pid, kill_delay);
|
||||
LOG_DEBUG("[kill_gotcha] Intercepted '{}({}, SIGKILL)' triggered from "
|
||||
"process with id: {}. Sleeping for {} seconds...",
|
||||
_data.tool_id, _pid, _self_pid, kill_delay);
|
||||
|
||||
::sleep(kill_delay);
|
||||
}
|
||||
|
||||
+20
-19
@@ -24,7 +24,6 @@
|
||||
#include "api.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/mpi.hpp"
|
||||
#include "core/mproc.hpp"
|
||||
#include "library/components/category_region.hpp"
|
||||
@@ -36,6 +35,8 @@
|
||||
#include <timemory/signals/signal_mask.hpp>
|
||||
#include <timemory/utility/locking.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <thread>
|
||||
@@ -75,8 +76,10 @@ struct comm_rank_data
|
||||
|
||||
friend bool operator>(const comm_rank_data& _lhs, const comm_rank_data& _rhs)
|
||||
{
|
||||
ROCPROFSYS_CI_THROW(!_lhs.updated() && !_rhs.updated(),
|
||||
"Error! comparing rank data that is not updated");
|
||||
if(get_is_continuous_integration() && !_lhs.updated() && !_rhs.updated())
|
||||
{
|
||||
throw std::runtime_error("Error! Comparing rank data that is not updated");
|
||||
}
|
||||
|
||||
if(_lhs.updated() && !_rhs.updated()) return true;
|
||||
if(!_lhs.updated() && _rhs.updated()) return false;
|
||||
@@ -112,7 +115,7 @@ rocprofsys_mpi_copy(MPI_Comm, int, void*, void*, void*, int*)
|
||||
int
|
||||
rocprofsys_mpi_fini(MPI_Comm, int, void*, void*)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("MPI Comm attribute finalize\n");
|
||||
LOG_DEBUG("MPI Comm attribute finalize");
|
||||
auto _blocked = get_sampling_signals();
|
||||
if(!_blocked.empty())
|
||||
tim::signals::block_signals(_blocked, tim::signals::sigmask_scope::process);
|
||||
@@ -222,9 +225,8 @@ mpi_gotcha::update()
|
||||
rocprofsys::mpi::set_size(_size);
|
||||
rocprofsys::settings::default_process_suffix() = _rank;
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE(0, "[pid=%i] MPI rank: %i (%i), MPI size: %i (%i)\n",
|
||||
process::get_id(), rocprofsys::mpi::rank(), _rank,
|
||||
rocprofsys::mpi::size(), _size);
|
||||
LOG_DEBUG("[pid={}] MPI rank: {} ({}), MPI size: {} ({})", process::get_id(),
|
||||
rocprofsys::mpi::rank(), _rank, rocprofsys::mpi::size(), _size);
|
||||
last_comm_record = _rank_data;
|
||||
config::get_use_pid() = true;
|
||||
return true;
|
||||
@@ -244,7 +246,7 @@ mpi_gotcha::disable_comm_intercept()
|
||||
void
|
||||
mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***)
|
||||
{
|
||||
ROCPROFSYS_BASIC_DEBUG_F("%s(int*, char***)\n", _data.tool_id.c_str());
|
||||
LOG_DEBUG("{}(int*, char***)", _data.tool_id);
|
||||
|
||||
rocprofsys_push_trace_hidden(_data.tool_id.c_str());
|
||||
#if !defined(ROCPROFSYS_USE_MPI) && defined(ROCPROFSYS_USE_MPI_HEADERS)
|
||||
@@ -256,7 +258,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***)
|
||||
void
|
||||
mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***, int, int*)
|
||||
{
|
||||
ROCPROFSYS_BASIC_DEBUG_F("%s(int*, char***, int, int*)\n", _data.tool_id.c_str());
|
||||
LOG_DEBUG("{}(int*, char***, int, int*)", _data.tool_id);
|
||||
|
||||
rocprofsys_push_trace_hidden(_data.tool_id.c_str());
|
||||
#if !defined(ROCPROFSYS_USE_MPI) && defined(ROCPROFSYS_USE_MPI_HEADERS)
|
||||
@@ -266,9 +268,9 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, int*, char***, in
|
||||
}
|
||||
|
||||
void
|
||||
mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
|
||||
mpi_gotcha::audit([[maybe_unused]] const gotcha_data_t& _data, audit::incoming)
|
||||
{
|
||||
ROCPROFSYS_BASIC_DEBUG_F("%s()\n", _data.tool_id.c_str());
|
||||
LOG_DEBUG("{}()", _data.tool_id);
|
||||
|
||||
auto _blocked = get_sampling_signals();
|
||||
if(!_blocked.empty())
|
||||
@@ -289,7 +291,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)
|
||||
void
|
||||
mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, comm_t _comm, int* _val)
|
||||
{
|
||||
ROCPROFSYS_BASIC_DEBUG_F("%s(comm_t _comm, int* _val)\n", _data.tool_id.c_str());
|
||||
LOG_DEBUG("{}(comm_t _comm, int* _val)", _data.tool_id);
|
||||
|
||||
rocprofsys_push_trace_hidden(_data.tool_id.c_str());
|
||||
if(_data.tool_id.find("MPI_Comm_rank") == 0 ||
|
||||
@@ -306,15 +308,15 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming, comm_t _comm, int
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_BASIC_PRINT_F("%s(<comm>, %p) :: unexpected function wrapper\n",
|
||||
_data.tool_id.c_str(), static_cast<void*>(_val));
|
||||
LOG_WARNING("{}(<comm>, {}) :: unexpected function wrapper", _data.tool_id,
|
||||
static_cast<void*>(_val));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
|
||||
{
|
||||
ROCPROFSYS_BASIC_DEBUG_F("%s() returned %i\n", _data.tool_id.c_str(), (int) _retval);
|
||||
LOG_DEBUG("{}() returned {}", _data.tool_id, (int) _retval);
|
||||
|
||||
if(!settings::use_output_suffix()) settings::use_output_suffix() = true;
|
||||
|
||||
@@ -329,7 +331,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
|
||||
// were excluded via a regex expression)
|
||||
if(get_use_mpip())
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(2, "Activating MPI wrappers...\n");
|
||||
LOG_DEBUG("Activating MPI wrappers...");
|
||||
|
||||
// use env vars ROCPROFSYS_MPIP_PERMIT_LIST and ROCPROFSYS_MPIP_REJECT_LIST
|
||||
// to control the gotcha bindings at runtime
|
||||
@@ -380,9 +382,8 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::outgoing, int _retval)
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
0, "%s() returned %i :: unexpected function wrapper\n",
|
||||
_data.tool_id.c_str(), (int) _retval);
|
||||
LOG_WARNING("{}() returned {} :: unexpected function wrapper",
|
||||
_data.tool_id, (int) _retval);
|
||||
}
|
||||
|
||||
if(_comm_entry.updated())
|
||||
|
||||
@@ -35,6 +35,8 @@
|
||||
#include <timemory/utility/demangle.hpp>
|
||||
#include <timemory/variadic/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include <set>
|
||||
#include <string>
|
||||
@@ -189,7 +191,7 @@ rocprofsys::component::activate_mpip()
|
||||
<< rocprofsys::utility::demangle<Tag>();
|
||||
return ss.str();
|
||||
}();
|
||||
ROCPROFSYS_BASIC_DEBUG_F("Adding cleanup for %s", _label.c_str());
|
||||
LOG_DEBUG("Adding cleanup for {}", _label);
|
||||
tim::manager::instance()->add_cleanup(_label, cleanup_functor);
|
||||
return 1;
|
||||
}
|
||||
@@ -214,7 +216,7 @@ rocprofsys::component::deactivate_mpip(uint64_t id)
|
||||
<< rocprofsys::utility::demangle<Tag>();
|
||||
return ss.str();
|
||||
}();
|
||||
ROCPROFSYS_BASIC_DEBUG_F("Removing cleanup for %s", _label.c_str());
|
||||
LOG_DEBUG("Removing cleanup for {}", _label);
|
||||
tim::manager::instance()->cleanup(_label);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include "library/components/numa_gotcha.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
#include "library/components/category_region.hpp"
|
||||
|
||||
+51
-62
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "library/components/pthread_create_gotcha.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/locking.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/utility.hpp"
|
||||
@@ -43,6 +42,8 @@
|
||||
#include <timemory/units.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <csignal>
|
||||
#include <dlfcn.h>
|
||||
#include <ostream>
|
||||
@@ -83,8 +84,7 @@ inline void
|
||||
start_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args)
|
||||
{
|
||||
if(!get_use_timemory() && !get_use_perfetto()) return;
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(3, "starting bundle '%s' in thread %li...\n",
|
||||
_bundle.key().c_str(), _tid);
|
||||
LOG_TRACE("Starting bundle '{}' in thread {}...", _bundle.key(), _tid);
|
||||
if constexpr(sizeof...(Args) > 0)
|
||||
{
|
||||
const char* _name = nullptr;
|
||||
@@ -117,8 +117,7 @@ stop_bundle(bundle_t& _bundle, int64_t _tid, Args&&... _args)
|
||||
_this_manager->is_finalized())
|
||||
return;
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(3, "stopping bundle '%s' in thread %li...\n",
|
||||
_bundle.key().c_str(), _tid);
|
||||
LOG_TRACE("Stopping bundle '{}' in thread {}...", _bundle.key(), _tid);
|
||||
if(get_use_timemory())
|
||||
{
|
||||
auto _wc = *_bundle.get<comp::wall_clock>();
|
||||
@@ -188,11 +187,9 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
{
|
||||
static std::once_flag thread_limit_warning_flag;
|
||||
std::call_once(thread_limit_warning_flag, []() {
|
||||
ROCPROFSYS_WARNING_F(
|
||||
1,
|
||||
"[rocprof-sys][WARNING] Maximum allowed thread limit (%zu) "
|
||||
"reached. Further thread creation and profiling will be "
|
||||
"disabled to prevent resource exhaustion.\n",
|
||||
LOG_WARNING(
|
||||
"Maximum allowed thread limit ({}) reached. Further thread creation and "
|
||||
"profiling will be disabled to prevent resource exhaustion.",
|
||||
static_cast<size_t>(ROCPROFSYS_MAX_THREADS));
|
||||
});
|
||||
return m_routine(m_arg);
|
||||
@@ -225,10 +222,9 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
_thr_bundle->stop();
|
||||
if(_bundle) stop_bundle(*_bundle, _tid);
|
||||
pthread_create_gotcha::shutdown(_tid);
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
1, "[PID=%i][rank=%i] Thread %s (parent: %s) exited\n", process::get_id(),
|
||||
dmp::rank(), _info->index_data->as_string().c_str(),
|
||||
_parent_info->index_data->as_string().c_str());
|
||||
LOG_DEBUG("[PID={}][rank={}] Thread {} (parent: {}) exited",
|
||||
process::get_id(), dmp::rank(), _info->index_data->as_string(),
|
||||
_parent_info->index_data->as_string());
|
||||
}
|
||||
};
|
||||
|
||||
@@ -244,10 +240,9 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
if(_active && !_coverage && !m_config.offset)
|
||||
{
|
||||
_tid = _info->index_data->sequent_value;
|
||||
ROCPROFSYS_BASIC_VERBOSE(1, "[PID=%i][rank=%i] Thread %s (parent: %s) created\n",
|
||||
process::get_id(), dmp::rank(),
|
||||
_info->index_data->as_string().c_str(),
|
||||
_parent_info->index_data->as_string().c_str());
|
||||
LOG_DEBUG("[PID={}][rank={}] Thread {} (parent: {}) created", process::get_id(),
|
||||
dmp::rank(), _info->index_data->as_string(),
|
||||
_parent_info->index_data->as_string());
|
||||
threading::set_thread_name(TIMEMORY_JOIN(" ", "Thread", _tid).c_str());
|
||||
auto _manager = tim::manager::instance();
|
||||
if(_manager) _manager->initialize();
|
||||
@@ -288,11 +283,10 @@ pthread_create_gotcha::wrapper::operator()() const
|
||||
}
|
||||
else if(m_config.offset)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
2,
|
||||
"[PID=%i][rank=%i] Thread %s (parent: %s) created [started by rocprof-sys]\n",
|
||||
process::get_id(), dmp::rank(), _info->index_data->as_string().c_str(),
|
||||
_parent_info->index_data->as_string().c_str());
|
||||
LOG_DEBUG(
|
||||
"[PID={}][rank={}] Thread {} (parent: {}) created [started by rocprof-sys]",
|
||||
process::get_id(), dmp::rank(), _info->index_data->as_string(),
|
||||
_parent_info->index_data->as_string());
|
||||
}
|
||||
|
||||
// notify the wrapper that all internal work is completed
|
||||
@@ -466,11 +460,15 @@ pthread_create_gotcha::shutdown()
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds{ 50 });
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_BASIC_FAIL(
|
||||
shutdown_signals_delivered != _expected_shutdown_signals_delivered,
|
||||
"Number of signals delivered (%zu) != expected number of signals delievered "
|
||||
"(%zu)",
|
||||
shutdown_signals_delivered, _expected_shutdown_signals_delivered);
|
||||
if(shutdown_signals_delivered != _expected_shutdown_signals_delivered)
|
||||
{
|
||||
LOG_CRITICAL("Number of signals delivered ({}) != expected number of signals "
|
||||
"delievered "
|
||||
"({})",
|
||||
shutdown_signals_delivered,
|
||||
_expected_shutdown_signals_delivered);
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// restore existing signal handler
|
||||
@@ -493,17 +491,9 @@ pthread_create_gotcha::shutdown()
|
||||
|
||||
bundles->clear();
|
||||
|
||||
if(config::settings_are_configured())
|
||||
if(_ndangling > 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2 && _ndangling > 0,
|
||||
"[pthread_create_gotcha] cleaned up %lu dangling bundles\n",
|
||||
_ndangling);
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(
|
||||
2 && _ndangling > 0,
|
||||
"[pthread_create_gotcha] cleaned up %lu dangling bundles\n", _ndangling);
|
||||
LOG_DEBUG("Cleaned up {} dangling bundles", _ndangling);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -552,8 +542,7 @@ is_rocm_internal_thread(void* func_ptr)
|
||||
Dl_info info;
|
||||
if(dladdr(func_ptr, &info) == 0 || info.dli_fname == nullptr)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(4, "dladdr failed or returned no filename for func_ptr=%p\n",
|
||||
func_ptr);
|
||||
LOG_TRACE("dladdr failed or returned no filename for func_ptr={:p}", func_ptr);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -606,39 +595,39 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
|
||||
static bool debug_threading_get_id =
|
||||
get_env<bool>(TIMEMORY_SETTINGS_PREFIX "DEBUG_THREADING_GET_ID", false);
|
||||
|
||||
auto _verbose = (debug_threading_get_id) ? 0 : 3;
|
||||
ROCPROFSYS_VERBOSE(
|
||||
_verbose,
|
||||
"Creating new thread :: global_state=%s, thread_state=%s, mode=%s, active=%s, "
|
||||
"coverage=%s, use_causal=%s, use_sampling=%s, sample_children=%s, tid=%li, "
|
||||
"use_bundle=%s, enable_causal=%s, enable_sampling=%s, thread_info=(%s)...\n",
|
||||
std::to_string(_glob_state).c_str(), std::to_string(_thr_state).c_str(),
|
||||
std::to_string(_mode).c_str(), std::to_string(_active).c_str(),
|
||||
std::to_string(_coverage).c_str(), std::to_string(_use_causal).c_str(),
|
||||
std::to_string(_use_sampling).c_str(), std::to_string(_sample_child).c_str(),
|
||||
_tid, std::to_string(_use_bundle).c_str(), std::to_string(_enable_causal).c_str(),
|
||||
std::to_string(_enable_sampling).c_str(), JOIN("", *_info).c_str());
|
||||
|
||||
if(debug_threading_get_id)
|
||||
{
|
||||
timemory_print_demangled_backtrace<8>(std::cerr, std::string{},
|
||||
LOG_TRACE(
|
||||
"Creating new thread :: global_state={}, thread_state={}, mode={}, "
|
||||
"active={}, "
|
||||
"coverage={}, use_causal={}, use_sampling={}, sample_children={}, tid={}, "
|
||||
"use_bundle={}, enable_causal={}, enable_sampling={}, thread_info={}...",
|
||||
std::to_string(_glob_state), std::to_string(_thr_state),
|
||||
std::to_string(_mode), std::to_string(_active), std::to_string(_coverage),
|
||||
std::to_string(_use_causal), std::to_string(_use_sampling),
|
||||
std::to_string(_sample_child), std::to_string(_tid),
|
||||
std::to_string(_use_bundle), std::to_string(_enable_causal),
|
||||
std::to_string(_enable_sampling), JOIN("", *_info));
|
||||
|
||||
std::stringstream _backtrace_ss;
|
||||
timemory_print_demangled_backtrace<8>(_backtrace_ss, std::string{},
|
||||
std::string{ "threading::get_id() [id=" } +
|
||||
std::to_string(_tid) +
|
||||
std::string{ "]" },
|
||||
std::string{ " " }, false);
|
||||
LOG_TRACE("Backtrace: {}", _backtrace_ss.str());
|
||||
}
|
||||
|
||||
if(_active && !_disabled && !_info->is_offset)
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE(2, "[PID=%i][rank=%i] Starting new thread on %s...\n",
|
||||
process::get_id(), dmp::rank(),
|
||||
_info->index_data->as_string().c_str());
|
||||
LOG_DEBUG("[PID={}][rank={}] Starting new thread on {}", process::get_id(),
|
||||
dmp::rank(), _info->index_data->as_string().c_str());
|
||||
}
|
||||
|
||||
// ensure that cpu cid stack exists on the parent thread if active
|
||||
if(_active && !_coverage)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("blocking signals...\n");
|
||||
LOG_DEBUG("Locking signals...");
|
||||
get_cpu_cid_stack();
|
||||
}
|
||||
|
||||
@@ -653,7 +642,7 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
|
||||
// block the signals in entire process
|
||||
if(_enable_sampling && !_blocked.empty())
|
||||
{
|
||||
ROCPROFSYS_DEBUG("blocking signals...\n");
|
||||
LOG_DEBUG("Blocking signals...");
|
||||
tim::signals::block_signals(_blocked, tim::signals::sigmask_scope::process);
|
||||
}
|
||||
|
||||
@@ -673,7 +662,7 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
|
||||
// wait for thread to set promise
|
||||
if(_promise)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("waiting for child to signal it is setup...\n");
|
||||
LOG_DEBUG("Waiting for child to signal it is setup...");
|
||||
_promise->get_future().wait_for(std::chrono::milliseconds{ 500 });
|
||||
}
|
||||
|
||||
@@ -683,11 +672,11 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
|
||||
// unblock the signals in the entire process
|
||||
if(_enable_sampling && !_blocked.empty())
|
||||
{
|
||||
ROCPROFSYS_DEBUG("unblocking signals...\n");
|
||||
LOG_DEBUG("Unblocking signals...");
|
||||
tim::signals::unblock_signals(_blocked, tim::signals::sigmask_scope::process);
|
||||
}
|
||||
|
||||
ROCPROFSYS_DEBUG("returning success...\n");
|
||||
LOG_DEBUG("Returning success...");
|
||||
return _ret;
|
||||
}
|
||||
} // namespace component
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "library/components/pthread_gotcha.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/utility.hpp"
|
||||
#include "library/components/pthread_create_gotcha.hpp"
|
||||
#include "library/components/pthread_mutex_gotcha.hpp"
|
||||
|
||||
+12
-9
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "library/components/pthread_mutex_gotcha.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/utility.hpp"
|
||||
#include "library/components/category_region.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
@@ -32,6 +31,8 @@
|
||||
#include <timemory/utility/signals.hpp>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <pthread.h>
|
||||
#include <stdexcept>
|
||||
@@ -76,14 +77,16 @@ pthread_mutex_gotcha::get_hashes()
|
||||
else
|
||||
{
|
||||
if(_skip.count(i) > 0) continue;
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1,
|
||||
"WARNING!!! pthread_mutex_gotcha tool id at index %zu was empty!\n",
|
||||
i);
|
||||
LOG_WARNING("pthread_mutex_gotcha tool id at index {} was empty!", i);
|
||||
}
|
||||
|
||||
if(get_is_continuous_integration() && (_id.empty() || _init.at(i) == 0))
|
||||
{
|
||||
LOG_CRITICAL("pthread_mutex_gotcha tool id at index {} has no hash value",
|
||||
i);
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
ROCPROFSYS_CI_FAIL(
|
||||
_id.empty() || _init.at(i) == 0,
|
||||
"pthread_mutex_gotcha tool id at index %zu has no hash value\n", i);
|
||||
}
|
||||
return _init;
|
||||
}();
|
||||
@@ -183,7 +186,7 @@ pthread_mutex_gotcha::operator()(uintptr_t&&, int (*_callee)(Args...),
|
||||
{
|
||||
if(m_data)
|
||||
{
|
||||
ROCPROFSYS_PRINT("Warning! nullptr to %s\n", m_data->tool_id.c_str());
|
||||
LOG_WARNING("nullptr to {}", m_data->tool_id);
|
||||
}
|
||||
return EINVAL;
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include "library/components/vaapi_gotcha.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
#include "library/components/category_region.hpp"
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include "library/coverage.hpp"
|
||||
#include "api.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "library/coverage/impl.hpp"
|
||||
#include "library/thread_data.hpp"
|
||||
|
||||
@@ -31,6 +30,8 @@
|
||||
#include <timemory/tpls/cereal/cereal.hpp>
|
||||
#include <timemory/utility/popen.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
@@ -108,9 +109,7 @@ post_process()
|
||||
|
||||
if(_coverage.size == 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
0,
|
||||
"Warning! Code coverage enabled but no code coverage data is available!\n");
|
||||
LOG_WARNING("Code coverage enabled but no code coverage data is available!");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -150,11 +149,9 @@ post_process()
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(0,
|
||||
"Warning! No matching coverage data for "
|
||||
"%s :: %s (0x%x)\n",
|
||||
func.first.data(), file.first.data(),
|
||||
(unsigned int) addr.first);
|
||||
LOG_WARNING("No matching coverage data for {} :: {} (0x{:X})",
|
||||
func.first, file.first,
|
||||
(unsigned int) addr.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -208,21 +205,20 @@ post_process()
|
||||
std::swap(_coverage_data, _tmp);
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(0, "code coverage :: %6.2f%s\n", _coverage() * 100.0, "%");
|
||||
ROCPROFSYS_VERBOSE(0, "module coverage :: %6.2f%s\n",
|
||||
_coverage(code_coverage::MODULE) * 100.0, "%");
|
||||
ROCPROFSYS_VERBOSE(0, "function coverage :: %6.2f%s\n",
|
||||
_coverage(code_coverage::FUNCTION) * 100.0, "%");
|
||||
|
||||
if(get_verbose() >= 0) fprintf(stderr, "\n");
|
||||
LOG_INFO("code coverage :: {:.2f}%", _coverage() * 100.0);
|
||||
LOG_INFO("module coverage :: {:.2f}%", _coverage(code_coverage::MODULE) * 100.0);
|
||||
LOG_INFO("function coverage :: {:.2f}%", _coverage(code_coverage::FUNCTION) * 100.0);
|
||||
|
||||
std::sort(_coverage_data.begin(), _coverage_data.end(),
|
||||
std::greater<coverage_data>{});
|
||||
|
||||
auto _get_setting = [](const std::string& _v) {
|
||||
auto&& _b = config::get_setting_value<bool>(_v);
|
||||
ROCPROFSYS_CI_THROW(!_b, "Error! No configuration setting named '%s'",
|
||||
_v.c_str());
|
||||
if(!_b && get_is_continuous_integration())
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! No configuration setting named '{}'", _v));
|
||||
}
|
||||
return _b.value_or(true);
|
||||
};
|
||||
|
||||
@@ -255,7 +251,8 @@ post_process()
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("Error opening coverage output file: %s", _fname.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error opening coverage output file: {}", _fname));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -287,11 +284,10 @@ post_process()
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("Error opening coverage output file: %s", _fname.c_str());
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error opening coverage output file: {}", _fname));
|
||||
}
|
||||
}
|
||||
|
||||
if(get_verbose() >= 0) fprintf(stderr, "\n");
|
||||
}
|
||||
} // namespace coverage
|
||||
} // namespace rocprofsys
|
||||
@@ -308,8 +304,8 @@ rocprofsys_register_source_hidden(const char* file, const char* func, size_t lin
|
||||
|
||||
using coverage_data = coverage::coverage_data;
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(4, "[0x%x] :: %-20s :: %20s:%zu :: %s\n",
|
||||
(unsigned int) address, func, file, line, source);
|
||||
LOG_DEBUG("[0x{:X}] :: {:20s} :: {:20s}:{} :: {}", (unsigned int) address, func, file,
|
||||
line, source);
|
||||
|
||||
coverage::get_coverage_data().emplace_back(
|
||||
coverage_data{ size_t{ 0 }, address, line, file, func,
|
||||
@@ -339,8 +335,6 @@ rocprofsys_register_coverage_hidden(const char* file, const char* func, size_t a
|
||||
else if(rocprofsys::get_state() >= rocprofsys::State::Finalized)
|
||||
return;
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(3, "[0x%x] %-20s :: %20s\n", (unsigned int) address, func,
|
||||
file);
|
||||
(*coverage::get_coverage_count())[file][func][address] += 1;
|
||||
}
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
#include "core/agent_manager.hpp"
|
||||
#include "core/common.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/node_info.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
@@ -41,6 +40,8 @@
|
||||
#include <timemory/utility/procfs/cpuinfo.hpp>
|
||||
#include <timemory/utility/type_list.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
@@ -332,9 +333,8 @@ write_perfetto_counter_track(index&& _idx, Args... _args)
|
||||
void
|
||||
post_process()
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(1,
|
||||
"Post-processing %zu cpu frequency and memory usage entries...\n",
|
||||
data.size());
|
||||
LOG_DEBUG("Post-processing {} cpu frequency and memory usage entries...",
|
||||
data.size());
|
||||
|
||||
auto& enabled_cpus = component::cpu_freq::get_enabled_cpus();
|
||||
|
||||
@@ -342,7 +342,10 @@ post_process()
|
||||
using freq_track = perfetto_counter_track<category::cpu_freq>;
|
||||
|
||||
const auto& _thread_info = thread_info::get(0, InternalTID);
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "Missing thread info for thread 0");
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error("Missing thread info for thread 0");
|
||||
}
|
||||
if(!_thread_info) return;
|
||||
|
||||
if(!freq_track::exists(_idx))
|
||||
@@ -379,7 +382,10 @@ post_process()
|
||||
}
|
||||
|
||||
const auto& _thread_info = thread_info::get(0, InternalTID);
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "Missing thread info for thread 0");
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error("Missing thread info for thread 0");
|
||||
}
|
||||
if(!_thread_info) return;
|
||||
|
||||
for(auto& itr : data)
|
||||
|
||||
@@ -27,7 +27,6 @@
|
||||
#include "core/agent_manager.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/node_info.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
@@ -47,6 +46,8 @@
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
@@ -226,7 +227,7 @@ extern "C"
|
||||
{
|
||||
_standalone_initialized = true;
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(0, "Parsing arguments...\n");
|
||||
LOG_DEBUG("Parsing arguments...");
|
||||
std::string _command_line = {};
|
||||
for(int i = 0; i < argc; ++i)
|
||||
{
|
||||
@@ -250,16 +251,16 @@ extern "C"
|
||||
if(_version > 0) _settings->requires_global_fencing = false;
|
||||
}
|
||||
|
||||
void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
|
||||
void kokkosp_init_library([[maybe_unused]] const int loadSeq,
|
||||
[[maybe_unused]] const uint64_t interfaceVer,
|
||||
const uint32_t devInfoCount, void* deviceInfo)
|
||||
{
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
tim::consume_parameters(devInfoCount, deviceInfo);
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(
|
||||
0,
|
||||
"Initializing rocprof-sys kokkos connector (sequence %d, version: %llu)... ",
|
||||
loadSeq, (unsigned long long) interfaceVer);
|
||||
LOG_DEBUG(
|
||||
"Initializing rocprof-sys kokkos connector (sequence {}, version: {})...",
|
||||
loadSeq, interfaceVer);
|
||||
|
||||
if(_standalone_initialized ||
|
||||
(!rocprofsys::config::settings_are_configured() &&
|
||||
@@ -283,20 +284,22 @@ extern "C"
|
||||
{
|
||||
std::stringstream _libs_str{};
|
||||
for(const auto& litr : _libs)
|
||||
_libs_str << " " << litr << "\n";
|
||||
ROCPROFSYS_ABORT(
|
||||
"%s was invoked with librocprof-sys.so as the "
|
||||
"KOKKOS_TOOLS_LIBS.\n"
|
||||
_libs_str << "- " << litr << "\n";
|
||||
LOG_CRITICAL(
|
||||
"{} was invoked with librocprof-sys.so as the "
|
||||
"KOKKOS_TOOLS_LIBS."
|
||||
"However, librocprof-sys-dl.so has already been loaded by "
|
||||
"the process.\nTo avoid duplicate collections culminating "
|
||||
"is an error, please set KOKKOS_TOOLS_LIBS=%s.\nLoaded "
|
||||
"libraries:\n%s",
|
||||
__FUNCTION__, itr.c_str(), _libs_str.str().c_str());
|
||||
"the process. To avoid duplicate collections culminating "
|
||||
"is an error, please set KOKKOS_TOOLS_LIBS={}.Loaded "
|
||||
"libraries: {}",
|
||||
__FUNCTION__, itr, _libs_str.str());
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(0, "Initializing rocprof-sys (standalone)... ");
|
||||
LOG_DEBUG("Initializing rocprof-sys (standalone)... ");
|
||||
auto _mode = tim::get_env<std::string>("ROCPROFSYS_MODE", "trace");
|
||||
auto _arg0 = (_initialize_arguments.empty()) ? std::string{ "unknown" }
|
||||
: _initialize_arguments.at(0);
|
||||
@@ -315,11 +318,7 @@ extern "C"
|
||||
tim::trait::runtime_enabled<kokkosp::memory_tracker>::set(
|
||||
rocprofsys::config::get_use_timemory());
|
||||
|
||||
if(rocprofsys::get_verbose() >= 0)
|
||||
{
|
||||
fprintf(stderr, "%sDone\n%s", tim::log::color::info(),
|
||||
tim::log::color::end());
|
||||
}
|
||||
LOG_DEBUG("Done");
|
||||
|
||||
_name_len_limit = rocprofsys::config::get_setting_value<int64_t>(
|
||||
"ROCPROFSYS_KOKKOSP_NAME_LENGTH_MAX")
|
||||
@@ -339,15 +338,13 @@ extern "C"
|
||||
if(_standalone_initialized)
|
||||
{
|
||||
rocprofsys_pop_trace_hidden("kokkos_main");
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
0, "Finalizing kokkos rocprof-sys connector (standalone)...\n");
|
||||
LOG_DEBUG("Finalizing kokkos rocprof-sys connector (standalone)...");
|
||||
rocprofsys_finalize_hidden();
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(0, "Finalizing kokkos rocprof-sys connector... ");
|
||||
LOG_DEBUG("Finalizing kokkos rocprof-sys connector... ");
|
||||
kokkosp::cleanup();
|
||||
if(rocprofsys::get_verbose() >= 0) fprintf(stderr, "Done\n");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "library/perf.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/locking.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
@@ -32,6 +31,8 @@
|
||||
#include <timemory/log/macros.hpp>
|
||||
#include <timemory/units.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <asm/unistd.h>
|
||||
#include <ctime>
|
||||
#include <fcntl.h>
|
||||
@@ -59,14 +60,6 @@
|
||||
}
|
||||
#endif
|
||||
|
||||
#if !defined(ROCPROFSYS_FATAL)
|
||||
# define ROCPROFSYS_FATAL TIMEMORY_FATAL
|
||||
#endif
|
||||
|
||||
#if !defined(ROCPROFSYS_ASSERT)
|
||||
# define ROCPROFSYS_ASSERT(COND) (COND) ? ::tim::log::base() : TIMEMORY_FATAL
|
||||
#endif
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace perf
|
||||
@@ -98,7 +91,7 @@ perf_event::perf_event(perf_event&& rhs) noexcept
|
||||
if(m_fd != -1 && m_fd != rhs.m_fd)
|
||||
{
|
||||
::close(m_fd);
|
||||
ROCPROFSYS_VERBOSE(1, "Closed perf event fd %li\n", m_fd);
|
||||
LOG_DEBUG("Closed perf event fd {}", m_fd);
|
||||
}
|
||||
|
||||
if(m_mapping != nullptr && m_mapping != rhs.m_mapping) munmap(m_mapping, sizes.mmap);
|
||||
@@ -247,8 +240,11 @@ uint64_t
|
||||
perf_event::get_count() const
|
||||
{
|
||||
uint64_t count;
|
||||
ROCPROFSYS_REQUIRE(read(m_fd, &count, sizeof(uint64_t)) == sizeof(uint64_t))
|
||||
<< "Failed to read event count from perf_event file";
|
||||
if(read(m_fd, &count, sizeof(uint64_t)) != sizeof(uint64_t))
|
||||
{
|
||||
LOG_CRITICAL("Failed to read event count from perf_event file");
|
||||
std::exit(1);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
@@ -259,8 +255,11 @@ perf_event::start() const
|
||||
if(m_fd != -1)
|
||||
{
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
ROCPROFSYS_REQUIRE(ioctl(m_fd, PERF_EVENT_IOC_ENABLE, 0) != -1)
|
||||
<< "Failed to start perf event: " << strerror(errno);
|
||||
if(ioctl(m_fd, PERF_EVENT_IOC_ENABLE, 0) == -1)
|
||||
{
|
||||
LOG_CRITICAL("Failed to start perf event: {}", strerror(errno));
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
return (m_fd != -1);
|
||||
}
|
||||
@@ -272,8 +271,11 @@ perf_event::stop() const
|
||||
if(m_fd != -1)
|
||||
{
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
ROCPROFSYS_REQUIRE(ioctl(m_fd, PERF_EVENT_IOC_DISABLE, 0) != -1)
|
||||
<< "Failed to stop perf event: " << strerror(errno) << " (" << m_fd << ")";
|
||||
if(ioctl(m_fd, PERF_EVENT_IOC_DISABLE, 0) == -1)
|
||||
{
|
||||
LOG_CRITICAL("Failed to stop perf event: {}", strerror(errno));
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
return (m_fd != -1);
|
||||
}
|
||||
@@ -308,16 +310,26 @@ perf_event::set_ready_signal(int sig) const
|
||||
{
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
// Set the perf_event file to async
|
||||
ROCPROFSYS_REQUIRE(fcntl(m_fd, F_SETFL, fcntl(m_fd, F_GETFL, 0) | O_ASYNC) != -1)
|
||||
<< "failed to set perf_event file to async mode";
|
||||
if(fcntl(m_fd, F_SETFL, fcntl(m_fd, F_GETFL, 0) | O_ASYNC) == -1)
|
||||
{
|
||||
LOG_CRITICAL("Failed to set perf_event file to async mode: {}", strerror(errno));
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
// Set the notification signal for the perf file
|
||||
ROCPROFSYS_REQUIRE(fcntl(m_fd, F_SETSIG, sig) != -1)
|
||||
<< "failed to set perf_event file signal";
|
||||
if(fcntl(m_fd, F_SETSIG, sig) == -1)
|
||||
{
|
||||
LOG_CRITICAL("Failed to set perf_event file signal: {}", strerror(errno));
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
// Set the current thread as the owner of the file (to target signal delivery)
|
||||
ROCPROFSYS_REQUIRE(fcntl(m_fd, F_SETOWN, gettid()) != -1)
|
||||
<< "failed to set the owner of the perf_event file";
|
||||
if(fcntl(m_fd, F_SETOWN, gettid()) == -1)
|
||||
{
|
||||
LOG_CRITICAL("Failed to set the owner of the perf_event file: {}",
|
||||
strerror(errno));
|
||||
std::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
@@ -448,69 +460,84 @@ perf_event::copy_from_ring_buffer(struct perf_event_mmap_page* _mapping, ptrdiff
|
||||
uint64_t
|
||||
perf_event::record::get_ip() const
|
||||
{
|
||||
ROCPROFSYS_ASSERT(is_sample() && m_source != nullptr &&
|
||||
m_source->is_sampling(sample::ip))
|
||||
<< "Record does not have an ip field (" << is_sample() << "|" << m_source << ")";
|
||||
if(!is_sample() || m_source == nullptr || !m_source->is_sampling(sample::ip))
|
||||
{
|
||||
LOG_CRITICAL("Record does not have an ip field ({}|{:p})", is_sample(),
|
||||
static_cast<const void*>(m_source));
|
||||
std::abort();
|
||||
}
|
||||
return *locate_field<sample::ip, uint64_t*>();
|
||||
}
|
||||
|
||||
uint64_t
|
||||
perf_event::record::get_pid() const
|
||||
{
|
||||
ROCPROFSYS_ASSERT(is_sample() && m_source != nullptr &&
|
||||
m_source->is_sampling(sample::pid_tid))
|
||||
<< "Record does not have a `pid` field (" << is_sample() << "|" << m_source
|
||||
<< ")";
|
||||
if(!is_sample() || m_source == nullptr || !m_source->is_sampling(sample::pid_tid))
|
||||
{
|
||||
LOG_CRITICAL("Record does not have a `pid` field ({}|{:p})", is_sample(),
|
||||
static_cast<const void*>(m_source));
|
||||
std::abort();
|
||||
}
|
||||
return locate_field<sample::pid_tid, uint32_t*>()[0];
|
||||
}
|
||||
|
||||
uint64_t
|
||||
perf_event::record::get_tid() const
|
||||
{
|
||||
ROCPROFSYS_ASSERT(is_sample() && m_source != nullptr &&
|
||||
m_source->is_sampling(sample::pid_tid))
|
||||
<< "Record does not have a `tid` field (" << is_sample() << "|" << m_source
|
||||
<< ")";
|
||||
if(!is_sample() || m_source == nullptr || !m_source->is_sampling(sample::pid_tid))
|
||||
{
|
||||
LOG_CRITICAL("Record does not have a `tid` field ({}|{:p})", is_sample(),
|
||||
static_cast<const void*>(m_source));
|
||||
std::abort();
|
||||
}
|
||||
return locate_field<sample::pid_tid, uint32_t*>()[1];
|
||||
}
|
||||
|
||||
uint64_t
|
||||
perf_event::record::get_time() const
|
||||
{
|
||||
ROCPROFSYS_ASSERT(is_sample() && m_source != nullptr &&
|
||||
m_source->is_sampling(sample::time))
|
||||
<< "Record does not have a 'time' field (" << is_sample() << "|" << m_source
|
||||
<< ")";
|
||||
if(!is_sample() || m_source == nullptr || !m_source->is_sampling(sample::time))
|
||||
{
|
||||
LOG_CRITICAL("Record does not have a 'time' field ({}|{:p})", is_sample(),
|
||||
static_cast<const void*>(m_source));
|
||||
std::abort();
|
||||
}
|
||||
return *locate_field<sample::time, uint64_t*>();
|
||||
}
|
||||
|
||||
uint64_t
|
||||
perf_event::record::get_period() const
|
||||
{
|
||||
ROCPROFSYS_ASSERT(is_sample() && m_source != nullptr &&
|
||||
m_source->is_sampling(sample::period))
|
||||
<< "Record does not have a 'period' field (" << is_sample() << "|" << m_source
|
||||
<< ")";
|
||||
if(!is_sample() || m_source == nullptr || !m_source->is_sampling(sample::period))
|
||||
{
|
||||
LOG_CRITICAL("Record does not have a 'period' field ({}|{:p})", is_sample(),
|
||||
static_cast<const void*>(m_source));
|
||||
std::abort();
|
||||
}
|
||||
return *locate_field<sample::period, uint64_t*>();
|
||||
}
|
||||
|
||||
uint32_t
|
||||
perf_event::record::get_cpu() const
|
||||
{
|
||||
ROCPROFSYS_ASSERT(is_sample() && m_source != nullptr &&
|
||||
m_source->is_sampling(sample::cpu))
|
||||
<< "Record does not have a 'cpu' field (" << is_sample() << "|" << m_source
|
||||
<< ")";
|
||||
if(!is_sample() || m_source == nullptr || !m_source->is_sampling(sample::cpu))
|
||||
{
|
||||
LOG_CRITICAL("Record does not have a 'cpu' field ({}|{:p})", is_sample(),
|
||||
static_cast<const void*>(m_source));
|
||||
std::abort();
|
||||
}
|
||||
return *locate_field<sample::cpu, uint32_t*>();
|
||||
}
|
||||
|
||||
container::c_array<uint64_t>
|
||||
perf_event::record::get_callchain() const
|
||||
{
|
||||
ROCPROFSYS_ASSERT(is_sample() && m_source != nullptr &&
|
||||
m_source->is_sampling(sample::callchain))
|
||||
<< "Record does not have a callchain field (" << is_sample() << "|" << m_source
|
||||
<< ")";
|
||||
if(!is_sample() || m_source == nullptr || !m_source->is_sampling(sample::callchain))
|
||||
{
|
||||
LOG_CRITICAL("Record does not have a callchain field ({}|{:p})", is_sample(),
|
||||
static_cast<const void*>(m_source));
|
||||
std::abort();
|
||||
}
|
||||
|
||||
uint64_t* _base = locate_field<sample::callchain, uint64_t*>();
|
||||
uint64_t _size = *_base;
|
||||
@@ -617,22 +644,31 @@ perf_event::record::locate_field() const
|
||||
// branch_stack
|
||||
if constexpr(SampleT == sample::branch_stack) return reinterpret_cast<Tp>(p);
|
||||
if(m_source != nullptr && m_source->is_sampling(sample::branch_stack))
|
||||
ROCPROFSYS_FATAL << "Branch stack sampling is not supported";
|
||||
|
||||
{
|
||||
LOG_CRITICAL("Branch stack sampling is not supported");
|
||||
std::abort();
|
||||
}
|
||||
// regs
|
||||
if constexpr(SampleT == sample::regs) return reinterpret_cast<Tp>(p);
|
||||
if(m_source != nullptr && m_source->is_sampling(sample::regs))
|
||||
ROCPROFSYS_FATAL << "Register sampling is not supported";
|
||||
{
|
||||
LOG_CRITICAL("Register sampling is not supported");
|
||||
std::abort();
|
||||
}
|
||||
|
||||
// stack
|
||||
if constexpr(SampleT == sample::stack) return reinterpret_cast<Tp>(p);
|
||||
if(m_source != nullptr && m_source->is_sampling(sample::stack))
|
||||
ROCPROFSYS_FATAL << "Stack sampling is not supported";
|
||||
{
|
||||
LOG_CRITICAL("Stack sampling is not supported");
|
||||
std::abort();
|
||||
}
|
||||
|
||||
// end
|
||||
if constexpr(SampleT == sample::last) return reinterpret_cast<Tp>(p);
|
||||
|
||||
ROCPROFSYS_FATAL << "Unsupported sample field requested!";
|
||||
LOG_CRITICAL("Unsupported sample field requested!");
|
||||
std::abort();
|
||||
|
||||
if constexpr(std::is_pointer<Tp>::value)
|
||||
return nullptr;
|
||||
|
||||
+14
-13
@@ -22,11 +22,12 @@
|
||||
|
||||
#include "library/process_sampler.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "library/amd_smi.hpp"
|
||||
#include "library/cpu_freq.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
@@ -83,8 +84,8 @@ sampler::poll(std::atomic<State>* _state, nsec_t _interval, promise_t* _ready)
|
||||
for(auto& itr : instances)
|
||||
itr->config();
|
||||
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1, "Background process sampling polling at an interval of %f seconds...\n",
|
||||
LOG_DEBUG(
|
||||
"Background process sampling polling at an interval of {:.2f} seconds...",
|
||||
std::chrono::duration_cast<std::chrono::duration<double>>(_interval).count());
|
||||
|
||||
auto _duration = config::get_process_sampling_duration();
|
||||
@@ -113,15 +114,12 @@ sampler::poll(std::atomic<State>* _state, nsec_t _interval, promise_t* _ready)
|
||||
|
||||
if(_has_duration && _now >= _end && get_state() < State::Finalized)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
1,
|
||||
"Background process sampling duration of %f seconds has elapsed. "
|
||||
"Shutting down process sampling...\n",
|
||||
_duration);
|
||||
LOG_DEBUG("Background process sampling duration of {:.2f} seconds has elapsed. "
|
||||
"Shutting down process sampling...",
|
||||
_duration);
|
||||
}
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_BASIC_PRINT(get_debug(),
|
||||
"Thread sampler polling completed...\n");
|
||||
LOG_DEBUG("Thread sampler polling completed...");
|
||||
|
||||
if(polling_finished) polling_finished->set_value();
|
||||
}
|
||||
@@ -131,11 +129,11 @@ sampler::setup()
|
||||
{
|
||||
if(!get_use_process_sampling())
|
||||
{
|
||||
ROCPROFSYS_DEBUG("Background sampler is disabled...\n");
|
||||
LOG_DEBUG("Background sampler is disabled...");
|
||||
return;
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(1, "Setting up background sampler...\n");
|
||||
LOG_DEBUG("Setting up background sampler...");
|
||||
|
||||
// shutdown if already running
|
||||
shutdown();
|
||||
@@ -204,7 +202,10 @@ sampler::shutdown()
|
||||
}
|
||||
|
||||
// during CI, throw an error if polling_finished is not valid
|
||||
ROCPROFSYS_CI_THROW(!polling_finished, "polling_finished is not valid\n");
|
||||
if(!polling_finished && get_is_continuous_integration())
|
||||
{
|
||||
throw std::runtime_error("polling_finished is not valid");
|
||||
}
|
||||
if(polling_finished)
|
||||
{
|
||||
// wait for the thread to finish
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "library/ptl.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "library/runtime.hpp"
|
||||
@@ -33,6 +32,8 @@
|
||||
#include <PTL/ThreadPool.hh>
|
||||
#include <PTL/UserTaskQueue.hh>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <timemory/backends/threading.hpp>
|
||||
#include <timemory/utility/declaration.hpp>
|
||||
|
||||
@@ -129,7 +130,7 @@ join()
|
||||
{
|
||||
if(general::get_thread_pool_state() == State::Active)
|
||||
{
|
||||
ROCPROFSYS_DEBUG_F("waiting for all general tasks to complete...\n");
|
||||
LOG_DEBUG("waiting for all general tasks to complete...");
|
||||
for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
|
||||
general::get_task_group(i).join();
|
||||
}
|
||||
@@ -140,7 +141,7 @@ shutdown()
|
||||
{
|
||||
if(general::get_thread_pool_state() == State::Active)
|
||||
{
|
||||
ROCPROFSYS_DEBUG_F("Waiting on completion of general tasks...\n");
|
||||
LOG_DEBUG("Waiting on completion of general tasks...");
|
||||
for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
|
||||
{
|
||||
general::get_task_group(i).join();
|
||||
@@ -152,13 +153,13 @@ shutdown()
|
||||
|
||||
if(get_thread_pool_state() == State::Active)
|
||||
{
|
||||
ROCPROFSYS_DEBUG_F("Destroying the rocprof-sys thread pool...\n");
|
||||
LOG_DEBUG("Destroying the rocprof-sys thread pool...");
|
||||
get_thread_pool().destroy_threadpool();
|
||||
get_thread_pool_state() = State::Finalized;
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_DEBUG_F("thread-pool is not active...\n");
|
||||
LOG_DEBUG("thread-pool is not active...");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "library/rocm.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/dynamic_library.hpp"
|
||||
#include "core/gpu.hpp"
|
||||
#include "library/amd_smi.hpp"
|
||||
|
||||
+141
-98
@@ -27,7 +27,6 @@
|
||||
#include "core/common_types.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/containers/stable_vector.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/gpu.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
@@ -68,6 +67,8 @@
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <cctype>
|
||||
#include <cstdint>
|
||||
@@ -217,10 +218,10 @@ create_agent_profile(rocprofiler_agent_id_t agent_id,
|
||||
auto agent_info_it = data->agent_counter_info.find(agent_id);
|
||||
if(agent_info_it == data->agent_counter_info.end())
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(0,
|
||||
"Skipping GPU agent %lu (device %lu) due to unsupported "
|
||||
"architecture or missing counter info\n",
|
||||
agent_id.handle, tool_agent_v->device_id);
|
||||
LOG_WARNING("Skipping GPU agent {} (device {}) due to unsupported "
|
||||
"architecture or missing counter info",
|
||||
agent_id.handle, tool_agent_v->device_id);
|
||||
|
||||
data->agent_counter_profiles.emplace(agent_id, profile);
|
||||
return counter_vec_t{};
|
||||
}
|
||||
@@ -235,17 +236,20 @@ create_agent_profile(rocprofiler_agent_id_t agent_id,
|
||||
name_v = itr.substr(0, pos);
|
||||
auto dev_id_s = itr.substr(pos + device_qualifier.length());
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_ABORT(dev_id_s.empty() ||
|
||||
dev_id_s.find_first_not_of("0123456789") !=
|
||||
std::string::npos,
|
||||
"invalid device qualifier format (':device=N) "
|
||||
"where N is the GPU id: %s\n",
|
||||
itr.c_str());
|
||||
if(dev_id_s.empty() ||
|
||||
dev_id_s.find_first_not_of("0123456789") != std::string::npos)
|
||||
{
|
||||
LOG_CRITICAL("invalid device qualifier format (':device=N) "
|
||||
"where N is the GPU id: {}",
|
||||
itr);
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
|
||||
auto dev_id_v = std::stoul(dev_id_s);
|
||||
|
||||
ROCPROFSYS_PRINT_F("tool agent device id=%lu, name=%s, device_id=%lu\n",
|
||||
tool_agent_v->device_id, name_v.c_str(), dev_id_v);
|
||||
LOG_DEBUG("tool agent device id={}, name={}, device_id={}",
|
||||
tool_agent_v->device_id, name_v, dev_id_v);
|
||||
|
||||
// skip this counter if the counter is for a specific device id (which
|
||||
// doesn't this agent's device id)
|
||||
@@ -264,14 +268,13 @@ create_agent_profile(rocprofiler_agent_id_t agent_id,
|
||||
|
||||
if(name_v != _old_name_v)
|
||||
{
|
||||
ROCPROFSYS_PRINT_F("tool agent device id=%lu, old_name=%s, name=%s\n",
|
||||
tool_agent_v->device_id, _old_name_v.c_str(),
|
||||
name_v.c_str());
|
||||
LOG_DEBUG("tool agent device id={}, old_name={}, name={}",
|
||||
tool_agent_v->device_id, _old_name_v, name_v);
|
||||
}
|
||||
else if(name_v == itr)
|
||||
{
|
||||
ROCPROFSYS_PRINT_F("tool agent device id=%lu, name=%s\n",
|
||||
tool_agent_v->device_id, name_v.c_str());
|
||||
LOG_DEBUG("tool agent device id={}, name={}", tool_agent_v->device_id,
|
||||
name_v);
|
||||
}
|
||||
|
||||
// search the gpu agent counter info for a counter with a matching name
|
||||
@@ -303,23 +306,23 @@ create_agent_profile(rocprofiler_agent_id_t agent_id,
|
||||
timemory::join::array_config{ ", ", "", "" }, missing_counters);
|
||||
|
||||
// In production, warn and continue with available counters
|
||||
ROCPROFSYS_WARNING_F(0,
|
||||
"Unable to find all counters for agent %i (gpu-%li, %s). "
|
||||
"Requested: %s. Found: %s. Missing: %s. Continuing with "
|
||||
"available counters.\n",
|
||||
tool_agent_v->agent->node_id, tool_agent_v->device_id,
|
||||
tool_agent_v->agent->name.c_str(),
|
||||
requested_counters.c_str(), found_counters.c_str(),
|
||||
missing_counters_str.c_str());
|
||||
LOG_WARNING("Unable to find all counters for agent {} (gpu-{}, {}). "
|
||||
"Requested: {}. Found: {}. Missing: {}. Continuing with "
|
||||
"available counters.",
|
||||
tool_agent_v->agent->node_id, tool_agent_v->device_id,
|
||||
tool_agent_v->agent->name, requested_counters, found_counters,
|
||||
missing_counters_str);
|
||||
|
||||
// In CI, throw to catch issues early
|
||||
ROCPROFSYS_CI_THROW(
|
||||
true,
|
||||
"Unable to find all counters for agent %i (gpu-%li, %s). Requested: %s. "
|
||||
"Found: %s. Missing: %s",
|
||||
tool_agent_v->agent->node_id, tool_agent_v->device_id,
|
||||
tool_agent_v->agent->name.c_str(), requested_counters.c_str(),
|
||||
found_counters.c_str(), missing_counters_str.c_str());
|
||||
if(get_is_continuous_integration())
|
||||
{
|
||||
LOG_CRITICAL(
|
||||
"Unable to find all counters for agent {} (gpu-{}, {}) in {}. Found: {}",
|
||||
tool_agent_v->agent->node_id, tool_agent_v->device_id,
|
||||
tool_agent_v->agent->name, requested_counters, found_counters);
|
||||
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::abort();
|
||||
}
|
||||
}
|
||||
|
||||
if(!counters_v.empty())
|
||||
@@ -725,10 +728,13 @@ tool_tracing_callback_stop(
|
||||
{
|
||||
case ROCPROFILER_MARKER_CORE_API_ID_roctxRangePop:
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(
|
||||
get_marker_pushed_ranges().empty(),
|
||||
"roctxRangePop does not have corresponding roctxRangePush on "
|
||||
"this thread");
|
||||
if(get_marker_pushed_ranges().empty())
|
||||
{
|
||||
LOG_CRITICAL("roctxRangePop does not have corresponding "
|
||||
"roctxRangePush on this thread");
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::abort();
|
||||
}
|
||||
|
||||
auto _hash = get_marker_pushed_ranges().back().first;
|
||||
_name = tim::get_hash_identifier_fast(_hash);
|
||||
@@ -738,11 +744,13 @@ tool_tracing_callback_stop(
|
||||
}
|
||||
case ROCPROFILER_MARKER_CORE_API_ID_roctxRangeStop:
|
||||
{
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(
|
||||
get_marker_started_ranges().empty(),
|
||||
"roctxRangeStop does not have corresponding roctxRangeStart "
|
||||
"on "
|
||||
"this thread");
|
||||
if(get_marker_started_ranges().empty())
|
||||
{
|
||||
LOG_CRITICAL("roctxRangeStop does not have corresponding "
|
||||
"roctxRangeStart on this thread");
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::abort();
|
||||
}
|
||||
|
||||
auto _hash = get_marker_started_ranges().back().first;
|
||||
_name = tim::get_hash_identifier_fast(_hash);
|
||||
@@ -1300,8 +1308,7 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
|
||||
|
||||
if(rocprofsys::get_state() != rocprofsys::State::Active)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(0, "Callback called when tool is not active.\n\t%s\n",
|
||||
info.str().c_str());
|
||||
LOG_WARNING("Callback called when tool is not active. {}", info.str().c_str());
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1376,14 +1383,23 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
|
||||
case ROCPROFILER_CALLBACK_TRACING_HIP_STREAM:
|
||||
#endif
|
||||
{
|
||||
ROCPROFSYS_CI_ABORT(true, "unhandled callback record kind: %i\n",
|
||||
record.kind);
|
||||
if(get_is_continuous_integration())
|
||||
{
|
||||
LOG_CRITICAL("Unhandled callback record kind: {}",
|
||||
static_cast<int>(record.kind));
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
ROCPROFSYS_CI_ABORT(true, "Unhandled callback record: \n\t%s\n",
|
||||
info.str().c_str());
|
||||
if(get_is_continuous_integration())
|
||||
{
|
||||
LOG_CRITICAL("Unhandled callback record: {}", info.str());
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1461,14 +1477,23 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
|
||||
case ROCPROFILER_CALLBACK_TRACING_HIP_STREAM:
|
||||
#endif
|
||||
{
|
||||
ROCPROFSYS_CI_ABORT(true, "unhandled callback record kind: %i\n",
|
||||
record.kind);
|
||||
if(get_is_continuous_integration())
|
||||
{
|
||||
LOG_CRITICAL("Unhandled callback record kind: {}",
|
||||
static_cast<int>(record.kind));
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
ROCPROFSYS_CI_ABORT(true, "Unhandled callback record\n\t%s\n",
|
||||
info.str().c_str());
|
||||
if(get_is_continuous_integration())
|
||||
{
|
||||
LOG_CRITICAL("Unhandled callback record: {}", info.str());
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -1554,31 +1579,32 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
|
||||
break;
|
||||
}
|
||||
default:
|
||||
ROCPROFSYS_WARNING_F(
|
||||
1,
|
||||
"tool_tracing_callback: unhandled PHASE_NONE "
|
||||
"callback record\n\t%s\n",
|
||||
info.str().c_str());
|
||||
LOG_WARNING("tool_tracing_callback: unhandled PHASE_NONE "
|
||||
"callback record: {}",
|
||||
info.str());
|
||||
}
|
||||
}
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(1,
|
||||
"tool_tracing_callback: unhandled PHASE_NONE "
|
||||
"callback record\n\t%s\n",
|
||||
info.str().c_str());
|
||||
LOG_WARNING("tool_tracing_callback: unhandled PHASE_NONE "
|
||||
"callback record: {}",
|
||||
info.str());
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_CI_ABORT(true, "unhandled callback record phase: %i\n", record.phase);
|
||||
ROCPROFSYS_WARNING_F(1,
|
||||
"tool_tracing_callback: unhandled callback record\n\t%s\n",
|
||||
info.str().c_str());
|
||||
if(get_is_continuous_integration())
|
||||
{
|
||||
LOG_CRITICAL("unhandled callback record phase: {}",
|
||||
static_cast<int>(record.phase));
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::abort();
|
||||
}
|
||||
LOG_WARNING("tool_tracing_callback: unhandled callback record: {}", info.str());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1863,17 +1889,18 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW(
|
||||
throw std::runtime_error(fmt::format(
|
||||
"unexpected rocprofiler_record_header_t buffer tracing category "
|
||||
"kind. category: %i, kind: %i\n",
|
||||
header->category, header->kind);
|
||||
"kind. category: {}, kind: {}",
|
||||
static_cast<int>(header->category), static_cast<int>(header->kind)));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_THROW("unexpected rocprofiler_record_header_t tracing category "
|
||||
"kind. category: %i, kind: %i\n",
|
||||
header->category, header->kind);
|
||||
throw std::runtime_error(fmt::format(
|
||||
"unexpected rocprofiler_record_header_t buffer tracing category "
|
||||
"kind. category: {}, kind: {}",
|
||||
static_cast<int>(header->category), static_cast<int>(header->kind)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1951,13 +1978,21 @@ counter_record_callback(rocprofiler_dispatch_counting_service_data_t dispatch_da
|
||||
const auto* _agent = tool_data->get_gpu_tool_agent(_agent_id);
|
||||
const auto* _info = tool_data->get_tool_counter_info(_agent_id, itr.first);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(
|
||||
!_agent, "unable to find tool agent for agent (id=%zu)\n",
|
||||
_agent_id.handle);
|
||||
ROCPROFSYS_CONDITIONAL_ABORT_F(!_info,
|
||||
"unable to find counter info for counter "
|
||||
"(id=%zu) on agent (id=%zu)\n",
|
||||
itr.first.handle, _agent_id.handle);
|
||||
if(!_agent)
|
||||
{
|
||||
LOG_CRITICAL("unable to find tool agent for agent (id={})",
|
||||
_agent_id.handle);
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::abort();
|
||||
}
|
||||
if(!_info)
|
||||
{
|
||||
LOG_CRITICAL(
|
||||
"unable to find counter info for counter (id={}) on agent (id={})",
|
||||
itr.first.handle, _agent_id.handle);
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::abort();
|
||||
}
|
||||
|
||||
auto _dev_id = static_cast<uint32_t>(_agent->device_id);
|
||||
|
||||
@@ -2059,11 +2094,11 @@ tool_hip_stream_callback(rocprofiler_callback_tracing_record_t record,
|
||||
// STREAM_HANDLE_CREATE and DESTROY are no-ops
|
||||
if(record.operation == ROCPROFILER_HIP_STREAM_CREATE)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(3, " operation = ROCPROFILER_HIP_STREAM_CREATE\n");
|
||||
LOG_TRACE(" operation = ROCPROFILER_HIP_STREAM_CREATE");
|
||||
}
|
||||
else if(record.operation == ROCPROFILER_HIP_STREAM_DESTROY)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(3, " operation = ROCPROFILER_HIP_STREAM_DESTROY\n");
|
||||
LOG_TRACE(" operation = ROCPROFILER_HIP_STREAM_DESTROY");
|
||||
}
|
||||
else if(record.operation == ROCPROFILER_HIP_STREAM_SET)
|
||||
{
|
||||
@@ -2071,25 +2106,25 @@ tool_hip_stream_callback(rocprofiler_callback_tracing_record_t record,
|
||||
// called
|
||||
if(record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(3,
|
||||
" operation = ROCPROFILER_HIP_STREAM_SET, phase = "
|
||||
"ROCPROFILER_CALLBACK_PHASE_ENTER, stream_id=%lu\n",
|
||||
(unsigned long) stream_id.handle);
|
||||
LOG_TRACE(" operation = ROCPROFILER_HIP_STREAM_SET, phase = "
|
||||
"ROCPROFILER_CALLBACK_PHASE_ENTER, stream_id={}",
|
||||
(unsigned long) stream_id.handle);
|
||||
stream_id_push(stream_id);
|
||||
}
|
||||
// Pop stream ID off of stream stack after underlying HIP function is completed
|
||||
else if(record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(3,
|
||||
"operation = ROCPROFILER_HIP_STREAM_SET, phase = "
|
||||
"ROCPROFILER_CALLBACK_PHASE_EXIT, stream_id=%lu\n",
|
||||
(unsigned long) stream_id.handle);
|
||||
LOG_TRACE("operation = ROCPROFILER_HIP_STREAM_SET, phase = "
|
||||
"ROCPROFILER_CALLBACK_PHASE_EXIT, stream_id={}",
|
||||
(unsigned long) stream_id.handle);
|
||||
stream_id_pop();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_FAIL_F("Unknown operation for hip_stream_callback!");
|
||||
LOG_CRITICAL("Unknown operation for hip_stream_callback!");
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::exit(1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -2099,16 +2134,19 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* user_data)
|
||||
{
|
||||
auto domains = settings::instance()->at("ROCPROFSYS_ROCM_DOMAINS");
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(1, "Available ROCm Domains:\n");
|
||||
std::stringstream _domains_ss;
|
||||
for(const auto& itr : domains->get_choices())
|
||||
ROCPROFSYS_VERBOSE_F(1, "- %s\n", itr.c_str());
|
||||
_domains_ss << "- " << itr << "\n";
|
||||
LOG_DEBUG("Available ROCm Domains: \n {}", _domains_ss.str());
|
||||
|
||||
auto _callback_domains = rocprofiler_sdk::get_callback_domains();
|
||||
auto _buffered_domain = rocprofiler_sdk::get_buffered_domains();
|
||||
auto _counter_events = rocprofiler_sdk::get_rocm_events();
|
||||
auto _version = rocprofiler_sdk::get_version();
|
||||
ROCPROFSYS_WARNING_IF(_version.formatted == 0,
|
||||
"Warning! rocprofiler-sdk version not initialized\n");
|
||||
if(_version.formatted == 0)
|
||||
{
|
||||
LOG_WARNING("rocprofiler-sdk version not initialized");
|
||||
}
|
||||
|
||||
auto* _data = as_client_data(user_data);
|
||||
_data->client_fini = fini_func;
|
||||
@@ -2221,7 +2259,12 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* user_data)
|
||||
&_data->memory_alloc_buffer));
|
||||
if(_data->memory_alloc_buffer.handle == 0UL)
|
||||
{
|
||||
ROCPROFSYS_CI_ABORT(true, "Failed to create memory allocation buffer\n");
|
||||
if(get_is_continuous_integration())
|
||||
{
|
||||
LOG_CRITICAL("Failed to create memory allocation buffer");
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
::std ::abort();
|
||||
}
|
||||
}
|
||||
auto _ops =
|
||||
rocprofiler_sdk::get_operations(ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION);
|
||||
@@ -2300,7 +2343,7 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* user_data)
|
||||
|
||||
if(config::get_use_process_sampling() && config::get_use_amd_smi())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(1, "Setting amd_smi state to active...\n");
|
||||
LOG_DEBUG("Setting amd_smi state to active...");
|
||||
amd_smi::set_state(State::Active);
|
||||
}
|
||||
|
||||
@@ -2455,8 +2498,8 @@ rocprofiler_configure(uint32_t version, const char* runtime_version, uint32_t pr
|
||||
info << id->name << " is using rocprofiler-sdk v" << major << "." << minor << "."
|
||||
<< patch << " (" << runtime_version << ")";
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(0, "%s\n", info.str().c_str());
|
||||
ROCPROFSYS_VERBOSE_F(2, "client_id=%u, priority=%u\n", id->handle, priority);
|
||||
LOG_DEBUG("{}", info.str());
|
||||
LOG_DEBUG("client_id={}, priority={}", id->handle, priority);
|
||||
|
||||
ROCPROFILER_CALL(rocprofiler_at_internal_thread_create(
|
||||
rocprofsys::rocprofiler_sdk::thread_precreate,
|
||||
|
||||
+4
-3
@@ -30,6 +30,8 @@
|
||||
#include <memory>
|
||||
#include <timemory/utility/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace rocprofiler_sdk
|
||||
@@ -184,9 +186,8 @@ counter_storage::write(counter_storage_type* storage, const std::string& metric_
|
||||
{
|
||||
if(!trait::runtime_enabled<counter_data_tracker>::get())
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
1, "%s counter_data_tracker is disabled. Can't write storage.\n",
|
||||
metric_name.c_str());
|
||||
LOG_WARNING("{} counter_data_tracker is disabled. Can't write storage.",
|
||||
metric_name);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "common/synchronized.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
#include "library/rocprofiler-sdk/fwd.hpp"
|
||||
|
||||
+9
-11
@@ -21,11 +21,12 @@
|
||||
// SOFTWARE.
|
||||
|
||||
#include "library/rocprofiler-sdk/fwd.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
|
||||
#include <timemory/utility/join.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <exception>
|
||||
#include <rocprofiler-sdk/agent.h>
|
||||
#include <rocprofiler-sdk/cxx/name_info.hpp>
|
||||
@@ -98,10 +99,9 @@ get_agent_counter_info(const tool_agent_vec_t& _agents)
|
||||
|
||||
if(status != ROCPROFILER_STATUS_SUCCESS)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
0,
|
||||
"rocprofiler_iterate_agent_supported_counters failed for agent %lu "
|
||||
"with status %d (Agent HW architecture may not be supported)\n",
|
||||
LOG_WARNING(
|
||||
"rocprofiler_iterate_agent_supported_counters failed for agent {} "
|
||||
"with status {} (Agent HW architecture may not be supported)",
|
||||
_agent_id.handle, static_cast<int>(status));
|
||||
// Skip processing for this agent if it's not supported
|
||||
continue;
|
||||
@@ -181,11 +181,9 @@ client_data::initialize_event_info()
|
||||
auto agent_info_it = agent_counter_info.find(_agent_id);
|
||||
if(agent_info_it == agent_counter_info.end())
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(0,
|
||||
"Skipping GPU device %lu (%s, handle=0x%lx) due to "
|
||||
"counter not found for the specified architecture\n",
|
||||
_dev_index, aitr.agent->name.c_str(),
|
||||
aitr.agent->handle);
|
||||
LOG_WARNING("Skipping GPU device {} ({}, handle=0x{:X}) due to "
|
||||
"counter not found for the specified architecture",
|
||||
_dev_index, aitr.agent->name, aitr.agent->handle);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -259,7 +257,7 @@ client_data::initialize_event_info()
|
||||
}
|
||||
} catch(std::exception& _e)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(1, "Constructing ROCm event info failed: %s\n", _e.what());
|
||||
LOG_WARNING("Constructing ROCm event info failed: {}", _e.what());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3
-1
@@ -36,6 +36,8 @@
|
||||
#include <rocprofiler-sdk/fwd.h>
|
||||
#include <rocprofiler-sdk/registration.h>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
@@ -271,7 +273,7 @@ as_client_data(void* _ptr)
|
||||
<< "] failed with error code " \
|
||||
<< ROCPROFSYS_VARIABLE(_rocp_status_, __LINE__) \
|
||||
<< " :: " << status_msg; \
|
||||
ROCPROFSYS_WARNING(0, "%s\n", msg.str().c_str()); \
|
||||
LOG_WARNING("{}", msg.str()); \
|
||||
} \
|
||||
}
|
||||
#endif
|
||||
|
||||
+7
-3
@@ -23,11 +23,12 @@
|
||||
#include "library/rocprofiler-sdk/rccl.hpp"
|
||||
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
|
||||
#include "library/tracing.hpp"
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace rocprofiler_sdk
|
||||
@@ -84,8 +85,11 @@ rccl_type_size(ncclDataType_t datatype)
|
||||
case ncclUint64:
|
||||
case ncclFloat64: return 8;
|
||||
default:
|
||||
ROCPROFSYS_CI_ABORT(true, "Unsupported RCCL datatype: %i", datatype);
|
||||
return 0;
|
||||
{
|
||||
LOG_CRITICAL("Unsupported RCCL datatype: {}", static_cast<int>(datatype));
|
||||
::rocprofsys::set_state(::rocprofsys ::State ::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include "library/runtime.hpp"
|
||||
#include "api.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/utility.hpp"
|
||||
#include "library/thread_data.hpp"
|
||||
@@ -43,6 +42,8 @@
|
||||
#include <timemory/utility/declaration.hpp>
|
||||
#include <timemory/utility/signals.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <array>
|
||||
#include <csignal>
|
||||
#include <cstdint>
|
||||
@@ -192,8 +193,7 @@ setup_gotchas()
|
||||
if(_initialized) return;
|
||||
_initialized = true;
|
||||
|
||||
ROCPROFSYS_BASIC_DEBUG(
|
||||
"Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread\n");
|
||||
LOG_DEBUG("Configuring gotcha wrapper around fork, MPI_Init, and MPI_Init_thread");
|
||||
|
||||
component::mpi_gotcha::configure();
|
||||
component::exit_gotcha::configure();
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/components/fwd.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/locking.hpp"
|
||||
#include "core/node_info.hpp"
|
||||
@@ -74,6 +73,7 @@
|
||||
#include <timemory/utility/types.hpp>
|
||||
#include <timemory/variadic.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <array>
|
||||
@@ -174,7 +174,7 @@ generate_call_stack_json(const tim::unwind::processed_entry& stack_entry)
|
||||
nlohmann::json call_stack;
|
||||
|
||||
call_stack["name"] = std::string(rocprofsys::utility::demangle(stack_entry.name));
|
||||
call_stack["pc"] = as_hex(stack_entry.address);
|
||||
call_stack["pc"] = fmt::format("{:X}", stack_entry.address);
|
||||
call_stack["file"] = std::string(stack_entry.location);
|
||||
|
||||
return call_stack.dump();
|
||||
@@ -184,7 +184,7 @@ std::string
|
||||
generate_line_info_json(const tim::unwind::processed_entry& line_info_entry)
|
||||
{
|
||||
nlohmann::json line_info;
|
||||
line_info["line_address"] = as_hex(line_info_entry.line_address);
|
||||
line_info["line_address"] = fmt::format("{:X}", line_info_entry.line_address);
|
||||
line_info["name"] = std::string(rocprofsys::utility::demangle(line_info_entry.name));
|
||||
|
||||
if(line_info_entry.lineinfo && !line_info_entry.lineinfo.lines.empty())
|
||||
@@ -237,7 +237,10 @@ void
|
||||
metadata_initialize_thread_info(size_t tid)
|
||||
{
|
||||
const auto& _thread_info = thread_info::get(tid, SequentTID);
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "No valid thread info for tid=%li\n", tid);
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error(fmt::format("No valid thread info for tid={}", tid));
|
||||
}
|
||||
if(!_thread_info) return;
|
||||
|
||||
trace_cache::get_metadata_registry().add_thread_info(
|
||||
@@ -251,7 +254,10 @@ void
|
||||
metadata_initialize_track(int64_t tid)
|
||||
{
|
||||
const auto& _thread_info = thread_info::get(tid, SequentTID);
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "No valid thread info for tid=%li\n", tid);
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error(fmt::format("No valid thread info for tid={}", tid));
|
||||
}
|
||||
if(!_thread_info) return;
|
||||
|
||||
size_t thread_id = _thread_info->index_data->system_value;
|
||||
@@ -298,11 +304,16 @@ void
|
||||
cache_sampling_data(int64_t _tid, const std::vector<timer_sampling_data>& _timer_data,
|
||||
const std::vector<overflow_sampling_data>& _overflow_data)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3 || get_debug_sampling(),
|
||||
"[%li] Storing sampling data to trace cache...\n", _tid);
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("[{}] Storing sampling data to trace cache...", _tid);
|
||||
}
|
||||
|
||||
const auto& _thread_info = thread_info::get(_tid, SequentTID);
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "No valid thread info for tid=%li\n", _tid);
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error(fmt::format("No valid thread info for tid={}", _tid));
|
||||
}
|
||||
if(!_thread_info) return;
|
||||
|
||||
// Store timer sampling data
|
||||
@@ -566,9 +577,7 @@ start_duration_thread()
|
||||
if(_premature && !_finalized)
|
||||
{
|
||||
// protect against spurious wakeups
|
||||
ROCPROFSYS_VERBOSE(
|
||||
2, "%sSpurious wakeup of sampling duration thread...\n",
|
||||
tim::log::color::warning());
|
||||
LOG_WARNING("Spurious wakeup of sampling duration thread...");
|
||||
_wait = true;
|
||||
}
|
||||
else if(_finalized)
|
||||
@@ -578,17 +587,16 @@ start_duration_thread()
|
||||
else
|
||||
{
|
||||
get_duration_disabled().store(true);
|
||||
ROCPROFSYS_VERBOSE(1,
|
||||
"Sampling duration of %f seconds has elapsed. "
|
||||
"Shutting down sampling...\n",
|
||||
config::get_sampling_duration());
|
||||
LOG_INFO("Sampling duration of {:.6f} seconds has elapsed. "
|
||||
"Shutting down sampling...",
|
||||
config::get_sampling_duration());
|
||||
configure(false, 0);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
ROCPROFSYS_VERBOSE(1, "Sampling will be disabled after %f seconds...\n",
|
||||
config::get_sampling_duration());
|
||||
LOG_INFO("Sampling will be disabled after {:.6f} seconds",
|
||||
config::get_sampling_duration());
|
||||
|
||||
ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
|
||||
get_duration_thread() = std::make_unique<std::thread>(_func);
|
||||
@@ -604,9 +612,13 @@ get_offload_file()
|
||||
if(get_use_tmp_files())
|
||||
{
|
||||
auto _success = _tmp_v->open();
|
||||
ROCPROFSYS_CI_FAIL(!_success,
|
||||
"Error opening sampling offload temporary file '%s'\n",
|
||||
_tmp_v->filename.c_str());
|
||||
if(get_is_continuous_integration() && !_success)
|
||||
{
|
||||
LOG_CRITICAL("Error opening sampling offload temporary file '{}'",
|
||||
_tmp_v->filename);
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::abort();
|
||||
}
|
||||
}
|
||||
return _tmp_v;
|
||||
}();
|
||||
@@ -629,26 +641,40 @@ auto offload_seq_data = std::unordered_map<int64_t, std::set<pos_type>>{};
|
||||
void
|
||||
offload_buffer(int64_t _seq, sampler_buffer_t&& _buf)
|
||||
{
|
||||
ROCPROFSYS_REQUIRE(get_use_tmp_files())
|
||||
<< "Error! sampling allocator tries to offload buffer of samples but "
|
||||
"rocprof-sys was configured to not use temporary files\n";
|
||||
if(!get_use_tmp_files())
|
||||
{
|
||||
LOG_CRITICAL("sampling allocator tries to offload buffer of samples but "
|
||||
"rocprof-sys was configured to not use temporary files");
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
// use homemade atomic_mutex/atomic_lock since contention will be low
|
||||
// and using pthread_lock might trigger our wrappers
|
||||
auto _lk = locking::atomic_lock{ get_offload_mutex() };
|
||||
auto& _file = get_offload_file();
|
||||
|
||||
ROCPROFSYS_REQUIRE(_file)
|
||||
<< "Error! sampling allocator tried to offload buffer of samples for thread "
|
||||
<< _seq << " but the offload file does not exist\n";
|
||||
if(!_file)
|
||||
{
|
||||
LOG_CRITICAL("sampling allocator tried to offload buffer of samples for "
|
||||
"thread {} but the offload file does not exist",
|
||||
_seq);
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(2, "Offloading %zu samples for thread %li to %s...\n",
|
||||
_buf.count(), _seq, _file->filename.c_str());
|
||||
LOG_DEBUG("Offloading {} samples for thread {} to {}", _buf.count(), _seq,
|
||||
_file->filename);
|
||||
auto& _fs = _file->stream;
|
||||
|
||||
ROCPROFSYS_REQUIRE(_fs.good()) << "Error! temporary file for offloading buffer is in "
|
||||
"an invalid state during offload for thread "
|
||||
<< _seq << "\n";
|
||||
if(!_fs.good())
|
||||
{
|
||||
LOG_CRITICAL("temporary file for offloading buffer is in an invalid state "
|
||||
"during offload for thread {}",
|
||||
_seq);
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
offload_seq_data[_seq].emplace(_fs.tellg());
|
||||
_fs.write(reinterpret_cast<char*>(&_seq), sizeof(_seq));
|
||||
@@ -664,8 +690,8 @@ load_offload_buffer(int64_t _thread_idx)
|
||||
auto _data = std::vector<sampler_buffer_t>{};
|
||||
if(!get_use_tmp_files())
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
2, "[sampling] returning no data because using temporary files is disabled");
|
||||
LOG_WARNING(
|
||||
"[sampling] returning no data because using temporary files is disabled");
|
||||
return _data;
|
||||
}
|
||||
|
||||
@@ -675,8 +701,8 @@ load_offload_buffer(int64_t _thread_idx)
|
||||
auto& _file = get_offload_file();
|
||||
if(!_file)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
0, "[sampling] returning no data because the offload file no longer exists");
|
||||
LOG_WARNING(
|
||||
"[sampling] returning no data because the offload file no longer exists");
|
||||
return _data;
|
||||
}
|
||||
|
||||
@@ -686,7 +712,7 @@ load_offload_buffer(int64_t _thread_idx)
|
||||
|
||||
if(!_file->open(std::ios::binary | std::ios::in))
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(0, "[sampling] %s failed to open", _file->filename.c_str());
|
||||
LOG_WARNING("[sampling] {} failed to open", _file->filename);
|
||||
return _data;
|
||||
}
|
||||
|
||||
@@ -706,9 +732,8 @@ load_offload_buffer(int64_t _thread_idx)
|
||||
|
||||
if(_seq != _thread_idx)
|
||||
{
|
||||
ROCPROFSYS_WARNING_F(
|
||||
0,
|
||||
"[sampling] file position %zu returned %zi instead of (expected) %zi\n",
|
||||
LOG_WARNING(
|
||||
"[sampling] file position {} returned {} instead of (expected) {}",
|
||||
static_cast<uintptr_t>(itr), _seq, _thread_idx);
|
||||
continue;
|
||||
}
|
||||
@@ -716,8 +741,7 @@ load_offload_buffer(int64_t _thread_idx)
|
||||
_data.emplace_back(std::move(_buffer));
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(2, "[sampling] Loaded %zu samples for thread %li...\n", _count,
|
||||
_thread_idx);
|
||||
LOG_DEBUG("[sampling] Loaded {} samples for thread {}", _count, _thread_idx);
|
||||
|
||||
_file->close();
|
||||
|
||||
@@ -734,9 +758,11 @@ configure(bool _setup, int64_t _tid)
|
||||
bool _is_running = (!_running) ? false : *_running;
|
||||
auto& _signal_types = sampling::get_signal_types(_tid);
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
get_use_causal(), "Internal error! configuring sampling not permitted when "
|
||||
"causal profiling is enabled");
|
||||
if(get_use_causal())
|
||||
{
|
||||
throw std::runtime_error("Internal error! configuring sampling not permitted "
|
||||
"when causal profiling is enabled");
|
||||
}
|
||||
|
||||
ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
|
||||
ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);
|
||||
@@ -750,7 +776,7 @@ configure(bool _setup, int64_t _tid)
|
||||
{
|
||||
if(_tids.count(_tid) == 0)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3, "Disabling SIG%i from thread %li\n", _signum, _tid);
|
||||
LOG_DEBUG("Disabling SIG{} from thread {}", _signum, _tid);
|
||||
_signal_types->erase(_signum);
|
||||
}
|
||||
}
|
||||
@@ -785,10 +811,10 @@ configure(bool _setup, int64_t _tid)
|
||||
auto _verbose = std::min<int>(get_verbose() - 2, 2);
|
||||
if(get_debug_sampling()) _verbose = 2;
|
||||
|
||||
ROCPROFSYS_DEBUG("Requesting allocator for sampler on thread %lu...\n", _tid);
|
||||
LOG_DEBUG("Requesting allocator for sampler on thread {}", _tid);
|
||||
auto _alloc = get_sampler_allocator();
|
||||
|
||||
ROCPROFSYS_DEBUG("Configuring sampler for thread %lu...\n", _tid);
|
||||
LOG_DEBUG("Configuring sampler for thread {}", _tid);
|
||||
sampling::sampler_instances::construct(construct_on_thread{ _tid }, _alloc,
|
||||
"rocprofsys", _tid, _verbose);
|
||||
|
||||
@@ -847,8 +873,13 @@ configure(bool _setup, int64_t _tid)
|
||||
auto _perf_open_error =
|
||||
_perf_sampler->open(_pe, _info->index_data->system_value);
|
||||
|
||||
ROCPROFSYS_REQUIRE(!_perf_open_error)
|
||||
<< "perf backend for overflow failed to activate: " << *_perf_open_error;
|
||||
if(_perf_open_error)
|
||||
{
|
||||
LOG_CRITICAL("perf backend for overflow failed to activate: {}",
|
||||
*_perf_open_error);
|
||||
::rocprofsys::set_state(::rocprofsys::State::Finalized);
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
_perf_sampler->set_ready_signal(get_sampling_overflow_signal());
|
||||
_sampler->configure(overflow{
|
||||
@@ -879,18 +910,24 @@ configure(bool _setup, int64_t _tid)
|
||||
static_assert(tim::trait::buffer_size<sampling::sampler_t>::value > 0,
|
||||
"Error! Zero buffer size");
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_sampler->get_buffer_size() !=
|
||||
tim::trait::buffer_size<sampling::sampler_t>::value,
|
||||
"dynamic sampler has a buffer size different from static trait: %zu instead "
|
||||
"of %zu",
|
||||
_sampler->get_buffer_size(),
|
||||
tim::trait::buffer_size<sampling::sampler_t>::value);
|
||||
if(_sampler->get_buffer_size() !=
|
||||
tim::trait::buffer_size<sampling::sampler_t>::value)
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(
|
||||
_sampler->get_buffer_size() <= 0,
|
||||
"dynamic sampler requires a positive buffer size: %zu",
|
||||
_sampler->get_buffer_size());
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("dynamic sampler has a buffer size different from static "
|
||||
"trait: {} instead "
|
||||
"of {}",
|
||||
_sampler->get_buffer_size(),
|
||||
tim::trait::buffer_size<sampling::sampler_t>::value));
|
||||
}
|
||||
|
||||
if(_sampler->get_buffer_size() <= 0)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("dynamic sampler requires a positive buffer size: {}",
|
||||
_sampler->get_buffer_size()));
|
||||
}
|
||||
|
||||
for(auto itr : *_signal_types)
|
||||
{
|
||||
@@ -900,10 +937,9 @@ configure(bool _setup, int64_t _tid)
|
||||
auto _overflow_event =
|
||||
get_setting_value<std::string>("ROCPROFSYS_SAMPLING_OVERFLOW_EVENT")
|
||||
.value_or("perf::PERF_COUNT_HW_CACHE_REFERENCES");
|
||||
ROCPROFSYS_VERBOSE(2,
|
||||
"[SIG%i] Sampler for thread %lu will be triggered "
|
||||
"every %.1f %s events...\n",
|
||||
itr, _tid, _freq, _overflow_event.c_str());
|
||||
LOG_INFO("[SIG{}] Sampler for thread {} will be triggered every {:.1f} "
|
||||
"{} events...",
|
||||
itr, _tid, _freq, _overflow_event);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -913,10 +949,9 @@ configure(bool _setup, int64_t _tid)
|
||||
dynamic_cast<const timer*>(_sampler->get_trigger(itr));
|
||||
if(_timer)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
2,
|
||||
"[SIG%i] Sampler for thread %lu will be triggered %.1fx per "
|
||||
"second of %s-time (every %.3e milliseconds)...\n",
|
||||
LOG_INFO(
|
||||
"[SIG{}] Sampler for thread {} will be triggered {:.1f}x per "
|
||||
"second of {}-time (every {:.3e} milliseconds)...",
|
||||
itr, _tid, _timer->get_frequency(units::sec), _type,
|
||||
_timer->get_period(units::msec));
|
||||
}
|
||||
@@ -933,7 +968,7 @@ configure(bool _setup, int64_t _tid)
|
||||
}
|
||||
else if(!_setup && _sampler && _is_running)
|
||||
{
|
||||
ROCPROFSYS_DEBUG("Stopping sampler for thread %lu...\n", _tid);
|
||||
LOG_DEBUG("Stopping sampler for thread {}...", _tid);
|
||||
*_running = false;
|
||||
|
||||
if(_tid == threading::get_id() && !_signal_types->empty())
|
||||
@@ -982,7 +1017,7 @@ configure(bool _setup, int64_t _tid)
|
||||
if(trait::runtime_enabled<backtrace_metrics>::get())
|
||||
backtrace_metrics::configure(_setup, _tid);
|
||||
|
||||
ROCPROFSYS_DEBUG("Sampler destroyed for thread %lu\n", _tid);
|
||||
LOG_DEBUG("Sampler destroyed for thread {}...", _tid);
|
||||
}
|
||||
|
||||
return (_signal_types) ? *_signal_types : std::set<int>{};
|
||||
@@ -1058,12 +1093,12 @@ block_signals(std::set<int> _signals)
|
||||
if(_signals.empty()) _signals = *get_signal_types(threading::get_id());
|
||||
if(_signals.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "No signals to block...\n");
|
||||
LOG_DEBUG("No signals to block...");
|
||||
return;
|
||||
}
|
||||
|
||||
ROCPROFSYS_DEBUG("Blocking signals [%s] on thread #%lu...\n",
|
||||
get_signal_names(_signals).c_str(), threading::get_id());
|
||||
LOG_DEBUG("Blocking signals [{}] on thread #{}...", get_signal_names(_signals),
|
||||
threading::get_id());
|
||||
|
||||
sigset_t _v = get_signal_set(_signals);
|
||||
thread_sigmask(SIG_BLOCK, &_v, nullptr);
|
||||
@@ -1075,12 +1110,12 @@ unblock_signals(std::set<int> _signals)
|
||||
if(_signals.empty()) _signals = *get_signal_types(threading::get_id());
|
||||
if(_signals.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2, "No signals to unblock...\n");
|
||||
LOG_DEBUG("No signals to unblock...");
|
||||
return;
|
||||
}
|
||||
|
||||
ROCPROFSYS_DEBUG("Unblocking signals [%s] on thread #%lu...\n",
|
||||
get_signal_names(_signals).c_str(), threading::get_id());
|
||||
LOG_DEBUG("Unblocking signals [{}] on thread #{}...", get_signal_names(_signals),
|
||||
threading::get_id());
|
||||
|
||||
sigset_t _v = get_signal_set(_signals);
|
||||
thread_sigmask(SIG_UNBLOCK, &_v, nullptr);
|
||||
@@ -1096,7 +1131,10 @@ post_process()
|
||||
auto _external_samples = std::atomic<size_t>{ 0 };
|
||||
auto _internal_samples = std::atomic<size_t>{ 0 };
|
||||
|
||||
ROCPROFSYS_VERBOSE(2 || get_debug_sampling(), "Stopping sampling components...\n");
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Stopping sampling components...");
|
||||
}
|
||||
|
||||
rocprofsys::component::backtrace::stop();
|
||||
configure(false, 0);
|
||||
@@ -1111,10 +1149,8 @@ post_process()
|
||||
if(!_sampler)
|
||||
{
|
||||
// this should be relatively common
|
||||
ROCPROFSYS_CONDITIONAL_PRINT(
|
||||
get_debug() && get_verbose() >= 2,
|
||||
"Post-processing sampling entries for thread %lu skipped (no sampler)\n",
|
||||
i);
|
||||
LOG_DEBUG(
|
||||
"Post-processing sampling entries for thread {} skipped (no sampler)", i);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1123,16 +1159,18 @@ post_process()
|
||||
if(!_init)
|
||||
{
|
||||
// this is not common
|
||||
ROCPROFSYS_PRINT("Post-processing sampling entries for thread %lu skipped "
|
||||
"(not initialized)\n",
|
||||
i);
|
||||
LOG_DEBUG("Post-processing sampling entries for thread {} skipped "
|
||||
"(not initialized)",
|
||||
i);
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto& _thread_info = thread_info::get(i, SequentTID);
|
||||
|
||||
ROCPROFSYS_VERBOSE(3 || get_debug_sampling(),
|
||||
"Getting sampler data for thread %lu...\n", i);
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Getting sampler data for thread {}...", i);
|
||||
}
|
||||
|
||||
auto _raw_data = _sampler->get_data();
|
||||
auto _loaded_data = load_offload_buffer(i);
|
||||
@@ -1147,14 +1185,19 @@ post_process()
|
||||
line.destroy();
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(2 || get_debug_sampling(),
|
||||
"Sampler data for thread %lu has %zu initial entries...\n", i,
|
||||
_raw_data.size());
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Sampler data for thread {} has {} initial entries...", i,
|
||||
_raw_data.size());
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(
|
||||
_sampler->get_sample_count() != _raw_data.size(),
|
||||
"Error! sampler recorded %zu samples but %zu samples were returned\n",
|
||||
_sampler->get_sample_count(), _raw_data.size());
|
||||
if(get_is_continuous_integration() &&
|
||||
_sampler->get_sample_count() != _raw_data.size())
|
||||
{
|
||||
throw std::runtime_error(fmt::format(
|
||||
"Error! sampler recorded {} samples but {} samples were returned",
|
||||
_sampler->get_sample_count(), _raw_data.size()));
|
||||
}
|
||||
// single sample that is useless (backtrace to unblocking signals)
|
||||
if(_raw_data.size() == 1 && _raw_data.front().size() <= 1) _raw_data.clear();
|
||||
|
||||
@@ -1176,9 +1219,11 @@ post_process()
|
||||
|
||||
if(!_data.empty())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(2 || get_debug_sampling(),
|
||||
"Sampler data for thread %lu has %zu valid entries...\n",
|
||||
i, _data.size());
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Sampler data for thread {} has {} valid entries...", i,
|
||||
_data.size());
|
||||
}
|
||||
|
||||
auto _timer_data = parse_timer_data(i, _init, _data);
|
||||
auto _overflow_data = parse_overflow_data(i, _init, _data);
|
||||
@@ -1189,16 +1234,19 @@ post_process()
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(
|
||||
2 || get_debug_sampling(),
|
||||
"Sampler data for thread %lu has zero valid entries out of "
|
||||
"%zu... (skipped)\n",
|
||||
i, _raw_data.size());
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Sampler data for thread {} has zero valid entries out of "
|
||||
"{}... (skipped)",
|
||||
i, _raw_data.size());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(3 || get_debug_sampling(),
|
||||
"Destroying samplers and allocators...\n");
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Destroying samplers and allocators...");
|
||||
}
|
||||
|
||||
get_offload_file().reset(); // remove the temporary file
|
||||
|
||||
@@ -1216,11 +1264,13 @@ post_process()
|
||||
get_offload_file().reset();
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(1 || get_debug_sampling(),
|
||||
"Collected %zu samples from %zu threads... %zu samples out of %zu "
|
||||
"were taken while within instrumented routines\n",
|
||||
_total_data, _total_threads, _internal_samples.load(),
|
||||
(_internal_samples + _external_samples));
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("Collected {} samples from {} threads... {} samples out of {} "
|
||||
"were taken while within instrumented routines",
|
||||
_total_data, _total_threads, _internal_samples.load(),
|
||||
(_internal_samples + _external_samples));
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
@@ -1324,20 +1374,26 @@ post_process_perfetto(int64_t _tid, const std::vector<timer_sampling_data>& _tim
|
||||
|
||||
if(trait::runtime_enabled<backtrace_metrics>::get())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3 || get_debug_sampling(),
|
||||
"[%li] Post-processing metrics for perfetto...\n", _tid);
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("[{}] Post-processing metrics for perfetto...", _tid);
|
||||
}
|
||||
backtrace_metrics::init_perfetto(_tid, _valid_metrics);
|
||||
for(const auto& itr : _timer_data)
|
||||
itr.m_metrics.post_process_perfetto(_tid, 0.5 * (itr.m_beg + itr.m_end));
|
||||
backtrace_metrics::fini_perfetto(_tid, _valid_metrics);
|
||||
}
|
||||
|
||||
ROCPROFSYS_VERBOSE(3 || get_debug_sampling(),
|
||||
"[%li] Post-processing backtraces for perfetto...\n", _tid);
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("[{}] Post-processing backtraces for perfetto...", _tid);
|
||||
}
|
||||
|
||||
const auto& _thread_info = thread_info::get(_tid, SequentTID);
|
||||
ROCPROFSYS_CI_THROW(!_thread_info, "No valid thread info for tid=%li\n", _tid);
|
||||
|
||||
if(get_is_continuous_integration() && !_thread_info)
|
||||
{
|
||||
throw std::runtime_error(fmt::format("No valid thread info for tid={}", _tid));
|
||||
}
|
||||
if(!_thread_info) return;
|
||||
|
||||
auto _overflow_event =
|
||||
@@ -1393,10 +1449,11 @@ post_process_perfetto(int64_t _tid, const std::vector<timer_sampling_data>& _tim
|
||||
if(config::get_perfetto_annotations())
|
||||
{
|
||||
tracing::add_perfetto_annotation(ctx, "file", iitr.location);
|
||||
tracing::add_perfetto_annotation(ctx, "pc",
|
||||
as_hex(iitr.address));
|
||||
tracing::add_perfetto_annotation(ctx, "line_address",
|
||||
as_hex(iitr.line_address));
|
||||
tracing::add_perfetto_annotation(
|
||||
ctx, "pc", fmt::format("{:X}", iitr.address));
|
||||
tracing::add_perfetto_annotation(
|
||||
ctx, "line_address",
|
||||
fmt::format("{:X}", iitr.line_address));
|
||||
if(iitr.lineinfo)
|
||||
{
|
||||
auto _lines = iitr.lineinfo.lines;
|
||||
@@ -1533,10 +1590,11 @@ post_process_perfetto(int64_t _tid, const std::vector<timer_sampling_data>& _tim
|
||||
_common_annotate(ctx, true);
|
||||
tracing::add_perfetto_annotation(ctx, "file",
|
||||
iitr.location);
|
||||
tracing::add_perfetto_annotation(ctx, "pc",
|
||||
as_hex(iitr.address));
|
||||
tracing::add_perfetto_annotation(
|
||||
ctx, "line_address", as_hex(iitr.line_address));
|
||||
ctx, "pc", fmt::format("{:X}", iitr.address));
|
||||
tracing::add_perfetto_annotation(
|
||||
ctx, "line_address",
|
||||
fmt::format("{:X}", iitr.line_address));
|
||||
if(iitr.lineinfo)
|
||||
{
|
||||
auto _lines = iitr.lineinfo.lines;
|
||||
@@ -1576,8 +1634,10 @@ void
|
||||
post_process_timemory(int64_t _tid, const std::vector<timer_sampling_data>& _timer_data,
|
||||
const std::vector<overflow_sampling_data>& _overflow_data)
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3 || get_debug_sampling(),
|
||||
"[%li] Post-processing data for timemory...\n", _tid);
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("[{}] Post-processing data for timemory...", _tid);
|
||||
}
|
||||
|
||||
// compute the total number of entries
|
||||
int64_t _sum = 0;
|
||||
@@ -1756,8 +1816,10 @@ cache_backtrace_metrics(
|
||||
|
||||
if(trait::runtime_enabled<backtrace_metrics>::get() && get_use_rocpd())
|
||||
{
|
||||
ROCPROFSYS_VERBOSE(3 || get_debug_sampling(),
|
||||
"[%li] Post-processing metrics for rocpd...\n", _tid);
|
||||
if(get_debug_sampling())
|
||||
{
|
||||
LOG_DEBUG("[{}] Post-processing metrics for rocpd...", _tid);
|
||||
}
|
||||
backtrace_metrics::init_cache(_tid, _valid_metrics); // move to setup
|
||||
for(const auto& itr : _timer_data)
|
||||
itr.m_metrics.cache_backtrace_data(_tid, 0.5 * (itr.m_beg + itr.m_end));
|
||||
|
||||
@@ -26,7 +26,6 @@
|
||||
#include "core/concepts.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/containers/stable_vector.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/timemory.hpp"
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/concepts.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/state.hpp"
|
||||
#include "core/utility.hpp"
|
||||
#include "library/causal/delay.hpp"
|
||||
@@ -36,6 +35,8 @@
|
||||
#include <timemory/components/timing/backends.hpp>
|
||||
#include <timemory/process/threading.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace rocprofsys
|
||||
@@ -80,31 +81,18 @@ init_index_data(int64_t _tid, bool _offset = false)
|
||||
threading::offset_this_id(_offset);
|
||||
itr = thread_index_data{};
|
||||
|
||||
ROCPROFSYS_CONDITIONAL_THROW(itr->internal_value != _tid,
|
||||
"Error! thread_info::init_index_data was called for "
|
||||
"thread %zi on thread %zi\n",
|
||||
_tid, itr->internal_value);
|
||||
if(itr->internal_value != _tid)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! thread_info::init_index_data was called for "
|
||||
"thread {} on thread {}\n",
|
||||
_tid, itr->internal_value));
|
||||
}
|
||||
|
||||
int _verb = 2;
|
||||
// if thread created using finalization, bump up the minimum verbosity level
|
||||
if(get_state() >= State::Finalized && _offset) _verb += 2;
|
||||
if(!config::settings_are_configured())
|
||||
{
|
||||
ROCPROFSYS_BASIC_VERBOSE_F(_verb,
|
||||
"Thread %li on PID %i (rank: %i) assigned "
|
||||
"rocprof-sys TID %li (internal: %li)\n",
|
||||
itr->system_value, process::get_id(), dmp::rank(),
|
||||
itr->sequent_value, itr->internal_value);
|
||||
}
|
||||
else
|
||||
{
|
||||
ROCPROFSYS_VERBOSE_F(
|
||||
_verb,
|
||||
"Thread %li on PID %i (rank: %i) assigned rocprof-sys TID "
|
||||
"%li (internal: %li)\n",
|
||||
itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value,
|
||||
itr->internal_value);
|
||||
}
|
||||
LOG_TRACE("Thread {} on PID {} (rank: {}) assigned rocprof-sys TID {} "
|
||||
"(internal: {})",
|
||||
itr->system_value, process::get_id(), dmp::rank(), itr->sequent_value,
|
||||
itr->internal_value);
|
||||
}
|
||||
return itr;
|
||||
}
|
||||
@@ -125,8 +113,8 @@ std::string
|
||||
thread_index_data::as_string() const
|
||||
{
|
||||
auto _ss = std::stringstream{};
|
||||
_ss << sequent_value << " [" << as_hex(system_value) << "] (#" << internal_value
|
||||
<< ")";
|
||||
_ss << sequent_value << " [" << fmt::format("{:x}", system_value) << "] (#"
|
||||
<< internal_value << ")";
|
||||
return _ss.str();
|
||||
}
|
||||
|
||||
@@ -144,19 +132,16 @@ grow_data(int64_t _tid)
|
||||
// check again after locking
|
||||
if(_tid >= peak_num_threads)
|
||||
{
|
||||
TIMEMORY_PRINTF_WARNING(
|
||||
stderr, "[%li] Growing thread data from %li to %li...\n", _tid,
|
||||
peak_num_threads, peak_num_threads + max_supported_threads);
|
||||
fflush(stderr);
|
||||
LOG_WARNING("[{}] Growing thread data from {} to {}...", _tid,
|
||||
peak_num_threads, peak_num_threads + max_supported_threads);
|
||||
|
||||
for(auto itr : grow_functors())
|
||||
{
|
||||
if(itr)
|
||||
{
|
||||
int64_t _new_capacity = (*itr)(_tid + 1);
|
||||
TIMEMORY_PRINTF_WARNING(stderr,
|
||||
"[%li] Grew thread data from %li to %li...\n",
|
||||
_tid, peak_num_threads, _new_capacity);
|
||||
LOG_WARNING("[{}] Grew thread data from {} to {}...", _tid,
|
||||
peak_num_threads, _new_capacity);
|
||||
}
|
||||
}
|
||||
peak_num_threads += max_supported_threads;
|
||||
@@ -243,7 +228,10 @@ thread_info::get(native_handle_t&& _tid)
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
|
||||
if(get_is_continuous_integration() && unknown_thread)
|
||||
{
|
||||
throw std::runtime_error("Unknown thread has been assigned a value");
|
||||
}
|
||||
return unknown_thread;
|
||||
}
|
||||
|
||||
@@ -259,7 +247,11 @@ thread_info::get(std::thread::id _tid)
|
||||
}
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
|
||||
if(get_is_continuous_integration() && unknown_thread)
|
||||
{
|
||||
throw std::runtime_error("Unknown thread has been assigned a value");
|
||||
}
|
||||
|
||||
return unknown_thread;
|
||||
}
|
||||
|
||||
@@ -294,16 +286,20 @@ thread_info::get(int64_t _tid, ThreadIdType _type)
|
||||
}
|
||||
else if(_type == ThreadIdType::PthreadID)
|
||||
{
|
||||
ROCPROFSYS_THROW("rocprof-sys does not support thread_info::get(int64_t, "
|
||||
"ThreadIdType) with ThreadIdType::PthreadID\n");
|
||||
throw std::runtime_error("rocprof-sys does not support thread_info::get(int64_t, "
|
||||
"ThreadIdType) with ThreadIdType::PthreadID");
|
||||
}
|
||||
else if(_type == ThreadIdType::StlThreadID)
|
||||
{
|
||||
ROCPROFSYS_THROW("rocprof-sys does not support thread_info::get(int64_t, "
|
||||
"ThreadIdType) with ThreadIdType::StlThreadID\n");
|
||||
throw std::runtime_error("rocprof-sys does not support thread_info::get(int64_t, "
|
||||
"ThreadIdType) with ThreadIdType::StlThreadID");
|
||||
}
|
||||
|
||||
if(get_is_continuous_integration() && unknown_thread)
|
||||
{
|
||||
throw std::runtime_error("Unknown thread has been assigned a value");
|
||||
}
|
||||
|
||||
ROCPROFSYS_CI_THROW(unknown_thread, "Unknown thread has been assigned a value");
|
||||
return unknown_thread;
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
#include <timemory/hash/types.hpp>
|
||||
#include <timemory/process/threading.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
namespace rocprofsys
|
||||
{
|
||||
namespace tracing
|
||||
@@ -81,8 +83,16 @@ copy_timemory_hash_ids()
|
||||
// copy these over so that all hashes are known
|
||||
auto& _hmain = tim::hash::get_main_hash_ids();
|
||||
auto& _amain = tim::hash::get_main_hash_aliases();
|
||||
ROCPROFSYS_REQUIRE(_hmain != nullptr) << "no main timemory hash ids";
|
||||
ROCPROFSYS_REQUIRE(_amain != nullptr) << "no main timemory hash aliases";
|
||||
if(_hmain == nullptr)
|
||||
{
|
||||
LOG_CRITICAL("no main timemory hash ids");
|
||||
std::exit(1);
|
||||
}
|
||||
if(_amain == nullptr)
|
||||
{
|
||||
LOG_CRITICAL("no main timemory hash aliases");
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
// combine all the hash and alias info into one container
|
||||
for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
|
||||
@@ -159,11 +169,15 @@ thread_init()
|
||||
auto _tidx = (_tinfo && _tinfo->index_data) ? _tinfo->index_data->sequent_value
|
||||
: threading::get_id();
|
||||
|
||||
ROCPROFSYS_REQUIRE(_tidx >= 0)
|
||||
<< "thread setup failed. thread info not initialized: " << [&_tinfo]() {
|
||||
if(_tinfo) return JOIN("", *_tinfo);
|
||||
return std::string{ "no thread_info" };
|
||||
}();
|
||||
if(_tidx < 0)
|
||||
{
|
||||
LOG_CRITICAL("thread setup failed. thread info not initialized: {}",
|
||||
[&_tinfo]() {
|
||||
if(_tinfo) return JOIN("", *_tinfo);
|
||||
return std::string{ "no thread_info" };
|
||||
}());
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
if(_tidx > 0) threading::set_thread_name(JOIN(" ", "Thread", _tidx).c_str());
|
||||
thread_data<thread_bundle_t>::construct(
|
||||
@@ -173,10 +187,16 @@ thread_init()
|
||||
get_timemory_hash_ids(_tidx) = tim::get_hash_ids();
|
||||
get_timemory_hash_aliases(_tidx) = tim::get_hash_aliases();
|
||||
|
||||
ROCPROFSYS_REQUIRE(get_timemory_hash_ids(_tidx) != nullptr)
|
||||
<< "no timemory hash ids pointer for thread " << _tidx;
|
||||
ROCPROFSYS_REQUIRE(get_timemory_hash_aliases(_tidx) != nullptr)
|
||||
<< "no timemory hash aliases pointer for thread " << _tidx;
|
||||
if(get_timemory_hash_ids(_tidx) == nullptr)
|
||||
{
|
||||
LOG_CRITICAL("no timemory hash ids pointer for thread {}", _tidx);
|
||||
std::exit(1);
|
||||
}
|
||||
if(get_timemory_hash_aliases(_tidx) == nullptr)
|
||||
{
|
||||
LOG_CRITICAL("no timemory hash aliases pointer for thread {}", _tidx);
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
record_thread_start_time();
|
||||
return true;
|
||||
|
||||
@@ -26,7 +26,6 @@
|
||||
#include "core/common.hpp"
|
||||
#include "core/concepts.hpp"
|
||||
#include "core/config.hpp"
|
||||
#include "core/debug.hpp"
|
||||
#include "core/defines.hpp"
|
||||
#include "core/demangler.hpp"
|
||||
#include "core/perfetto.hpp"
|
||||
@@ -51,6 +50,8 @@
|
||||
#include <timemory/mpl/type_traits.hpp>
|
||||
#include <timemory/types.hpp>
|
||||
|
||||
#include "logger/debug.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
@@ -181,10 +182,9 @@ get_perfetto_track(CategoryT, FuncT&& _desc_generator, Args&&... _args)
|
||||
_desc.set_name(_name);
|
||||
::perfetto::TrackEvent::SetTrackDescriptor(_track, _desc);
|
||||
|
||||
ROCPROFSYS_VERBOSE_F(4, "[%s] Created %s(%zu) with description: \"%s\"\n",
|
||||
trait::name<CategoryT>::value,
|
||||
rocprofsys::utility::demangle<TrackT>().c_str(), _uuid,
|
||||
_name.c_str());
|
||||
LOG_TRACE("[{}] Created {}({}) with description: \"{}\"",
|
||||
trait::name<CategoryT>::value, rocprofsys::utility::demangle<TrackT>(),
|
||||
_uuid, _name);
|
||||
|
||||
_track_uuids.emplace(_uuid, _name);
|
||||
}
|
||||
@@ -193,10 +193,13 @@ get_perfetto_track(CategoryT, FuncT&& _desc_generator, Args&&... _args)
|
||||
// overhead of generating string during releases
|
||||
#if defined(ROCPROFSYS_CI) && ROCPROFSYS_CI > 0
|
||||
auto _name = std::forward<FuncT>(_desc_generator)(std::forward<Args>(_args)...);
|
||||
ROCPROFSYS_CI_THROW(_track_uuids.at(_uuid) != _name,
|
||||
"Error! Multiple invocations of UUID %zu produced different "
|
||||
"descriptions: \"%s\" and \"%s\"\n",
|
||||
_uuid, _track_uuids.at(_uuid).c_str(), _name.c_str());
|
||||
if(get_is_continuous_integration() && _track_uuids.at(_uuid) != _name)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
fmt::format("Error! Multiple invocations of UUID {} produced different "
|
||||
"descriptions: \"{}\" and \"{}\"",
|
||||
_uuid, _track_uuids.at(_uuid), _name));
|
||||
}
|
||||
#endif
|
||||
|
||||
return TrackT(_uuid, ::perfetto::ProcessTrack::Current());
|
||||
@@ -325,8 +328,7 @@ get_timemory(CategoryT, std::string_view name)
|
||||
auto& _data = tracing::get_instrumentation_bundles();
|
||||
if(ROCPROFSYS_UNLIKELY(_data == nullptr || _data->empty()))
|
||||
{
|
||||
ROCPROFSYS_DEBUG("[%s] skipped %s :: empty bundle stack\n",
|
||||
"rocprofsys_pop_trace", name.data());
|
||||
LOG_DEBUG("[rocprofsys_pop_trace] skipped {} :: empty bundle stack", name);
|
||||
return return_type{ nullptr, -1 };
|
||||
}
|
||||
|
||||
|
||||
Daži faili netika attēloti, jo izmaiņu fails ir pārāk liels Rādīt vairāk
Atsaukties uz šo jaunā problēmā
Block a user