From e820b8862a0f6ddc856e711711d2c3014e6346ed Mon Sep 17 00:00:00 2001 From: "Madsen, Jonathan" Date: Fri, 21 Mar 2025 01:23:45 -0500 Subject: [PATCH] Logging updates (#7) * General logging updates * ROCm 6.1.x fatal defect: unconditional abort if tool found but rocprofiler-sdk missing - Adding rocprofiler-sdk support breaks backwards compatibility in ROCm 6.1.x because `rocprofiler_configure` symbol triggers looking for rocprofiler-sdk library (which was not released until ROCm 6.2) - Bump version to 0.5.0 * revert to using fatal error * Include header updates * Fix CodeQL suggestions * CMake and CI updates - minimum cmake version is 3.22.0 - added requirements.txt - improved rocprofiler_register_{formatting,linting}.cmake * Disable deprecated declarations warnings * Use "overwrite" instead of "override" - override is a keyword in C++ * Disable REQUIRED for formatting and linting when BUILD_DEVELOPER=ON --------- Co-authored-by: Jonathan R. Madsen [ROCm/rocprofiler-register commit: cb1b43025168f351589d85672fac07e0fca8c929] --- .../workflows/continuous-integration.yml | 3 +- projects/rocprofiler-register/CMakeLists.txt | 2 +- projects/rocprofiler-register/README.md | 29 ++- projects/rocprofiler-register/VERSION | 2 +- .../rocprofiler_register_build_settings.cmake | 3 +- .../rocprofiler_register_formatting.cmake | 49 +++- .../cmake/rocprofiler_register_linting.cmake | 35 ++- .../cmake/rocprofiler_register_options.cmake | 4 +- .../external/CMakeLists.txt | 2 +- projects/rocprofiler-register/external/glog | 2 +- .../rocprofiler-register/requirements.txt | 5 + .../details/CMakeLists.txt | 4 +- .../details/environment.hpp | 32 ++- .../rocprofiler-register/details/logging.cpp | 229 ++++++++++++++++++ .../rocprofiler-register/details/logging.hpp | 37 +++ .../rocprofiler_register.cpp | 44 +++- .../rocprofiler-register/tests/CMakeLists.txt | 5 + .../tests/bin/CMakeLists.txt | 108 +++++++++ .../tests/bin/simple-hip.cpp | 174 +++++++++++++ 19 files changed, 720 insertions(+), 49 deletions(-) create mode 100644 projects/rocprofiler-register/requirements.txt create mode 100644 projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.cpp create mode 100644 projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.hpp create mode 100644 projects/rocprofiler-register/tests/bin/CMakeLists.txt create mode 100644 projects/rocprofiler-register/tests/bin/simple-hip.cpp diff --git a/projects/rocprofiler-register/.github/workflows/continuous-integration.yml b/projects/rocprofiler-register/.github/workflows/continuous-integration.yml index 7c1220f87c..b437726573 100644 --- a/projects/rocprofiler-register/.github/workflows/continuous-integration.yml +++ b/projects/rocprofiler-register/.github/workflows/continuous-integration.yml @@ -81,7 +81,8 @@ jobs: sudo update-alternatives --install /usr/bin/cc cc /usr/bin/${CC} 100 && sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/${CXX} 100 && python3 -m pip install --upgrade pip && - python3 -m pip install 'cmake==3.21.4' + python3 -m pip install 'cmake==3.22.0' && + python3 -m pip install -r requirements.txt - name: Setup GCov timeout-minutes: 25 diff --git a/projects/rocprofiler-register/CMakeLists.txt b/projects/rocprofiler-register/CMakeLists.txt index 30d13080e7..2fdf5c4ec4 100644 --- a/projects/rocprofiler-register/CMakeLists.txt +++ b/projects/rocprofiler-register/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.16 FATAL_ERROR) +cmake_minimum_required(VERSION 3.22.0 FATAL_ERROR) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR AND CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) diff --git a/projects/rocprofiler-register/README.md b/projects/rocprofiler-register/README.md index 5ce882f48e..227b3e4475 100644 --- a/projects/rocprofiler-register/README.md +++ b/projects/rocprofiler-register/README.md @@ -2,18 +2,27 @@ ## Overview -The rocprofiler-register library is a helper library that coordinates the modification of the intercept API table(s) of the HSA/HIP/ROCTx -runtime libraries by the ROCprofiler (v2) library. The purpose of this library is to provide a consistent and automated mechanism +The rocprofiler-register library is a helper library that coordinates the modification of the intercept API table(s) of the HSA/HIP/ROCTx +runtime libraries by the ROCprofiler (v2) library. The purpose of this library is to provide a consistent and automated mechanism of enabling performance analysis in the ROCm runtimes which does not rely on environment variables or unique methods for each runtime -library. +library. When a runtime is initialized (either explicitly and lazily) and the intercept API table is constructed, it passes this API table to -rocprofiler-register. Rocprofiler-register scans the symbols in the address space and if it detects there is at least one visible symbol named -`rocprofiler_configure` (which is a function provided by tools), it passes the intercept API table to the rocprofiler library (dlopening +rocprofiler-register. Rocprofiler-register scans the symbols in the address space and if it detects there is at least one visible symbol named +`rocprofiler_configure` (which is a function provided by tools), it passes the intercept API table to the rocprofiler library (dlopening the rocprofiler library if it is not already loaded). The rocprofiler library then does an extensive scan for _all_ the instances of the `rocprofiler_configure` symbols and invokes each of them. The `rocprofiler_configure` function (again, provided by a tool) returns effectively tells rocprofiler which behaviors it wants to be notified about, features it wants to use (e.g. API tracing, kernel dispatch timing), -etc. +etc. + +## Environment Variables + +| Environment Variable | Description | Default Value | +|-----------------------------------|---------------------------------------------------------------------------|----------------| +| `ROCP_TOOL_LIBRARIES` | List of rocprofiler-sdk tool libraries (space, comma, or colon separated) | Empty (string) | +| `ROCPROFILER_REGISTER_ENABLED` | Set to 0/false/no to disable rocprofiler-register | true (bool) | +| `ROCPROFILER_REGISTER_SECURE` | Additional checks to ensure authenticity of runtime libraries | false (bool) | +| `ROCPROFILER_REGISTER_FORCE_LOAD` | Load rocprofiler-sdk library regardless of whether there is a tool or not | false (bool) | ## Contributing @@ -24,7 +33,7 @@ The default branch is `amd-mainline` but the only branch that should target that ### Creating a feature branch ```console -# fetch any updates +# fetch any updates git fetch origin # switch to staging branch @@ -44,18 +53,18 @@ branch as well. ### Pulling in updates to `amd-staging` to your feature branch -Linear histories are preferred so if another PR is merged into `amd-staging` while your PR is still open, please +Linear histories are preferred so if another PR is merged into `amd-staging` while your PR is still open, please select the "Update with rebase" option (i.e. try to avoid a merge commit). From the command line, the git command would be `git pull --rebase origin amd-staging`. ## Build and Installation rocprofiler-register has a standard CMake build and install process. E.g. the following configure -rocprofiler-register to build with optimizations and without debug info in a `build-rocp-reg` subdirectory, +rocprofiler-register to build with optimizations and without debug info in a `build-rocp-reg` subdirectory, build using 4 jobs, and install to `/opt/rocprofiler-register`: ```console -cmake -B build-rocp-reg . -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-register +cmake -B build-rocp-reg . -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/rocprofiler-register cmake --build build-rocp-reg --target all --parallel 4 cmake --build build-rocp-reg --target install ``` diff --git a/projects/rocprofiler-register/VERSION b/projects/rocprofiler-register/VERSION index 1d0ba9ea18..8f0916f768 100644 --- a/projects/rocprofiler-register/VERSION +++ b/projects/rocprofiler-register/VERSION @@ -1 +1 @@ -0.4.0 +0.5.0 diff --git a/projects/rocprofiler-register/cmake/rocprofiler_register_build_settings.cmake b/projects/rocprofiler-register/cmake/rocprofiler_register_build_settings.cmake index 00f8a3b402..e7ec51b29d 100644 --- a/projects/rocprofiler-register/cmake/rocprofiler_register_build_settings.cmake +++ b/projects/rocprofiler-register/cmake/rocprofiler_register_build_settings.cmake @@ -88,7 +88,8 @@ rocprofiler_register_target_compile_options( rocprofiler_register_target_compile_options( rocprofiler-register-developer-flags LANGUAGES C CXX - INTERFACE "-Werror" "-Wdouble-promotion" "-Wshadow" "-Wextra") + INTERFACE "-Werror" "-Wdouble-promotion" "-Wshadow" "-Wextra" + "-Wno-deprecated-declarations") if(ROCPROFILER_REGISTER_BUILD_DEVELOPER) target_link_libraries(rocprofiler-register-build-flags diff --git a/projects/rocprofiler-register/cmake/rocprofiler_register_formatting.cmake b/projects/rocprofiler-register/cmake/rocprofiler_register_formatting.cmake index 05e3c74a4a..9414a83ded 100644 --- a/projects/rocprofiler-register/cmake/rocprofiler_register_formatting.cmake +++ b/projects/rocprofiler-register/cmake/rocprofiler_register_formatting.cmake @@ -13,10 +13,41 @@ include_guard(DIRECTORY) -find_program(ROCPROFILER_REGISTER_CLANG_FORMAT_EXE NAMES clang-format-11 - clang-format-mp-11) -find_program(ROCPROFILER_REGISTER_CMAKE_FORMAT_EXE NAMES cmake-format) -find_program(ROCPROFILER_REGISTER_BLACK_FORMAT_EXE NAMES black) +if(NOT ROCPROFILER_REGISTER_CLANG_FORMAT_EXE AND EXISTS + $ENV{HOME}/.local/bin/clang-format) + execute_process( + COMMAND $ENV{HOME}/.local/bin/clang-format --version + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + OUTPUT_VARIABLE _CLANG_FMT_OUT + RESULT_VARIABLE _CLANG_FMT_RET + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(_CLANG_FMT_RET EQUAL 0) + if("${_CLANG_FMT_OUT}" MATCHES "version 11\\.([0-9]+)\\.([0-9]+)") + set(ROCPROFILER_REGISTER_CLANG_FORMAT_EXE + "$ENV{HOME}/.local/bin/clang-format" + CACHE FILEPATH "clang-format exe") + endif() + endif() +endif() + +find_program( + ROCPROFILER_REGISTER_CLANG_FORMAT_EXE + NAMES clang-format-11 clang-format-mp-11 clang-format + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) +find_program( + ROCPROFILER_REGISTER_CMAKE_FORMAT_EXE + NAMES cmake-format + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) +find_program( + ROCPROFILER_REGISTER_BLACK_FORMAT_EXE + NAMES black + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) add_custom_target(format-rocprofiler-register) if(NOT TARGET format) @@ -43,12 +74,14 @@ if(ROCPROFILER_REGISTER_CLANG_FORMAT_EXE set(${_TYPE}) endforeach() file(GLOB_RECURSE header_files ${PROJECT_SOURCE_DIR}/${_DIR}/*.h - ${PROJECT_SOURCE_DIR}/${_DIR}/*.hpp) + ${PROJECT_SOURCE_DIR}/${_DIR}/*.hpp ${PROJECT_SOURCE_DIR}/${_DIR}/*.h.in + ${PROJECT_SOURCE_DIR}/${_DIR}/*.hpp.in) file(GLOB_RECURSE source_files ${PROJECT_SOURCE_DIR}/${_DIR}/*.c ${PROJECT_SOURCE_DIR}/${_DIR}/*.cpp) file(GLOB_RECURSE cmake_files ${PROJECT_SOURCE_DIR}/${_DIR}/*CMakeLists.txt ${PROJECT_SOURCE_DIR}/${_DIR}/*.cmake) - file(GLOB_RECURSE python_files ${PROJECT_SOURCE_DIR}/${_DIR}/*.py) + file(GLOB_RECURSE python_files ${PROJECT_SOURCE_DIR}/${_DIR}/*.py + ${PROJECT_SOURCE_DIR}/${_DIR}/*.py.in) foreach(_TYPE header_files source_files cmake_files python_files) list(APPEND rocp_${_TYPE} ${${_TYPE}}) endforeach() @@ -76,7 +109,7 @@ if(ROCPROFILER_REGISTER_CLANG_FORMAT_EXE format-rocprofiler-register-python ${ROCPROFILER_REGISTER_BLACK_FORMAT_EXE} -q ${rocp_python_files} COMMENT - "[rocprofiler-register] Running Python formatter ${ROCPROFILER_REGISTER_BLACK_FORMAT_EXE}..." + "[rocprofiler-register] Running python formatter ${ROCPROFILER_REGISTER_BLACK_FORMAT_EXE}..." ) endif() @@ -85,7 +118,7 @@ if(ROCPROFILER_REGISTER_CLANG_FORMAT_EXE format-rocprofiler-register-cmake ${ROCPROFILER_REGISTER_CMAKE_FORMAT_EXE} -i ${rocp_cmake_files} COMMENT - "[rocprofiler-register] Running CMake formatter ${ROCPROFILER_REGISTER_CMAKE_FORMAT_EXE}..." + "[rocprofiler-register] Running cmake formatter ${ROCPROFILER_REGISTER_CMAKE_FORMAT_EXE}..." ) endif() diff --git a/projects/rocprofiler-register/cmake/rocprofiler_register_linting.cmake b/projects/rocprofiler-register/cmake/rocprofiler_register_linting.cmake index c8398bcd93..d5bfa11109 100644 --- a/projects/rocprofiler-register/cmake/rocprofiler_register_linting.cmake +++ b/projects/rocprofiler-register/cmake/rocprofiler_register_linting.cmake @@ -1,13 +1,30 @@ -include_guard(GLOBAL) - # ----------------------------------------------------------------------------------------# # # Clang Tidy # # ----------------------------------------------------------------------------------------# +include_guard(DIRECTORY) + +if(NOT ROCPROFILER_REGISTER_CLANG_TIDY_EXE AND EXISTS $ENV{HOME}/.local/bin/clang-tidy) + execute_process( + COMMAND $ENV{HOME}/.local/bin/clang-tidy --version + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + OUTPUT_VARIABLE _CLANG_TIDY_OUT + RESULT_VARIABLE _CLANG_TIDY_RET + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + + if(_CLANG_TIDY_RET EQUAL 0) + if("${_CLANG_TIDY_OUT}" MATCHES "version 1[5-9]\\.([0-9]+)\\.([0-9]+)") + set(ROCPROFILER_REGISTER_CLANG_TIDY_EXE + "$ENV{HOME}/.local/bin/clang-tidy" + CACHE FILEPATH "clang-tidy exe") + endif() + endif() +endif() + find_program( - ROCPROFILER_REGISTER_CLANG_TIDY_COMMAND + ROCPROFILER_REGISTER_CLANG_TIDY_EXE NAMES clang-tidy-18 clang-tidy-17 clang-tidy-16 @@ -16,19 +33,25 @@ find_program( clang-tidy-13 clang-tidy-12 clang-tidy-11 - clang-tidy) + clang-tidy + PATHS $ENV{HOME}/.local + HINTS $ENV{HOME}/.local + PATH_SUFFIXES bin) macro(ROCPROFILER_REGISTER_ACTIVATE_CLANG_TIDY) if(ROCPROFILER_REGISTER_ENABLE_CLANG_TIDY) - if(NOT ROCPROFILER_REGISTER_CLANG_TIDY_COMMAND) + if(NOT ROCPROFILER_REGISTER_CLANG_TIDY_EXE) message( FATAL_ERROR "ROCPROFILER_REGISTER_ENABLE_CLANG_TIDY is ON but clang-tidy is not found!" ) endif() + rocprofiler_register_add_feature(ROCPROFILER_REGISTER_CLANG_TIDY_EXE + "path to clang-tidy executable") + set(CMAKE_CXX_CLANG_TIDY - ${ROCPROFILER_REGISTER_CLANG_TIDY_COMMAND} + ${ROCPROFILER_REGISTER_CLANG_TIDY_EXE} -header-filter=${PROJECT_SOURCE_DIR}/source/.* --warnings-as-errors=*,-misc-header-include-cycle) diff --git a/projects/rocprofiler-register/cmake/rocprofiler_register_options.cmake b/projects/rocprofiler-register/cmake/rocprofiler_register_options.cmake index 41e31bd088..790c0ca280 100644 --- a/projects/rocprofiler-register/cmake/rocprofiler_register_options.cmake +++ b/projects/rocprofiler-register/cmake/rocprofiler_register_options.cmake @@ -41,7 +41,9 @@ rocprofiler_register_add_option( ${ROCM_DEP_ROCMCORE}) # In the future, we will do this even with clang-tidy enabled -if(ROCPROFILER_REGISTER_BUILD_CI AND NOT ROCPROFILER_REGISTER_ENABLE_CLANG_TIDY) +if(ROCPROFILER_REGISTER_BUILD_CI + AND NOT ROCPROFILER_REGISTER_ENABLE_CLANG_TIDY + AND NOT ROCPROFILER_REGISTER_BUILD_DEVELOPER) message( STATUS "Forcing ROCPROFILER_REGISTER_BUILD_DEVELOPER=ON because ROCPROFILER_REGISTER_BUILD_CI=ON" diff --git a/projects/rocprofiler-register/external/CMakeLists.txt b/projects/rocprofiler-register/external/CMakeLists.txt index 339f0a02b3..1bd36497a5 100644 --- a/projects/rocprofiler-register/external/CMakeLists.txt +++ b/projects/rocprofiler-register/external/CMakeLists.txt @@ -15,7 +15,7 @@ if(ROCPROFILER_REGISTER_BUILD_GLOG) # May want to use GFLAGS in the future set(WITH_GFLAGS OFF) set(WITH_GTEST OFF) - set(WITH_UNWIND OFF) + set(WITH_UNWIND "none") add_subdirectory(glog EXCLUDE_FROM_ALL) else() find_package(glog REQUIRED) diff --git a/projects/rocprofiler-register/external/glog b/projects/rocprofiler-register/external/glog index 3a0d4d22c5..7b134a5c82 160000 --- a/projects/rocprofiler-register/external/glog +++ b/projects/rocprofiler-register/external/glog @@ -1 +1 @@ -Subproject commit 3a0d4d22c5ae0b9a2216988411cfa6bf860cc372 +Subproject commit 7b134a5c82c0c0b5698bb6bf7a835b230c5638e4 diff --git a/projects/rocprofiler-register/requirements.txt b/projects/rocprofiler-register/requirements.txt new file mode 100644 index 0000000000..58acea9e69 --- /dev/null +++ b/projects/rocprofiler-register/requirements.txt @@ -0,0 +1,5 @@ +black +clang-format>=11.0.0,<12.0.0 +clang-tidy>=15.0.0,<19.0.0 +cmake>=3.22.0 +cmake-format diff --git a/projects/rocprofiler-register/source/lib/rocprofiler-register/details/CMakeLists.txt b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/CMakeLists.txt index 71587dbd81..aa074aef5f 100644 --- a/projects/rocprofiler-register/source/lib/rocprofiler-register/details/CMakeLists.txt +++ b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/CMakeLists.txt @@ -1,10 +1,10 @@ # # builds the rocprofiler-register library # -set(rocprofiler_register_details_sources dl.cpp utility.cpp) +set(rocprofiler_register_details_sources dl.cpp logging.cpp utility.cpp) set(rocprofiler_register_details_headers environment.hpp dl.hpp filesystem.hpp - utility.hpp) + logging.hpp utility.hpp) target_sources(rocprofiler-register PRIVATE ${rocprofiler_register_details_sources} ${rocprofiler_register_details_headers}) diff --git a/projects/rocprofiler-register/source/lib/rocprofiler-register/details/environment.hpp b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/environment.hpp index 4250ed4de0..a0f10b5753 100644 --- a/projects/rocprofiler-register/source/lib/rocprofiler-register/details/environment.hpp +++ b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/environment.hpp @@ -20,8 +20,8 @@ #pragma once -#include "fmt/core.h" -#include "glog/logging.h" +#include +#include #include #include @@ -106,6 +106,21 @@ get_env_impl(std::string_view env_id, bool _default) } return _default; } + +inline int +set_env_impl(std::string_view env_id, bool value, int overwrite) +{ + return ::setenv(env_id.data(), (value) ? "1" : "0", overwrite); +} + +template +int +set_env_impl(std::string_view env_id, Tp value, int overwrite) +{ + auto str_value = std::stringstream{}; + str_value << value; + return ::setenv(env_id.data(), str_value.str().c_str(), overwrite); +} } // namespace template @@ -124,18 +139,25 @@ get_env(std::string_view env_id, Tp&& _default) } } +template +inline auto +set_env(std::string_view env_id, Tp&& value, int overwrite = 0) +{ + return set_env_impl(env_id, std::forward(value), overwrite); +} + struct env_config { std::string env_name = {}; std::string env_value = {}; - int override = 0; + int overwrite = 0; auto operator()() const { if(env_name.empty()) return -1; LOG(INFO) << fmt::format( - "setenv({}, {}, {})", env_name.c_str(), env_value.c_str(), override); - return setenv(env_name.c_str(), env_value.c_str(), override); + "setenv({}, {}, {})", env_name.c_str(), env_value.c_str(), overwrite); + return setenv(env_name.c_str(), env_value.c_str(), overwrite); } }; } // namespace common diff --git a/projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.cpp b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.cpp new file mode 100644 index 0000000000..3f2005b7cb --- /dev/null +++ b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.cpp @@ -0,0 +1,229 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "logging.hpp" +#include "environment.hpp" +#include "filesystem.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +namespace rocprofiler_register +{ +namespace logging +{ +namespace +{ +struct logging_config +{ + bool logtostderr = true; + bool alsologtostderr = false; + bool logdir_gitignore = false; // add .gitignore to logdir + int32_t loglevel = google::WARNING; + std::string vlog_modules = {}; + std::string name = {}; + std::string logdir = {}; +}; + +struct log_level_info +{ + int32_t google_level = 0; + int32_t verbose_level = 0; +}; + +void +update_logging(const logging_config& cfg, bool setup_env = false, int env_override = 0) +{ + static auto _mtx = std::mutex{}; + auto _lk = std::unique_lock{ _mtx }; + + FLAGS_timestamp_in_logfile_name = false; + FLAGS_logtostderr = cfg.logtostderr; + FLAGS_minloglevel = cfg.loglevel; + FLAGS_stderrthreshold = cfg.loglevel; + FLAGS_alsologtostderr = cfg.alsologtostderr; + + // if(!cfg.logdir.empty()) FLAGS_log_dir = cfg.logdir.c_str(); + + if(!cfg.logdir.empty() && !fs::exists(cfg.logdir)) + { + fs::create_directories(cfg.logdir); + + if(cfg.logdir_gitignore) + { + auto ignore = fs::path{ cfg.logdir } / ".gitignore"; + if(!fs::exists(ignore)) + { + std::ofstream ofs{ ignore.string() }; + ofs << "/**" << std::flush; + } + } + } + + if(setup_env) + { + common::set_env("GLOG_minloglevel", cfg.loglevel, env_override); + common::set_env("GLOG_logtostderr", cfg.logtostderr ? 1 : 0, env_override); + common::set_env( + "GLOG_alsologtostderr", cfg.alsologtostderr ? 1 : 0, env_override); + common::set_env("GLOG_stderrthreshold", cfg.loglevel, env_override); + if(!cfg.logdir.empty()) + { + common::set_env("GOOGLE_LOG_DIR", cfg.logdir, env_override); + common::set_env("GLOG_log_dir", cfg.logdir, env_override); + } + if(!cfg.vlog_modules.empty()) + common::set_env("GLOG_vmodule", cfg.vlog_modules, env_override); + } +} + +#define ROCP_REG_LOG_LEVEL_TRACE 4 +#define ROCP_REG_LOG_LEVEL_INFO 3 +#define ROCP_REG_LOG_LEVEL_WARNING 2 +#define ROCP_REG_LOG_LEVEL_ERROR 1 +#define ROCP_REG_LOG_LEVEL_NONE 0 + +void +init_logging(std::string_view env_prefix, logging_config cfg = logging_config{}) +{ + static auto _once = std::once_flag{}; + std::call_once(_once, [env_prefix, &cfg]() { + auto get_argv0 = []() { + auto ifs = std::ifstream{ "/proc/self/cmdline" }; + auto sarg = std::string{}; + while(ifs && !ifs.eof()) + { + ifs >> sarg; + if(!sarg.empty()) break; + } + return sarg; + }; + + auto to_lower = [](std::string val) { + for(auto& itr : val) + itr = tolower(itr); + return val; + }; + + const auto env_opts = std::unordered_map{ + { "trace", { google::INFO, ROCP_REG_LOG_LEVEL_TRACE } }, + { "info", { google::INFO, ROCP_REG_LOG_LEVEL_INFO } }, + { "warning", { google::WARNING, ROCP_REG_LOG_LEVEL_WARNING } }, + { "error", { google::ERROR, ROCP_REG_LOG_LEVEL_ERROR } }, + { "fatal", { google::FATAL, ROCP_REG_LOG_LEVEL_NONE } } + }; + + auto supported = std::vector{}; + supported.reserve(env_opts.size()); + for(auto itr : env_opts) + supported.emplace_back(itr.first); + + if(cfg.name.empty()) cfg.name = to_lower(std::string{ env_prefix }); + + cfg.logdir = common::get_env(fmt::format("{}_LOG_DIR", env_prefix), cfg.logdir); + cfg.vlog_modules = + common::get_env(fmt::format("{}_vmodule", env_prefix), cfg.vlog_modules); + cfg.logtostderr = cfg.logdir.empty(); // log to stderr if no log dir set + // cfg.alsologtostderr = !cfg.logdir.empty(); // log to file if log dir set + + auto loglvl = + to_lower(common::get_env(fmt::format("{}_LOG_LEVEL", env_prefix), "")); + // default to warning + auto& loglvl_v = cfg.loglevel; + if(!loglvl.empty() && + loglvl.find_first_not_of("-0123456789") == std::string::npos) + { + auto val = std::stol(loglvl); + if(val < 0) + { + loglvl_v = google::FATAL; + } + else + { + // default to trace in case val > ROCP_REG_LOG_LEVEL_TRACE + auto itr = env_opts.at("trace"); + for(auto oitr : env_opts) + { + if(oitr.second.verbose_level == val) + { + itr = oitr.second; + break; + } + } + loglvl_v = itr.google_level; + } + } + else if(!loglvl.empty()) + { + if(env_opts.find(loglvl) == env_opts.end()) + throw std::runtime_error{ fmt::format( + "invalid specifier for {}_LOG_LEVEL: {}. Supported: {}", + env_prefix, + loglvl, + fmt::format("{}", + fmt::join(supported.begin(), supported.end(), ", "))) }; + else + { + loglvl_v = env_opts.at(loglvl).google_level; + } + } + + update_logging(cfg, !google::IsGoogleLoggingInitialized()); + + if(!google::IsGoogleLoggingInitialized()) + { + static auto argv0 = get_argv0(); + // Prevent glog from crashing if vmodule is empty + if(FLAGS_vmodule.empty()) FLAGS_vmodule = " "; + + google::InitGoogleLogging(argv0.c_str()); + + // Swap out memory to avoid leaking the string + if(!FLAGS_vmodule.empty()) std::string{}.swap(FLAGS_vmodule); + if(!FLAGS_log_dir.empty()) std::string{}.swap(FLAGS_log_dir); + } + + update_logging(cfg); + + LOG(INFO) << "logging initialized via " << fmt::format("{}_LOG_LEVEL", env_prefix) + << ". Log Level: " << loglvl; + }); +} +} // namespace + +void +initialize() +{ + init_logging("ROCPROFILER_REGISTER"); +} +} // namespace logging +} // namespace rocprofiler_register diff --git a/projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.hpp b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.hpp new file mode 100644 index 0000000000..9f36d23b82 --- /dev/null +++ b/projects/rocprofiler-register/source/lib/rocprofiler-register/details/logging.hpp @@ -0,0 +1,37 @@ +// MIT License +// +// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include +#include +#include +#include + +namespace rocprofiler_register +{ +namespace logging +{ +void +initialize(); +} // namespace logging +} // namespace rocprofiler_register diff --git a/projects/rocprofiler-register/source/lib/rocprofiler-register/rocprofiler_register.cpp b/projects/rocprofiler-register/source/lib/rocprofiler-register/rocprofiler_register.cpp index 0905eddd66..ff95239ac0 100644 --- a/projects/rocprofiler-register/source/lib/rocprofiler-register/rocprofiler_register.cpp +++ b/projects/rocprofiler-register/source/lib/rocprofiler-register/rocprofiler_register.cpp @@ -25,7 +25,10 @@ #include "details/dl.hpp" #include "details/environment.hpp" #include "details/filesystem.hpp" -#include "glog/logging.h" +#include "details/logging.hpp" + +#include +#include #include #include @@ -92,8 +95,6 @@ static_assert(sizeof(bitset_t) == sizeof(rocprofiler_register_library_indentifier_t::handle), "bitset should be same at uint64_t"); -int rocprofiler_register_verbose = common::get_env("ROCPROFILER_REGISTER_VERBOSE", 0); -constexpr int rocprofiler_register_info_level = 2; constexpr auto rocprofiler_lib_name = "librocprofiler-sdk.so"; constexpr auto rocprofiler_lib_register_entrypoint = "rocprofiler_set_api_table"; constexpr auto rocprofiler_register_lib_name = @@ -338,9 +339,8 @@ rocp_load_rocprofiler_lib(std::string _rocp_reg_lib) dlopen(_rocp_reg_lib_path.c_str(), RTLD_GLOBAL | RTLD_LAZY); } - if(rocprofiler_register_verbose >= rocprofiler_register_info_level) - LOG(INFO) << "loaded " << _rocp_reg_lib_path_fname.string() << " library at " - << _rocp_reg_lib_path.string(); + LOG(INFO) << "loaded " << _rocp_reg_lib_path_fname.string() << " library at " + << _rocp_reg_lib_path.string(); LOG_IF(WARNING, rocprofiler_lib_handle == nullptr) << _rocp_reg_lib << " failed to load\n"; @@ -356,7 +356,7 @@ rocp_load_rocprofiler_lib(std::string _rocp_reg_lib) } constexpr auto library_seq = std::make_index_sequence{}; -auto global_mutex = std::recursive_mutex{}; +auto global_count = std::atomic{ 0 }; auto import_info = rocp_reg_get_imports(library_seq); auto instance_counters = std::array{}; } // namespace @@ -373,11 +373,33 @@ rocprofiler_register_library_api_table( { if(api_table_length < 1) return ROCP_REG_BAD_API_TABLE_LENGTH; - (void) lib_version; - (void) api_tables; + rocprofiler_register::logging::initialize(); - auto _lk = std::unique_lock{ global_mutex, std::defer_lock }; - if(_lk.owns_lock()) return ROCP_REG_DEADLOCK; + // rocprofiler-register is disabled via environment + if(!common::get_env("ROCPROFILER_REGISTER_ENABLED", true)) + { + LOG(INFO) << "rocprofiler-register disabled via ROCPROFILER_REGISTER_ENABLED=0"; + return ROCP_REG_NO_TOOLS; + } + + struct scoped_count + { + scoped_count() + : value{ ++global_count } + { } + + ~scoped_count() { --global_count; } + + scoped_count(const scoped_count&) = delete; + scoped_count(scoped_count&&) noexcept = delete; + scoped_count& operator=(const scoped_count&) = delete; + scoped_count& operator=(scoped_count&&) noexcept = delete; + + uint32_t value = 0; + }; + + auto _count = scoped_count{}; + if(_count.value > 1) return ROCP_REG_DEADLOCK; auto _scan_result = rocp_reg_scan_for_tools(); diff --git a/projects/rocprofiler-register/tests/CMakeLists.txt b/projects/rocprofiler-register/tests/CMakeLists.txt index d0da1bcc22..bb4fb53ab2 100644 --- a/projects/rocprofiler-register/tests/CMakeLists.txt +++ b/projects/rocprofiler-register/tests/CMakeLists.txt @@ -81,6 +81,11 @@ add_subdirectory(rocprofiler) # add_subdirectory(generic-tool) +# +# test app +# +add_subdirectory(bin) + # # common to multiple tests # diff --git a/projects/rocprofiler-register/tests/bin/CMakeLists.txt b/projects/rocprofiler-register/tests/bin/CMakeLists.txt new file mode 100644 index 0000000000..d655952445 --- /dev/null +++ b/projects/rocprofiler-register/tests/bin/CMakeLists.txt @@ -0,0 +1,108 @@ +# +# +# + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(Threads REQUIRED) +find_package( + hip + HINTS + ${ROCM_PATH} + ENV + ROCM_PATH + /opt/rocm + PATHS + ${ROCM_PATH} + ENV + ROCM_PATH + /opt/rocm) + +if(hip_FOUND) + add_executable(simple-hip) + target_sources(simple-hip PRIVATE simple-hip.cpp) + target_compile_options(simple-hip PRIVATE -W -Wall -Wextra -Werror) + target_link_libraries(simple-hip PRIVATE Threads::Threads hip::host) + set(PRELOAD_TESTS_DISABLED OFF) +else() + add_executable(simple-hip EXCLUDE_FROM_ALL) + target_sources(simple-hip PRIVATE simple-hip.cpp) + target_link_libraries(simple-hip PRIVATE Threads::Threads) + set(PRELOAD_TESTS_DISABLED ON) +endif() + +# +# INSTALL +# +set(TEST_DESTDIR "${CMAKE_BINARY_DIR}/tests/install") +set(simple-hip-install-env "DESTDIR=${TEST_DESTDIR}") + +add_test( + NAME test-simple-hip-install + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install --parallel 4 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + +set_tests_properties( + test-simple-hip-install + PROPERTIES TIMEOUT + 120 + LABELS + "integration-test" + ENVIRONMENT + "${simple-hip-install-env}" + DISABLED + ${PRELOAD_TESTS_DISABLED} + FIXTURES_SETUP + "simple-hip-install") + +# +# NORMAL (no profiling library) +# +string(REPLACE "//" "/" TEST_INSTALL_PREFIX "${TEST_DESTDIR}/${CMAKE_INSTALL_PREFIX}") +set(simple-hip-normal-env + "LD_LIBRARY_PATH=${TEST_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}:${ROCM_PATH}/${CMAKE_INSTALL_LIBDIR}" + ) + +add_test(NAME test-simple-hip-normal COMMAND $) + +set_tests_properties( + test-simple-hip-normal + PROPERTIES TIMEOUT + 120 + LABELS + "integration-test" + ENVIRONMENT + "${simple-hip-normal-env}" + DEPENDS + test-simple-hip-install + DISABLED + ${PRELOAD_TESTS_DISABLED} + FIXTURES_SETUP + "simple-hip-works" + FIXTURES_REQUIRED + "simple-hip-install") + +# +# PRELOAD +# +set(simple-hip-preload-env ${simple-hip-normal-env} + "LD_PRELOAD=$") + +add_test(NAME test-simple-hip-preload COMMAND $) + +set_tests_properties( + test-simple-hip-preload + PROPERTIES TIMEOUT + 120 + LABELS + "integration-test" + ENVIRONMENT + "${simple-hip-preload-env}" + DEPENDS + test-simple-hip-install + DISABLED + ${PRELOAD_TESTS_DISABLED} + FIXTURES_REQUIRED + "simple-hip-install;simple-hip-works") diff --git a/projects/rocprofiler-register/tests/bin/simple-hip.cpp b/projects/rocprofiler-register/tests/bin/simple-hip.cpp new file mode 100644 index 0000000000..99ebe7cca2 --- /dev/null +++ b/projects/rocprofiler-register/tests/bin/simple-hip.cpp @@ -0,0 +1,174 @@ +// MIT License +// +// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HIP_API_CALL(CALL) \ + if(hipError_t error_ = (CALL); error_ != hipSuccess) \ + { \ + auto _hip_api_print_lk = auto_lock_t{ print_lock }; \ + fprintf(stderr, \ + "%s:%d :: HIP error : %s\n", \ + __FILE__, \ + __LINE__, \ + hipGetErrorString(error_)); \ + fflush(stderr); \ + std::exit(EXIT_FAILURE); \ + } + +static constexpr int64_t Bi = 1; +static constexpr int64_t KiB = 1024 * Bi; +static constexpr int64_t MiB = 1024 * KiB; + +void +run(size_t rank, size_t tid, const char* prefix); + +namespace +{ +using auto_lock_t = std::unique_lock; +auto print_lock = std::mutex{}; +size_t xdim = 4960 * 2; +size_t ydim = 4960 * 2; +size_t nthreads = 2; +size_t nitr = 1; +} // namespace + +int +main(int argc, char** argv) +{ + for(int i = 1; i < argc; ++i) + { + auto _arg = std::string{ argv[i] }; + if(_arg == "?" || _arg == "-h" || _arg == "--help") + { + fprintf(stderr, + "usage: %s [M (%zu)] [N (%zu)] [NUM_THREADS (%zu)] [NUM_ITERATION " + "(%zu)]\n", + ::basename(argv[0]), + xdim, + ydim, + nthreads, + nitr); + exit(EXIT_SUCCESS); + } + } + + if(argc > 1) xdim = atoll(argv[1]); + if(argc > 2) ydim = atoll(argv[2]); + if(argc > 3) nthreads = atoll(argv[3]); + if(argc > 4) nitr = atoll(argv[4]); + + int ndevice = 0; + HIP_API_CALL(hipGetDeviceCount(&ndevice)); + + printf("[%s] Number of devices found: %i\n", ::basename(argv[0]), ndevice); + printf("[%s] Number of matrix: %zu x %zu\n", ::basename(argv[0]), xdim, ydim); + printf("[%s] Number of threads: %zu\n", ::basename(argv[0]), nthreads); + printf("[%s] Number of iterations: %zu\n", ::basename(argv[0]), nitr); + + for(size_t j = 0; j < nitr; ++j) + { + printf("\n[%s] Iteration #%zu\n\n", ::basename(argv[0]), j); + + auto threads = std::vector{}; + threads.reserve(nthreads); + + for(size_t i = 0; i < nthreads; ++i) + threads.emplace_back(run, getpid(), i, ::basename(argv[0])); + + for(auto& itr : threads) + itr.join(); + } + + return 0; +} + +template +auto +allocate_memory(size_t M, size_t N, Tp val) +{ + const auto sz = M * N; + const auto nb = sz * sizeof(Tp); + + Tp* ptr = new Tp[M * N]; + ::memset(ptr, val, nb); + + HIP_API_CALL(hipHostRegister(ptr, nb, hipHostRegisterDefault)) + + return ptr; +} + +template +auto +deallocate_memory(Tp* ptr) +{ + HIP_API_CALL(hipHostUnregister(ptr)) + delete[] ptr; +} + +void +run(size_t rank, size_t tid, const char* label) +{ + const auto M = xdim; + const auto N = ydim; + + auto* inp = allocate_memory(M, N, 0); + auto* out = allocate_memory(M, N, 1); + + auto _engine = + std::default_random_engine{ std::random_device{}() * (rank + 1) * (tid + 1) }; + std::uniform_int_distribution _dist{ 0, 1000 }; + + for(size_t i = 0; i < (M * N); i++) + inp[i] = _dist(_engine); + + // lock during malloc to get more accurate memory info + { + auto_lock_t _lk{ print_lock }; + size_t free_gpu_mem = 0; + size_t total_gpu_mem = 0; + + HIP_API_CALL(hipMemGetInfo(&free_gpu_mem, &total_gpu_mem)); + free_gpu_mem /= MiB; + total_gpu_mem /= MiB; + + std::cout << "[" << label << "][" << rank << "][" << tid + << "] Available GPU memory (MiB): " << std::setw(6) << free_gpu_mem + << " / " << std::setw(6) << total_gpu_mem << std::endl; + } + + deallocate_memory(inp); + deallocate_memory(out); +}