From 055a3fba87a23e32c84702aa885e1f954ea43113 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Thu, 2 Sep 2021 13:14:58 -0500 Subject: [PATCH] Updated documentation + misc (#3) - tweaked some CMake option names - moved merge-trace.jl to hosttrace-merge.jl - removed Windows line encodings from hosttrace-merge.jl - improved handling of !perfetto and !timemory [ROCm/rocprofiler-systems commit: fc15967f8f125a945f1bd646386a4c6d11ed95f3] --- projects/rocprofiler-systems/CMakeLists.txt | 36 +++- projects/rocprofiler-systems/README.md | 160 ++++++++++++++---- .../cmake/BuildSettings.cmake | 42 ++--- .../{merge-trace.jl => hosttrace-merge.jl} | 85 +++++----- projects/rocprofiler-systems/src/library.cpp | 17 +- 5 files changed, 230 insertions(+), 110 deletions(-) rename projects/rocprofiler-systems/scripts/{merge-trace.jl => hosttrace-merge.jl} (96%) diff --git a/projects/rocprofiler-systems/CMakeLists.txt b/projects/rocprofiler-systems/CMakeLists.txt index 0defe5a4bc..bbb503dca5 100644 --- a/projects/rocprofiler-systems/CMakeLists.txt +++ b/projects/rocprofiler-systems/CMakeLists.txt @@ -31,6 +31,12 @@ add_option(CMAKE_CXX_STANDARD_REQUIRED "Require C++ language standard" ON) add_option(CMAKE_CXX_EXTENSIONS "Compiler specific language extensions" OFF) add_option(CMAKE_INSTALL_RPATH_USE_LINK_PATH "Enable rpath to linked libraries" ON) add_option(HOSTTRACE_USE_CLANG_TIDY "Enable clang-tidy" OFF) +add_option(HOSTTRACE_USE_MPI "Enable MPI support" OFF) +add_option(HOSTTRACE_CUSTOM_DATA_SOURCE "Enable custom data source" OFF) + +# ensure synced +set(TIMEMORY_USE_MPI ${HOSTTRACE_USE_MPI} CACHE BOOL "Enable MPI support" FORCE) +set(TRANSPOSE_USE_MPI ${HOSTTRACE_USE_MPI} CACHE BOOL "Enable MPI support" FORCE) include(Packages) # finds third-party libraries @@ -42,8 +48,6 @@ hosttrace_activate_clang_tidy() # #------------------------------------------------------------------------------# -option(HOSTTRACE_CUSTOM_DATA_SOURCE "Enable custom data source" OFF) - add_library(hosttrace-library SHARED ${CMAKE_CURRENT_LIST_DIR}/src/library.cpp ${CMAKE_CURRENT_LIST_DIR}/src/libmisc.cpp @@ -61,12 +65,11 @@ target_compile_definitions(hosttrace-library PRIVATE target_link_libraries(hosttrace-library PRIVATE hosttrace::hosttrace-threading + hosttrace::hosttrace-compile-options $ $ $ - $ - $ - $,hosttrace::hosttrace-sanitizer,>) + $,hosttrace::hosttrace-sanitizer,>) if(DYNINST_API_RT) get_filename_component(DYNINST_API_RT_DIR "${DYNINST_API_RT}" DIRECTORY) @@ -96,10 +99,10 @@ target_include_directories(hosttrace-exe PRIVATE ${CMAKE_CURRENT_LIST_DIR}/include) target_link_libraries(hosttrace-exe PRIVATE - $ hosttrace::hosttrace-dyninst hosttrace::hosttrace-compile-options - $,hosttrace::hosttrace-sanitizer,>) + $ + $,hosttrace::hosttrace-sanitizer,>) set_target_properties(hosttrace-exe PROPERTIES OUTPUT_NAME hosttrace @@ -110,6 +113,25 @@ install( DESTINATION ${CMAKE_INSTALL_BINDIR} OPTIONAL) +# build the timemory-avail exe +add_dependencies(hosttrace-exe timemory-avail) + +#------------------------------------------------------------------------------# +# +# miscellaneous installs +# +#------------------------------------------------------------------------------# + +install( + PROGRAMS ${PROJECT_SOURCE_DIR}/scripts/hosttrace-merge.jl + DESTINATION ${CMAKE_INSTALL_BINDIR} + OPTIONAL) + +install( + FILES ${PROJECT_SOURCE_DIR}/roctrace.cfg + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR} + OPTIONAL) + #------------------------------------------------------------------------------# # # examples diff --git a/projects/rocprofiler-systems/README.md b/projects/rocprofiler-systems/README.md index 67abb66310..9c6d19144d 100755 --- a/projects/rocprofiler-systems/README.md +++ b/projects/rocprofiler-systems/README.md @@ -1,63 +1,151 @@ -# hosttrace: application tracing with static/dynamic binary instrumentation and perfetto +# hosttrace: application tracing with static/dynamic binary instrumentation +## Dependencies -## 1. load necessary modules -``` -module load julia -module load dyninst -module load rocm/VERSION +- [DynInst](https://github.com/dyninst/dyninst) for dynamic or static instrumentation +- [Julia](https://julialang.org/) for merging perfetto traces + +## Installing DynInst + +The easiest way to install Dyninst is via spack + +```console +git clone https://github.com/spack/spack.git +source ./spack/share/spack/setup-env.sh +spack compiler find +spack external find +spack install dyninst +spack load -r dyninst ``` -## 2. to install julia packages, only need it the first time. -``` -$ julia -type ']' to get into pkg mode, then -(@v1.6) pkg> add JSON, DataFrames, Dates, CSV, Chain, PrettyTables +## Installing Julia + +Julia is available via Linux package managers or may be available via a module. Debian-based distributions such as Ubuntu can run (as a super-user): + +```console +apt-get install julia ``` -## 3. hosttrace usage +Once Julia is installed, install the necessary packages (this operation only needs to be performed once): + +```console +julia -e 'using Pkg; for name in ["JSON", "DataFrames", "Dates", "CSV", "Chain", "PrettyTables"]; Pkg.add(name); end' ``` -export PATH=$PATH:$HTRACE_PATH/bin + +## Installing hosttrace + +```console +HOSTTRACE_ROOT=${HOME}/sw/hosttrace +git clone https://github.com/AARInternal/hosttrace-dyninst.git +cmake -B build-hosttrace -DHOSTTRACE_USE_MPI=ON -DCMAKE_INSTALL_PREFIX=${HOSTTRACE_ROOT} hosttrace-dyninst +cmake --build build-hosttrace --target all --parallel 8 +cmake --build build-hosttrace --target install +export PATH=${HOSTTRACE_ROOT}/bin:${PATH} +export LD_LIBRARY_PATH=${HOSTTRACE_ROOT}/lib64:${HOSTTRACE_ROOT}/lib:${LD_LIBRARY_PATH} +``` + +## Using hosttrace + +```console hosttrace --help +hosttrace -- ``` -## 4. instrument the binaries -``` -hosttrace -L $HTRACE_PATH/bin/libhosttrace.so -o app.inst -- path_to_your_app +### Example Instrumentation -hosttrace -L $HTRACE_PATH/bin/libhosttrace.so -E 'hipApiName|hipGetCmdName' -o libamdhip64.so.4 -- /opt/rocm-VERSION/lib/libamdhip64.so.4 +#### Binary Rewrite -hosttrace -L $HTRACE_PATH/bin/libhosttrace.so -E 'rocr::atomic|rocr::core|rocr::HSA' -o libhsa-runtime64.so.1 -- /opt/rocm-VERSION/lib/libhsa-runtime64.so.1 +Rewrite the text section of an executable with instrumentation: + +```console +hosttrace -o app.inst -- /path/to/app ``` -## 5. run the app + +In binary rewrite mode, if you also want instrumentation in the linked libraries, you must also rewrite those libraries. +Example of rewriting the functions starting with `"hip"` with instrumentation in the amdhip64 library: + +```console +mkdir -p ./lib +hosttrace -R '^hip' -o ./lib/libamdhip64.so.4 -- /opt/rocm/lib/libamdhip64.so.4 +export LD_LIBRARY_PATH=${PWD}/lib:${LD_LIBRARY_PATH} ``` -export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH + +> NOTE: Verify via `ldd` that your executable will load the instrumented library -- if you built your executable with +> an RPATH to the original library's directory, then prefixing `LD_LIBRARY_PATH` will have no effect. + +Once you have rewritten your executable and/or libraries with instrumentation, you can just run the (instrumented) executable +or exectuable which loads the instrumented libraries normally, e.g.: + +```console +./app.inst rocprof --hip-trace --roctx-trace --stats ./app.inst ``` -## 6. you may need to increase the buffer size if the hosttrace.perfetto-trace file is close to 1GB. for example to set it to 10GB. -``` -export HOSTTRACE_BUFFER_SIZE_KB=10240000 +#### Runtime Instrumentation + +Runtime instrumentation will not only instrument the text section of the executable but also the text sections of the +linked libraries. Thus, it may be useful to exclude those libraries via the `-ME` (module exclude) regex option. + +```console +hosttrace -- /path/to/app +hosttrace -ME '^(libhsa-runtime64|libz\\.so)' -- /path/to/app +hosttrace -E 'rocr::atomic|rocr::core|rocr::HSA' -- /path/to/app ``` -## 7. merge the traces from rocprof and hosttrace -``` -julia $HTRACE_PATH/bin/merge-trace.jl results.json hosttrace.perfetto-trace* +## Miscellaneous Features and Caveats + +- You may need to increase the default perfetto buffer size (1 GB) to capture all the information + - E.g. `export HOSTTRACE_BUFFER_SIZE_KB=10240000` +- Perfetto tooling is enabled by default +- Timemory tooling is disabled by default +- Enabling/disabling one of the aformentioned tools but not specifying enabling/disable the other will assume the inverse of the other's enabled state, e.g. + - `HOSTTRACE_USE_PERFETTO=OFF` yields the same result `HOSTTRACE_USE_TIMEMORY=ON` + - `HOSTTRACE_USE_PERFETTO=ON` yields the same result as `HOSTTRACE_USE_TIMEMORY=OFF` + - In order to enable _both_ timemory and perfetto, set both `HOSTTRACE_USE_TIMEMORY=ON` and `HOSTTRACE_USE_PERFETTO=ON` + - Setting `HOSTTRACE_USE_TIMEMORY=OFF` and `HOSTTRACE_USE_PERFETTO=OFF` will disable all instrumentation +- Use `timemory-avail -S` to view the various settings for timemory +- Set `HOSTTRACE_COMPONENTS=""` to control which components timemory collects + - The list of components and their descriptions can be viewed via `timemory-avail -Cd` + - The list of components and their string identifiers can be view via `timemory-avail -Cbs` +- You can filter any `timemory-avail` results via `-r -hl` + +## Hosttrace Output + +`hosttrace` will create an output directory named `hosttrace--output`, e.g. if your executable +is named `app.inst`, the output directory will be `hosttrace-app.inst-output`. Depending on whether +`TIMEMORY_TIME_OUTPUT=ON` (the default when perfetto is enabled), there will be a subdirectory with the date and time, +e.g. `2021-09-02_01.03_PM`. Within this directory, all perfetto files will be named `perfetto-trace..proto` or +when `HOSTTRACE_USE_MPI=ON`, `perfetto-trace..proto` (assuming hosttrace was built with MPI support). + +You can explicitly control the output path and naming scheme of the files via the `HOSTTRACE_OUTPUT_FILE` environment +variable. The special character sequences `%pid%` and `%rank%` will be replaced with the PID or MPI rank, respectively. + +## Merging the traces from rocprof and hosttrace + +Use the `hosttrace-merge.jl` Julia script to merge rocprof and perfetto traces. + +```console +hosttrace-merge.jl results.json hosttrace-app.inst-output/2021-09-02_01.03_PM/*.proto ``` -## 8. another mode of Perfetto tracing is to use system backend. To do it: +## Use Perfetto tracing with System Backend -### in a separate window -``` - pkill traced; traced --background; perfetto --out ./htrace.out --txt -c $HTRACE/roctrace.cfg +In a separate window run: + +```console +pkill traced +traced --background +perfetto --out ./htrace.out --txt -c $HTRACE/roctrace.cfg ``` -### then in the app running window do this before running rocprof or other app -``` - export HOSTTRACE_BACKEND_SYSTEM=1 +then in the window running the application, configure the hosttrace instrumentation to use the system backend: + +```console +export HOSTTRACE_BACKEND_SYSTEM=1 ``` -### for the merge use the htrace.out +for the merge use the `htrace.out`: + +```console +hosttrace-merge.jl results.json htrace.out ``` -julia $HTRACE_PATH/bin/merge-trace.jl results.json htrace.out -``` \ No newline at end of file diff --git a/projects/rocprofiler-systems/cmake/BuildSettings.cmake b/projects/rocprofiler-systems/cmake/BuildSettings.cmake index 209a64ca82..ef2624ac4c 100644 --- a/projects/rocprofiler-systems/cmake/BuildSettings.cmake +++ b/projects/rocprofiler-systems/cmake/BuildSettings.cmake @@ -12,16 +12,16 @@ include(Compilers) include(FindPackageHandleStandardArgs) include(MacroUtilities) -option(hosttrace_BUILD_DEVELOPER "Extra build flags for development like -Werror" OFF) -option(hosttrace_BUILD_EXTRA_OPTIMIZATIONS "Extra optimization flags" OFF) -option(hosttrace_BUILD_LTO "Build with link-time optimization" OFF) -option(hosttrace_USE_COMPILE_TIMING "" OFF) -option(hosttrace_USE_COVERAGE "" OFF) -option(hosttrace_USE_SANITIZER "" OFF) +option(HOSTTRACE_BUILD_DEVELOPER "Extra build flags for development like -Werror" OFF) +option(HOSTTRACE_BUILD_EXTRA_OPTIMIZATIONS "Extra optimization flags" OFF) +option(HOSTTRACE_BUILD_LTO "Build with link-time optimization" OFF) +option(HOSTTRACE_USE_COMPILE_TIMING "" OFF) +option(HOSTTRACE_USE_COVERAGE "" OFF) +option(HOSTTRACE_USE_SANITIZER "" OFF) target_compile_definitions(hosttrace-compile-options INTERFACE $<$:DEBUG>) -set(hosttrace_SANITIZER_TYPE "leak" CACHE STRING "Sanitizer type") +set(HOSTTRACE_SANITIZER_TYPE "leak" CACHE STRING "Sanitizer type") #----------------------------------------------------------------------------------------# # dynamic linking and runtime libraries @@ -108,7 +108,7 @@ endif() # non-debug optimizations # add_interface_library(hosttrace-compile-extra "Extra optimization flags") -if(NOT hosttrace_USE_COVERAGE) +if(NOT HOSTTRACE_USE_COVERAGE) add_target_flag_if_avail(hosttrace-compile-extra "-finline-functions" "-funroll-loops" @@ -117,7 +117,7 @@ if(NOT hosttrace_USE_COVERAGE) "-ftree-loop-vectorize") endif() -if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" AND hosttrace_BUILD_EXTRA_OPTIMIZATIONS) +if(NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" AND HOSTTRACE_BUILD_EXTRA_OPTIMIZATIONS) target_link_libraries(hosttrace-compile-options INTERFACE $) add_flag_if_avail( @@ -136,7 +136,7 @@ endif() # add_cxx_flag_if_avail("-faligned-new") -if(hosttrace_BUILD_LTO) +if(HOSTTRACE_BUILD_LTO) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) endif() @@ -159,7 +159,7 @@ else() target_link_options(hosttrace-lto INTERFACE -flto=thin) endif() -if(hosttrace_BUILD_LTO) +if(HOSTTRACE_BUILD_LTO) target_link_libraries(hosttrace-compile-options INTERFACE hosttrace::hosttrace-lto) endif() @@ -180,7 +180,7 @@ else() add_target_flag_if_avail(hosttrace-compile-timing "-ftime-report") endif() -if(hosttrace_USE_COMPILE_TIMING) +if(HOSTTRACE_USE_COMPILE_TIMING) target_link_libraries(hosttrace-compile-options INTERFACE hosttrace-compile-timing) endif() @@ -192,7 +192,7 @@ endif() # developer build flags # add_interface_library(hosttrace-develop-options "Adds developer compiler flags") -if(hosttrace_BUILD_DEVELOPER) +if(HOSTTRACE_BUILD_DEVELOPER) add_target_flag_if_avail(hosttrace-develop-options # "-Wabi" "-Wdouble-promotion" @@ -235,16 +235,16 @@ endif() #----------------------------------------------------------------------------------------# # sanitizer # -set(hosttrace_SANITIZER_TYPES address memory thread leak undefined unreachable null bounds alignment) -set_property(CACHE hosttrace_SANITIZER_TYPE PROPERTY STRINGS "${hosttrace_SANITIZER_TYPES}") +set(HOSTTRACE_SANITIZER_TYPES address memory thread leak undefined unreachable null bounds alignment) +set_property(CACHE HOSTTRACE_SANITIZER_TYPE PROPERTY STRINGS "${HOSTTRACE_SANITIZER_TYPES}") add_interface_library(hosttrace-sanitizer-compile-options "Adds compiler flags for sanitizers") add_interface_library(hosttrace-sanitizer - "Adds compiler flags to enable ${hosttrace_SANITIZER_TYPE} sanitizer (-fsanitizer=${hosttrace_SANITIZER_TYPE})") + "Adds compiler flags to enable ${HOSTTRACE_SANITIZER_TYPE} sanitizer (-fsanitizer=${HOSTTRACE_SANITIZER_TYPE})") set(COMMON_SANITIZER_FLAGS "-fno-optimize-sibling-calls" "-fno-omit-frame-pointer" "-fno-inline-functions") add_target_flag(hosttrace-sanitizer-compile-options ${COMMON_SANITIZER_FLAGS}) -foreach(_TYPE ${hosttrace_SANITIZER_TYPES}) +foreach(_TYPE ${HOSTTRACE_SANITIZER_TYPES}) set(_FLAG "-fsanitize=${_TYPE}") add_interface_library(hosttrace-${_TYPE}-sanitizer "Adds compiler flags to enable ${_TYPE} sanitizer (${_FLAG})") @@ -258,8 +258,8 @@ endforeach() unset(_FLAG) unset(COMMON_SANITIZER_FLAGS) -if(hosttrace_USE_SANITIZER) - foreach(_TYPE ${hosttrace_SANITIZER_TYPE}) +if(HOSTTRACE_USE_SANITIZER) + foreach(_TYPE ${HOSTTRACE_SANITIZER_TYPE}) if(TARGET hosttrace-${_TYPE}-sanitizer) target_link_libraries(hosttrace-sanitizer INTERFACE hosttrace-${_TYPE}-sanitizer) else() @@ -267,8 +267,8 @@ if(hosttrace_USE_SANITIZER) endif() endforeach() else() - set(hosttrace_USE_SANITIZER OFF) - inform_empty_interface(hosttrace-sanitizer "${hosttrace_SANITIZER_TYPE} sanitizer") + set(HOSTTRACE_USE_SANITIZER OFF) + inform_empty_interface(hosttrace-sanitizer "${HOSTTRACE_SANITIZER_TYPE} sanitizer") endif() if (MSVC) diff --git a/projects/rocprofiler-systems/scripts/merge-trace.jl b/projects/rocprofiler-systems/scripts/hosttrace-merge.jl similarity index 96% rename from projects/rocprofiler-systems/scripts/merge-trace.jl rename to projects/rocprofiler-systems/scripts/hosttrace-merge.jl index b3017efd93..fad4e9c5b6 100755 --- a/projects/rocprofiler-systems/scripts/merge-trace.jl +++ b/projects/rocprofiler-systems/scripts/hosttrace-merge.jl @@ -1,42 +1,43 @@ -using Base: String -using JSON, DataFrames, Dates, CSV - -function check_input_args() - message = "Needs only one json file and at least one trace data file(s)." - @assert length(ARGS) >= 2 && occursin(".json", join(ARGS)) message - json_file, trace_files = nothing, String[] - for arg in ARGS - @assert Base.Filesystem.ispath(arg) "'$(arg)' not found!" - if endswith(arg, ".json") - json_file = arg - else - push!(trace_files, arg) - end - end - json_file, trace_files -end - -json_file, trace_files = check_input_args() - -out = JSON.parsefile(json_file) -for trace in trace_files - @info "processing $(trace) ..." - f = tempname() * ".json" - run(`trace_to_text json $trace $f`) - todo = JSON.parsefile(f) - out["traceEvents"] = vcat( - out["traceEvents"], - get(todo, "traceEvents", []) - ) - out["systemTraceEvents"] = (*)( - get(out, "systemTraceEvents", ""), - get(todo, "systemTraceEvents", "") - ) -end - -output_file = replace("$(string(now())).json", ":" => "-") -open(output_file, "w") do f - write(f, JSON.json(out)) - @info "output file: $(output_file)" -end - +#!/usr/bin/env julia + +using Base: String +using JSON, DataFrames, Dates, CSV + +function check_input_args() + message = "Needs only one json file and at least one trace data file(s)." + @assert length(ARGS) >= 2 && occursin(".json", join(ARGS)) message + json_file, trace_files = nothing, String[] + for arg in ARGS + @assert Base.Filesystem.ispath(arg) "'$(arg)' not found!" + if endswith(arg, ".json") + json_file = arg + else + push!(trace_files, arg) + end + end + json_file, trace_files +end + +json_file, trace_files = check_input_args() + +out = JSON.parsefile(json_file) +for trace in trace_files + @info "processing $(trace) ..." + f = tempname() * ".json" + run(`trace_to_text json $trace $f`) + todo = JSON.parsefile(f) + out["traceEvents"] = vcat( + out["traceEvents"], + get(todo, "traceEvents", []) + ) + out["systemTraceEvents"] = (*)( + get(out, "systemTraceEvents", ""), + get(todo, "systemTraceEvents", "") + ) +end + +output_file = replace("$(string(now())).json", ":" => "-") +open(output_file, "w") do f + write(f, JSON.json(out)) + @info "output file: $(output_file)" +end diff --git a/projects/rocprofiler-systems/src/library.cpp b/projects/rocprofiler-systems/src/library.cpp index 11d98d6176..7e1bc26efe 100644 --- a/projects/rocprofiler-systems/src/library.cpp +++ b/projects/rocprofiler-systems/src/library.cpp @@ -148,13 +148,22 @@ get_functors() } bool -hosttrace_init_perfetto() +hosttrace_init_tooling() { if(get_state() != State::PreInit) return false; HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__); + if(!get_use_timemory() && !get_use_perfetto()) + { + get_state() = State::Finalized; + HOSTTRACE_DEBUG("[%s] Both perfetto and timemory are disabled. Setting the state " + "to finalized\n", + __FUNCTION__); + return false; + } + // always initialize timemory because gotcha wrappers are always used tim::settings::flamegraph_output() = false; tim::settings::cout_output() = false; @@ -346,7 +355,7 @@ extern "C" if(get_state() == State::Finalized) return; - if(get_state() != State::Active && !hosttrace_init_perfetto()) + if(get_state() != State::Active && !hosttrace_init_tooling()) { HOSTTRACE_DEBUG("[%s] %s :: not active and perfetto not initialized\n", __FUNCTION__, name); @@ -376,7 +385,7 @@ extern "C" void hosttrace_trace_init(const char*, bool, const char*) { HOSTTRACE_DEBUG("[%s]\n", __FUNCTION__); - hosttrace_init_perfetto(); + hosttrace_init_tooling(); } void hosttrace_trace_finalize(void) @@ -485,6 +494,6 @@ namespace { // if static objects are destroyed randomly (relatively uncommon behavior) // this might call finalization before perfetto ends the tracing session -// but static variable in hosttrace_init_perfetto is more likely +// but static variable in hosttrace_init_tooling is more likely auto _ensure_finalization = ensure_finalization(); } // namespace