From f5a309980180533b604f53406dbdce5d3ef247ad Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Tue, 21 Jun 2022 00:28:20 -0500 Subject: [PATCH] Fix attaching to running process, i.e. omnitrace -p (#60) * Fix attaching to a process - e.g. omnitrace -p * Update /proc/sys/kernel/yama/ptrace_scope in CI * Query /proc/sys/kernel/yama/ptrace_scope * Use AUTHOR_WARNING instead of WARNING for ptrace_scope [ROCm/rocprofiler-systems commit: fe3d15a6c9c1e3782a5f322096c79f8763def805] --- .../source/bin/omnitrace/omnitrace.cpp | 60 +++++++++++------- .../rocprofiler-systems/tests/CMakeLists.txt | 62 +++++++++++++++++++ .../tests/run-omnitrace-pid.sh | 45 ++++++++++++++ 3 files changed, 144 insertions(+), 23 deletions(-) create mode 100755 projects/rocprofiler-systems/tests/run-omnitrace-pid.sh diff --git a/projects/rocprofiler-systems/source/bin/omnitrace/omnitrace.cpp b/projects/rocprofiler-systems/source/bin/omnitrace/omnitrace.cpp index b29b0d6f99..b64339150e 100644 --- a/projects/rocprofiler-systems/source/bin/omnitrace/omnitrace.cpp +++ b/projects/rocprofiler-systems/source/bin/omnitrace/omnitrace.cpp @@ -1000,20 +1000,20 @@ main(int argc, char** argv) // for runtime instrumentation, we need to set this before the process gets created if(!binary_rewrite) { + auto _hsa_val = tim::get_env("HSA_ENABLE_INTERRUPT", 1, false); tim::set_env("HSA_ENABLE_INTERRUPT", "0", 0); - if(_pid >= 0) + if(_pid >= 0 && _hsa_val != 0) { - verbprintf(-10, "#-------------------------------------------------------" - "-------------------------------------------#\n"); - verbprintf(-10, "\n"); - verbprintf(-10, "WARNING! Sampling may result in ioctl() deadlock within " - "the ROCR runtime.\n"); - verbprintf(-10, - "To avoid this, set HSA_ENABLE_INTERRUPT=0 in the environment " - "before starting your ROCm/HIP application\n"); - verbprintf(-10, "\n"); - verbprintf(-10, "#-------------------------------------------------------" - "-------------------------------------------#\n"); + verbprintf(0, "#-------------------------------------------------------" + "-------------------------------------------#\n"); + verbprintf(0, "\n"); + verbprintf(0, "WARNING! Sampling may result in ioctl() deadlock within " + "the ROCR runtime.\n"); + verbprintf(0, "To avoid this, set HSA_ENABLE_INTERRUPT=0 in the environment " + "before starting your ROCm/HIP application\n"); + verbprintf(0, "\n"); + verbprintf(0, "#-------------------------------------------------------" + "-------------------------------------------#\n"); } } @@ -2023,7 +2023,20 @@ main(int argc, char** argv) WIFSIGNALED(status)); \ } - if(!app_thread->isTerminated()) + auto _compute_exit_code = [app_thread, &code]() { + if(app_thread->terminationStatus() == ExitedNormally) + { + if(app_thread->isTerminated()) verbprintf(0, "End of omnitrace\n"); + } + else if(app_thread->terminationStatus() == ExitedViaSignal) + { + auto sign = app_thread->getExitSignal(); + fprintf(stderr, "\nApplication exited with signal: %i\n", int(sign)); + } + code = app_thread->getExitCode(); + }; + + if(!app_thread->isTerminated() && !is_attached) { pid_t cpid = app_thread->getPid(); int status = 0; @@ -2056,18 +2069,19 @@ main(int argc, char** argv) } } while(WIFEXITED(status) == 0 && WIFSIGNALED(status) == 0); } + else if(!app_thread->isTerminated() && is_attached) + { + app_thread->continueExecution(); + while(!app_thread->isTerminated()) + { + while(bpatch->waitForStatusChange()) + app_thread->continueExecution(); + } + _compute_exit_code(); + } else { - if(app_thread->terminationStatus() == ExitedNormally) - { - if(app_thread->isTerminated()) verbprintf(0, "End of omnitrace\n"); - } - else if(app_thread->terminationStatus() == ExitedViaSignal) - { - auto sign = app_thread->getExitSignal(); - fprintf(stderr, "\nApplication exited with signal: %i\n", int(sign)); - } - code = app_thread->getExitCode(); + _compute_exit_code(); } } diff --git a/projects/rocprofiler-systems/tests/CMakeLists.txt b/projects/rocprofiler-systems/tests/CMakeLists.txt index 502a336852..f8d0add95c 100644 --- a/projects/rocprofiler-systems/tests/CMakeLists.txt +++ b/projects/rocprofiler-systems/tests/CMakeLists.txt @@ -98,6 +98,22 @@ set(_python_environment "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" "PYTHONPATH=${PROJECT_BINARY_DIR}/lib/python/site-packages") +set(_attach_environment + "OMNITRACE_USE_PERFETTO=ON" + "OMNITRACE_USE_TIMEMORY=ON" + "OMNITRACE_USE_SAMPLING=OFF" + "OMNITRACE_USE_THREAD_SAMPLING=ON" + "OMNITRACE_USE_CRITICAL_TRACE=ON" + "OMNITRACE_USE_OMPT=ON" + "OMNITRACE_USE_KOKKOSP=ON" + "OMNITRACE_TIME_OUTPUT=OFF" + "OMNITRACE_USE_PID=OFF" + "OMP_PROC_BIND=spread" + "OMP_PLACES=threads" + "OMP_NUM_THREADS=2" + "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}:${OMNITRACE_DYNINST_API_RT_DIR}:$ENV{LD_LIBRARY_PATH}" + ) + # -------------------------------------------------------------------------------------- # set(MPIEXEC_EXECUTABLE_ARGS) @@ -804,6 +820,52 @@ omnitrace_add_test( RUNTIME_PASS_REGEX "(\\\[[0-9]+\\\]) function coverage :: 66.67%" REWRITE_RUN_PASS_REGEX "(\\\[[0-9]+\\\]) function coverage :: 66.67%") +# -------------------------------------------------------------------------------------- # +# +# attach tests +# +# -------------------------------------------------------------------------------------- # + +set(_VALID_PTRACE_SCOPE OFF) +if(EXISTS "/proc/sys/kernel/yama/ptrace_scope") + file(READ "/proc/sys/kernel/yama/ptrace_scope" _PTRACE_SCOPE LIMIT 1) + if("${_PTRACE_SCOPE}" EQUAL 0) + set(_VALID_PTRACE_SCOPE ON) + endif() +else() + omnitrace_message( + AUTHOR_WARNING + "Disabling attach tests. Run 'echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope' to enable attaching to process" + ) +endif() + +if(TARGET parallel-overhead AND _VALID_PTRACE_SCOPE) + add_test( + NAME parallel-overhead-attach + COMMAND + ${CMAKE_CURRENT_LIST_DIR}/run-omnitrace-pid.sh $ + -ME "\.c$" -e -v 1 --label return args file -l -- + $ 30 8 1000 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + + set(_parallel_overhead_attach_environ + "${_attach_environment}" "OMNITRACE_OUTPUT_PATH=omnitrace-tests-output" + "OMNITRACE_OUTPUT_PREFIX=parallel-overhead-attach/") + + set_tests_properties( + parallel-overhead-attach + PROPERTIES ENVIRONMENT + "${_parallel_overhead_attach_environ}" + TIMEOUT + 300 + LABELS + "parallel-overhead;attach" + PASS_REGULAR_EXPRESSION + "Outputting.*(perfetto-trace.proto).*Outputting.*(wall_clock.txt)" + FAIL_REGULAR_EXPRESSION + "Dyninst was unable to attach to the specified process") +endif() + # -------------------------------------------------------------------------------------- # # # python tests diff --git a/projects/rocprofiler-systems/tests/run-omnitrace-pid.sh b/projects/rocprofiler-systems/tests/run-omnitrace-pid.sh new file mode 100755 index 0000000000..4cc4354439 --- /dev/null +++ b/projects/rocprofiler-systems/tests/run-omnitrace-pid.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +cleanup() +{ + kill -s 9 ${_PID} +} + +trap cleanup SIGABRT SIGQUIT + +OMNITRACE_COMMAND="" +APPLICATION_COMMAND="" + +while [[ $# -gt 0 ]] +do + if [ "${1}" == "--" ]; then + shift + break + else + OMNITRACE_COMMAND="${OMNITRACE_COMMAND}${1} " + shift + fi +done + +${@} & +_PID=$! + +echo "" +ps ax | grep "${1}" +echo "" + +if [ "${_PID}" = "" ]; then + echo "Error! No PID for \"${@}\"" + exit -1 +fi + +echo "PID: ${_PID}" +echo "" + +${OMNITRACE_COMMAND} -p ${_PID} -- ${@} +RET=$? + +killall $(basename ${1}) &> /dev/null + +echo "Exiting with code: ${RET}" +exit ${RET}