diff --git a/projects/rocprofiler-systems/source/lib/core/config.cpp b/projects/rocprofiler-systems/source/lib/core/config.cpp index 3441a99f56..26deb03581 100644 --- a/projects/rocprofiler-systems/source/lib/core/config.cpp +++ b/projects/rocprofiler-systems/source/lib/core/config.cpp @@ -838,6 +838,18 @@ configure_settings(bool _init) "starting with '_' or containing '::_M'.", true, "causal", "analysis", "advanced"); + ROCPROFSYS_CONFIG_SETTING(int, "ROCPROFSYS_KILL_DELAY", + "Delay (in seconds) before terminating the process " + "after a kill signal is received.", + 0, "process", "advanced"); + + auto kill_delay_config = _config->find("ROCPROFSYS_KILL_DELAY")->second; + auto kill_delay_value = kill_delay_config->get().second; + if(kill_delay_value < 0) + { + kill_delay_config->set(0); + } + // set the defaults _config->get_flamegraph_output() = false; _config->get_ctest_notes() = false; @@ -2556,6 +2568,13 @@ get_caching_perfetto() return static_cast&>(*_v).get(); } +int +get_kill_delay() +{ + static auto _v = get_config()->find("ROCPROFSYS_KILL_DELAY"); + return static_cast&>(*_v->second).get(); +} + tmp_file::tmp_file(std::string _v) : filename{ std::move(_v) } {} diff --git a/projects/rocprofiler-systems/source/lib/core/config.hpp b/projects/rocprofiler-systems/source/lib/core/config.hpp index ddde939d13..49982e72d6 100644 --- a/projects/rocprofiler-systems/source/lib/core/config.hpp +++ b/projects/rocprofiler-systems/source/lib/core/config.hpp @@ -358,6 +358,9 @@ get_trace_thread_join(); bool get_use_tmp_files(); +int +get_kill_delay(); + std::string get_tmpdir(); diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/CMakeLists.txt b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/CMakeLists.txt index fb758625fd..a1a30d73f1 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/CMakeLists.txt +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/CMakeLists.txt @@ -8,6 +8,7 @@ set(component_sources ${CMAKE_CURRENT_LIST_DIR}/cpu_freq.cpp ${CMAKE_CURRENT_LIST_DIR}/exit_gotcha.cpp ${CMAKE_CURRENT_LIST_DIR}/fork_gotcha.cpp + ${CMAKE_CURRENT_LIST_DIR}/kill_gotcha.cpp ${CMAKE_CURRENT_LIST_DIR}/mpi_gotcha.cpp ${CMAKE_CURRENT_LIST_DIR}/numa_gotcha.cpp ${CMAKE_CURRENT_LIST_DIR}/vaapi_gotcha.cpp @@ -27,6 +28,7 @@ set(component_headers ${CMAKE_CURRENT_LIST_DIR}/ensure_storage.hpp ${CMAKE_CURRENT_LIST_DIR}/exit_gotcha.hpp ${CMAKE_CURRENT_LIST_DIR}/fork_gotcha.hpp + ${CMAKE_CURRENT_LIST_DIR}/kill_gotcha.hpp ${CMAKE_CURRENT_LIST_DIR}/mpip.hpp ${CMAKE_CURRENT_LIST_DIR}/mpi_gotcha.hpp ${CMAKE_CURRENT_LIST_DIR}/numa_gotcha.hpp diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/kill_gotcha.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/kill_gotcha.cpp new file mode 100644 index 0000000000..9d5f823665 --- /dev/null +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/kill_gotcha.cpp @@ -0,0 +1,71 @@ +// MIT License +// +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "kill_gotcha.hpp" +#include "core/config.hpp" +#include "core/debug.hpp" + +#include +#include + +namespace rocprofsys +{ +namespace component +{ +void +kill_gotcha::configure() +{ + kill_gotcha_t::get_initializer() = []() { + kill_gotcha_t::configure<0, int, pid_t, int>("kill"); + }; +} + +int +kill_gotcha::operator()(const gotcha_data& _data, kill_func_t _func, pid_t _pid, + int _sig) const +{ + // When profiling multi-process applications where a parent process sends SIGKILL to + // child processes, the termination can occur before the profiler has a chance to + // flush collected data. This introduces a configurable delay before SIGKILL + // signals are forwarded, allowing profiling data to be captured before process + // termination. + // NOTE: This is a workaround. + + auto kill_delay = get_kill_delay(); + if(kill_delay > 0) + { + auto _self_pid = getpid(); + + if(_sig == SIGKILL && _pid != _self_pid && _pid > 0) + { + ROCPROFSYS_DEBUG("[kill_gotcha] Intercepted '%s(%d, SIGKILL)' triggered from " + "process with id: %d. Sleeping for %d seconds...\n", + _data.tool_id.c_str(), _pid, _self_pid, kill_delay); + + ::sleep(kill_delay); + } + } + + return (*_func)(_pid, _sig); +} +} // namespace component +} // namespace rocprofsys diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/kill_gotcha.hpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/kill_gotcha.hpp new file mode 100644 index 0000000000..9a93f6ac0e --- /dev/null +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/components/kill_gotcha.hpp @@ -0,0 +1,59 @@ +// MIT License +// +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once + +#include "core/defines.hpp" + +#include +#include + +#include +#include + +namespace rocprofsys +{ +namespace component +{ +struct kill_gotcha : tim::component::base +{ + static constexpr size_t gotcha_capacity = 1; + + using gotcha_data = tim::component::gotcha_data; + using kill_func_t = int (*)(pid_t, int); + + ROCPROFSYS_DEFAULT_OBJECT(kill_gotcha) + + static std::string label() { return "kill_gotcha"; } + + static void configure(); + + static inline void start() {} + static inline void stop() {} + + int operator()(const gotcha_data&, kill_func_t, pid_t, int) const; +}; +} // namespace component + +using kill_gotcha_t = tim::component::gotcha, component::kill_gotcha>; +} // namespace rocprofsys diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.cpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.cpp index c80b4172ce..df56740424 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.cpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.cpp @@ -198,6 +198,7 @@ setup_gotchas() component::mpi_gotcha::configure(); component::exit_gotcha::configure(); component::fork_gotcha::configure(); + component::kill_gotcha::configure(); } } // namespace diff --git a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.hpp b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.hpp index 5bb8af63ef..3274f37509 100644 --- a/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.hpp +++ b/projects/rocprofiler-systems/source/lib/rocprof-sys/library/runtime.hpp @@ -30,6 +30,7 @@ #include "library/causal/components/causal_gotcha.hpp" #include "library/components/exit_gotcha.hpp" #include "library/components/fork_gotcha.hpp" +#include "library/components/kill_gotcha.hpp" #include "library/components/mpi_gotcha.hpp" #include "library/components/numa_gotcha.hpp" #include "library/components/pthread_gotcha.hpp" @@ -49,7 +50,7 @@ namespace rocprofsys { // started during preinit phase using preinit_bundle_t = - tim::lightweight_tuple; + tim::lightweight_tuple; // started during init phase using init_bundle_t = tim::lightweight_tuple