AI/rocm-systems
2
0
Atdalīts 0
Kods Problēmas Izmaiņu pieprasījumi Darbības Pakotnes Projekti Laidieni Vikivietne Aktivitāte
Files
32b15fe7b74c34a40ae838c8f5f3159e22d0c0ff
rocm-systems/source/lib/omnitrace/library/rocm.cpp
T
Jonathan R. Madsen 32b15fe7b7 Handle fork in target application (#191) 
* Always print PID in log messages

* omnitrace-dl updates

- omnitrace_preload does not call omnitrace_init or omnitrace_init_tooling
- omnitrace_preload will call omnitrace_set_mpi if OMNITRACE_USE_MPI
  or OMNITRACE_USE_MPIP in the env is true but not call it otherwise
  because doing so either overrides OMNITRACE_USE_PID (when true) or
  disable mpip from initialization (when false) and the MPI
  init can be caught later and override OMNITRACE_USE_PID

* config updates

- set_setting_value sets user update type
- remove volatile from get_settings_configured
- don't override settings::default_process_suffix
- don't kill process in omnitrace_exit_action
- set_state ignores updating state if >= State::Finalized

* Handle state > State::Finalized

* fork gotcha updates

- unsets LD_PRELOAD
- sets OMNITRACE_ROOT_PROCESS
- sets OMNITRACE_CHILD_PROCESS

* libomnitrace library.cpp updates

- basic_bundle for fini metrics
- handle finalization from child process

* sampling updates

- sampling::shutdown handles when child process

* Add example and test using fork

* Update run-ci script to support not submitting

* Tweak test envs

* Update build flags when codecov enabled

* remove unnecessary includes of sampling header

* Replace mpi copy/fini static lambda with free-funcs

* Update codecov job

* Fix OMPT segfaults after finalization

* Miscellaneous updates after rebase

* fixes for causal profiling

* revert some run-ci.sh changes

* Disable storing env in sampling::shutdown

* formatting fix

* Update timemory submodule

- fixed occasional synchronization issues with allocator offloading
- exclude protozero:: from internal samples

* improve root/child process detection

- avoid omnitrace_finalize in MPI when child process
- revert some testing tweaks
2023-02-08 01:31:38 -06:00

353 rindas
13 KiB
C++
Neapstrādāts Vainot Vēsture

// MIT License
//
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
#include "library/rocm.hpp"
#include "core/config.hpp"
#include "core/debug.hpp"
#include "core/dynamic_library.hpp"
#include "core/gpu.hpp"
#include "library/components/rocprofiler.hpp"
#include "library/components/roctracer.hpp"
#include "library/rocm/hsa_rsrc_factory.hpp"
#include "library/rocm_smi.hpp"
#include "library/rocprofiler.hpp"
#include "library/roctracer.hpp"
#include "library/runtime.hpp"
#include "library/thread_data.hpp"
#include "library/tracing.hpp"
#include <timemory/backends/cpu.hpp>
#include <timemory/backends/threading.hpp>
#include <timemory/utility/types.hpp>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <mutex>
#include <tuple>
#define HIP_PROF_HIP_API_STRING 1
#include <roctracer_ext.h>
#include <roctracer_hip.h>
#if OMNITRACE_HIP_VERSION < 50300
# include <roctracer_hcc.h>
#endif
#define AMD_INTERNAL_BUILD 1
#include <roctracer_hsa.h>
#if __has_include(<hip/amd_detail/hip_prof_str.h>) || (defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0)
# include <hip/amd_detail/hip_prof_str.h>
# define OMNITRACE_HIP_API_ARGS 1
#else
# define OMNITRACE_HIP_API_ARGS 0
#endif
#if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
# include <rocprofiler.h>
#endif
using namespace omnitrace;
namespace omnitrace
{
namespace rocm
{
std::mutex rocm_mutex = {};
bool is_loaded = false;
bool on_load_trace = (get_env<int>("ROCP_ONLOAD_TRACE", 0) > 0);
} // namespace rocm
} // namespace omnitrace
#if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
std::ostream&
operator<<(std::ostream& _os, const rocprofiler_settings_t& _v)
{
# define ROCPROF_SETTING_FIELD_STR(NAME) JOIN('=', # NAME, _v.NAME)
_os << JOIN(
", ", ROCPROF_SETTING_FIELD_STR(intercept_mode),
ROCPROF_SETTING_FIELD_STR(code_obj_tracking),
ROCPROF_SETTING_FIELD_STR(memcopy_tracking),
ROCPROF_SETTING_FIELD_STR(trace_size), ROCPROF_SETTING_FIELD_STR(trace_local),
ROCPROF_SETTING_FIELD_STR(timeout), ROCPROF_SETTING_FIELD_STR(timestamp_on),
ROCPROF_SETTING_FIELD_STR(hsa_intercepting),
ROCPROF_SETTING_FIELD_STR(k_concurrent), ROCPROF_SETTING_FIELD_STR(opt_mode),
ROCPROF_SETTING_FIELD_STR(obj_dumping));
return _os;
}
#endif
// HSA-runtime tool on-load method
extern "C"
{
#if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
void OnUnloadTool()
{
OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Unloading...\n");
rocm::lock_t _lk{ rocm::rocm_mutex, std::defer_lock };
if(!_lk.owns_lock()) _lk.lock();
if(!rocm::is_loaded)
{
OMNITRACE_BASIC_VERBOSE_F(1 || rocm::on_load_trace,
"rocprofiler is not loaded\n");
return;
}
rocm::is_loaded = false;
_lk.unlock();
// stop_top_level_timer_if_necessary();
// Final resources cleanup
omnitrace::rocprofiler::rocm_cleanup();
}
void OnLoadToolProp(rocprofiler_settings_t* settings)
{
OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Loading...\n");
rocm::lock_t _lk{ rocm::rocm_mutex, std::defer_lock };
if(!_lk.owns_lock()) _lk.lock();
if(rocm::is_loaded)
{
OMNITRACE_BASIC_VERBOSE_F(1 || rocm::on_load_trace,
"rocprofiler is already loaded\n");
return;
}
rocm::is_loaded = true;
_lk.unlock();
// Enable timestamping
settings->timestamp_on = 1;
settings->intercept_mode = 1;
settings->hsa_intercepting = 1;
settings->k_concurrent = 0;
settings->obj_dumping = 0;
// settings->code_obj_tracking = 0;
// settings->memcopy_tracking = 0;
// settings->trace_local = 1;
// settings->opt_mode = 1;
// settings->trace_size = 0;
// settings->timeout = 0;
OMNITRACE_BASIC_VERBOSE_F(1 || rocm::on_load_trace, "rocprofiler settings: %s\n",
JOIN("", *settings).c_str());
// Initialize profiling
omnitrace::rocprofiler::rocm_initialize();
HsaRsrcFactory::Instance().PrintGpuAgents("ROCm");
}
#endif
bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count,
const char* const* failed_tool_names)
{
tim::consume_parameters(table, runtime_version, failed_tool_count,
failed_tool_names);
OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Loading...\n");
OMNITRACE_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
if(!tim::get_env("OMNITRACE_INIT_TOOLING", true)) return true;
if(!tim::settings::enabled()) return true;
roctracer_is_init() = true;
OMNITRACE_BASIC_VERBOSE_F(1 || rocm::on_load_trace, "Loading ROCm tooling...\n");
if(!config::settings_are_configured() && get_state() < State::Active)
omnitrace_init_tooling_hidden();
OMNITRACE_SCOPED_THREAD_STATE(ThreadState::Internal);
static auto _setup = [=]() {
try
{
OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
"[OnLoad] setting up HSA...\n");
bool trace_hsa_api = get_trace_hsa_api();
// Enable HSA API callbacks/activity
if(trace_hsa_api)
{
std::vector<std::string> hsa_api_vec =
tim::delimit(get_trace_hsa_api_types());
// initialize HSA tracing
roctracer_set_properties(ACTIVITY_DOMAIN_HSA_API, (void*) table);
if(!hsa_api_vec.empty())
{
for(const auto& itr : hsa_api_vec)
{
uint32_t cid = HSA_API_ID_NUMBER;
const char* api = itr.c_str();
OMNITRACE_ROCTRACER_CALL(roctracer_op_code(
ACTIVITY_DOMAIN_HSA_API, api, &cid, nullptr));
OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_callback(
ACTIVITY_DOMAIN_HSA_API, cid, hsa_api_callback, nullptr));
OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
" HSA-trace(%s)", api);
}
}
else
{
OMNITRACE_VERBOSE(1 || rocm::on_load_trace, " HSA-trace()\n");
OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback(
ACTIVITY_DOMAIN_HSA_API, hsa_api_callback, nullptr));
}
}
bool trace_hsa_activity = get_trace_hsa_activity();
// Enable HSA GPU activity
if(trace_hsa_activity)
{
#if OMNITRACE_HIP_VERSION < 50300
using namespace roctracer;
// initialize HSA tracing
const char* output_prefix = nullptr;
hsa_ops_properties_t ops_properties{
table,
reinterpret_cast<activity_async_callback_t>(
hsa_activity_callback),
nullptr, output_prefix
};
#elif OMNITRACE_HIP_VERSION < 50301
hsa_ops_properties_t ops_properties;
ops_properties.table = table;
ops_properties.reserved1[0] =
reinterpret_cast<void*>(&hsa_activity_callback);
ops_properties.reserved1[1] = nullptr;
ops_properties.reserved1[2] = nullptr;
#else
hsa_ops_properties_t ops_properties{
table, reinterpret_cast<void*>(&hsa_activity_callback), nullptr,
nullptr
};
#endif
roctracer_set_properties(ACTIVITY_DOMAIN_HSA_OPS, &ops_properties);
OMNITRACE_VERBOSE(1 || rocm::on_load_trace,
" HSA-activity-trace()\n");
OMNITRACE_ROCTRACER_CALL(roctracer_enable_op_activity(
ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
}
} catch(std::exception& _e)
{
OMNITRACE_BASIC_PRINT("Exception was thrown in HSA setup: %s\n",
_e.what());
}
};
static auto _shutdown = []() {
OMNITRACE_DEBUG_F("roctracer_disable_domain_callback\n");
OMNITRACE_ROCTRACER_CALL(
roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HSA_API));
OMNITRACE_DEBUG_F("roctracer_disable_op_activity\n");
OMNITRACE_ROCTRACER_CALL(
roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
};
#if OMNITRACE_HIP_VERSION < 50300
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Computing the roctracer clock skew...\n");
(void) omnitrace::get_clock_skew();
#endif
comp::roctracer::add_setup("hsa", _setup);
comp::roctracer::add_shutdown("hsa", _shutdown);
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Setting rocm_smi state to active...\n");
rocm_smi::set_state(State::Active);
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Requesting roctracer to setup...\n");
comp::roctracer::setup();
#if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
bool _force_rocprofiler_init =
tim::get_env("OMNITRACE_FORCE_ROCPROFILER_INIT", false, false);
#else
bool _force_rocprofiler_init = false;
#endif
bool _success = true;
bool _is_empty =
(config::settings_are_configured() && config::get_rocm_events().empty());
if(_force_rocprofiler_init || (get_use_rocprofiler() && !_is_empty))
{
auto _rocprof =
dynamic_library{ "OMNITRACE_ROCPROFILER_LIBRARY",
find_library_path("librocprofiler64.so",
{ "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
{ OMNITRACE_DEFAULT_ROCM_PATH },
{ "lib", "lib64", "rocprofiler/lib",
"rocprofiler/lib64" }),
(RTLD_LAZY | RTLD_GLOBAL), false };
OMNITRACE_VERBOSE_F(1 || rocm::on_load_trace,
"Loading rocprofiler library (%s=%s)...\n",
_rocprof.envname.c_str(), _rocprof.filename.c_str());
_rocprof.open();
on_load_t _rocprof_load = nullptr;
_success = _rocprof.invoke("OnLoad", _rocprof_load, table, runtime_version,
failed_tool_count, failed_tool_names);
OMNITRACE_CONDITIONAL_PRINT_F(!_success,
"Warning! Invoking rocprofiler's OnLoad "
"failed! OMNITRACE_ROCPROFILER_LIBRARY=%s\n",
_rocprof.filename.c_str());
OMNITRACE_CI_THROW(!_success,
"Warning! Invoking rocprofiler's OnLoad "
"failed! OMNITRACE_ROCPROFILER_LIBRARY=%s\n",
_rocprof.filename.c_str());
}
else
{
HsaRsrcFactory::Instance().PrintGpuAgents("ROCm");
}
gpu::add_hip_device_metadata();
OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Loading... %s\n",
(_success) ? "Done" : "Failed");
return _success;
}
// HSA-runtime on-unload method
void OnUnload()
{
OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Unloading...\n");
omnitrace_finalize_hidden();
OMNITRACE_BASIC_VERBOSE_F(2 || rocm::on_load_trace, "Unloading... Done\n");
}
}
Atsaukties uz šo jaunā problēmā Aplūkot Git vainīgos Kopēt saiti