Critical trace updates (#24)
* Source code restructuring * Critical trace updates following restructuring * thread_sampler, timestamps - thread_sampler - CPU frequency managed via thread_sampler - rocm-smi managed via thread_sampler - Use consistent timestamps for perfetto - removed hsa_timer_t in favor of wall_clock::record() - disable KokkosP by default - re-enable critical-trace testing * cmake-format * Fix for defines.hpp.in * Remove OMNITRACE_ROCM_SMI_FREQ - thread_sampler freq is set via OMNITRACE_SAMPLING_FREQ w/ max of 1000 * Increase CI Install Dyninst timeout * Debug macros + omnitrace_init_tooling + config - new debug macros - extern "C" omnitrace_init_tooling - guard get_rocm_smi_devices * Miscellaneous tweaks - tweak to transpose - critical_trace::Device::ANY - perfetto "critical-trace" category - OMNITRACE_VERBOSE usage * Disable key and tid data for HIP API calls - non-kernels are ignored in activity callback * critical-trace exe updates - fix perfetto generation - improved logging - improved readability * timemory submodule update - lulesh example cmake tweaks
This commit is contained in:
Jonathan R. Madsen
committed by
GitHub
GitHub
parent
39f17ae8b8
commit
b016c8929f
@@ -0,0 +1,120 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/common.hpp"
|
||||
#include "library/components/fwd.hpp"
|
||||
#include "library/defines.hpp"
|
||||
#include "library/thread_data.hpp"
|
||||
#include "library/timemory.hpp"
|
||||
|
||||
#include <timemory/components/base.hpp>
|
||||
#include <timemory/components/papi/papi_array.hpp>
|
||||
#include <timemory/macros/language.hpp>
|
||||
#include <timemory/mpl/concepts.hpp>
|
||||
#include <timemory/sampling/sampler.hpp>
|
||||
#include <timemory/variadic/types.hpp>
|
||||
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
namespace component
|
||||
{
|
||||
struct backtrace
|
||||
: tim::component::empty_base
|
||||
, tim::concepts::component
|
||||
{
|
||||
static constexpr size_t num_hw_counters = 8;
|
||||
|
||||
using data_t = std::array<char[512], 128>;
|
||||
using clock_type = std::chrono::steady_clock;
|
||||
using time_point_type = typename clock_type::time_point;
|
||||
using value_type = void;
|
||||
using hw_counters = tim::component::papi_array<num_hw_counters>;
|
||||
using hw_counter_data_t = typename hw_counters::value_type;
|
||||
using system_clock = std::chrono::system_clock;
|
||||
using system_time_point = typename system_clock::time_point;
|
||||
|
||||
static void preinit();
|
||||
static std::string label();
|
||||
static std::string description();
|
||||
|
||||
backtrace() = default;
|
||||
~backtrace() = default;
|
||||
backtrace(backtrace&&) = default;
|
||||
backtrace(const backtrace&) = default;
|
||||
|
||||
backtrace& operator=(const backtrace&) = default;
|
||||
backtrace& operator=(backtrace&&) = default;
|
||||
|
||||
bool operator<(const backtrace& rhs) const;
|
||||
|
||||
static std::set<int> configure(bool, int64_t _tid = threading::get_id());
|
||||
static void post_process(int64_t _tid = threading::get_id());
|
||||
static hw_counter_data_t& get_last_hwcounters();
|
||||
|
||||
static void start();
|
||||
static void stop();
|
||||
void sample(int = -1);
|
||||
bool empty() const;
|
||||
size_t size() const;
|
||||
std::vector<std::string> get() const;
|
||||
time_point_type get_timestamp() const;
|
||||
int64_t get_thread_cpu_timestamp() const;
|
||||
|
||||
private:
|
||||
int64_t m_tid = 0;
|
||||
int64_t m_thr_cpu_ts = 0;
|
||||
int64_t m_mem_peak = 0;
|
||||
size_t m_size = 0;
|
||||
time_point_type m_ts = {};
|
||||
data_t m_data = {};
|
||||
hw_counter_data_t m_hw_counter = {};
|
||||
};
|
||||
} // namespace component
|
||||
} // namespace omnitrace
|
||||
|
||||
#if !defined(OMNITRACE_EXTERN_COMPONENTS) || \
|
||||
(defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0)
|
||||
|
||||
# include <timemory/operations.hpp>
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(
|
||||
TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_wall_clock>), true,
|
||||
double)
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(
|
||||
TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_cpu_clock>), true,
|
||||
double)
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(
|
||||
TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_fraction>), true,
|
||||
double)
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,52 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/common.hpp"
|
||||
#include "library/defines.hpp"
|
||||
#include "library/timemory.hpp"
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
// this is used to wrap fork()
|
||||
struct fork_gotcha : comp::base<fork_gotcha, void>
|
||||
{
|
||||
using gotcha_data_t = comp::gotcha_data;
|
||||
|
||||
TIMEMORY_DEFAULT_OBJECT(fork_gotcha)
|
||||
|
||||
// string id for component
|
||||
static std::string label() { return "fork_gotcha"; }
|
||||
|
||||
// generate the gotcha wrappers
|
||||
static void configure();
|
||||
|
||||
// this will get called right before fork
|
||||
static void audit(const gotcha_data_t& _data, audit::incoming);
|
||||
|
||||
// this will get called right after fork with the return value
|
||||
static void audit(const gotcha_data_t& _data, audit::outgoing, pid_t _pid);
|
||||
};
|
||||
|
||||
using fork_gotcha_t = comp::gotcha<4, tim::component_tuple<fork_gotcha>, api::omnitrace>;
|
||||
} // namespace omnitrace
|
||||
@@ -0,0 +1,188 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/defines.hpp"
|
||||
|
||||
#include <timemory/components/data_tracker/types.hpp>
|
||||
#include <timemory/components/macros.hpp>
|
||||
#include <timemory/components/user_bundle/types.hpp>
|
||||
#include <timemory/enum.h>
|
||||
#include <timemory/mpl/concepts.hpp>
|
||||
|
||||
TIMEMORY_DECLARE_COMPONENT(roctracer)
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
namespace component
|
||||
{
|
||||
template <typename... Tp>
|
||||
using data_tracker = tim::component::data_tracker<Tp...>;
|
||||
|
||||
struct omnitrace;
|
||||
struct backtrace;
|
||||
struct backtrace_wall_clock
|
||||
{};
|
||||
struct backtrace_cpu_clock
|
||||
{};
|
||||
struct backtrace_fraction
|
||||
{};
|
||||
struct backtrace_gpu_busy
|
||||
{};
|
||||
struct backtrace_gpu_temp
|
||||
{};
|
||||
struct backtrace_gpu_power
|
||||
{};
|
||||
struct backtrace_gpu_memory
|
||||
{};
|
||||
using sampling_wall_clock = data_tracker<double, backtrace_wall_clock>;
|
||||
using sampling_cpu_clock = data_tracker<double, backtrace_cpu_clock>;
|
||||
using sampling_percent = data_tracker<double, backtrace_fraction>;
|
||||
using sampling_gpu_busy = data_tracker<double, backtrace_gpu_busy>;
|
||||
using sampling_gpu_temp = data_tracker<double, backtrace_gpu_temp>;
|
||||
using sampling_gpu_power = data_tracker<double, backtrace_gpu_power>;
|
||||
using sampling_gpu_memory = data_tracker<double, backtrace_gpu_memory>;
|
||||
using roctracer = tim::component::roctracer;
|
||||
} // namespace component
|
||||
} // namespace omnitrace
|
||||
|
||||
#if !defined(OMNITRACE_USE_ROCTRACER)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, component::roctracer, false_type)
|
||||
#endif
|
||||
|
||||
#if !defined(TIMEMORY_USE_LIBUNWIND)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::api::sampling, false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::backtrace, false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_wall_clock,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_cpu_clock,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_percent,
|
||||
false_type)
|
||||
#endif
|
||||
|
||||
#if !defined(TIMEMORY_USE_LIBUNWIND) || !defined(OMNITRACE_USE_ROCM_SMI)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_busy,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_temp,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_power,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, omnitrace::component::sampling_gpu_memory,
|
||||
false_type)
|
||||
#endif
|
||||
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::omnitrace, OMNITRACE_COMPONENT,
|
||||
"omnitrace", "omnitrace_component")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::roctracer, OMNITRACE_ROCTRACER,
|
||||
"roctracer", "omnitrace_roctracer")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_wall_clock,
|
||||
OMNITRACE_SAMPLING_WALL_CLOCK, "sampling_wall_clock", "")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_cpu_clock,
|
||||
OMNITRACE_SAMPLING_CPU_CLOCK, "sampling_cpu_clock", "")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_percent,
|
||||
OMNITRACE_SAMPLING_PERCENT, "sampling_percent", "")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_busy,
|
||||
OMNITRACE_SAMPLING_GPU_BUSY, "sampling_gpu_busy",
|
||||
"sampling_gpu_util")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_memory,
|
||||
OMNITRACE_SAMPLING_GPU_MEMORY_USAGE,
|
||||
"sampling_gpu_memory_usage", "")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_power,
|
||||
OMNITRACE_SAMPLING_GPU_POWER, "sampling_gpu_power", "")
|
||||
TIMEMORY_PROPERTY_SPECIALIZATION(omnitrace::component::sampling_gpu_temp,
|
||||
OMNITRACE_SAMPLING_GPU_TEMP, "sampling_gpu_temp", "")
|
||||
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::roctracer, "roctracer",
|
||||
"High-precision ROCm API and kernel tracing", "")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_wall_clock,
|
||||
"sampling_wall_clock", "Wall-clock timing",
|
||||
"Derived from statistical sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_cpu_clock,
|
||||
"sampling_cpu_clock", "CPU-clock timing",
|
||||
"Derived from statistical sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_percent,
|
||||
"sampling_percent",
|
||||
"Fraction of wall-clock time spent in functions",
|
||||
"Derived from statistical sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_busy,
|
||||
"sampling_gpu_busy",
|
||||
"GPU Utilization (% busy) via ROCm-SMI",
|
||||
"Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_memory,
|
||||
"sampling_gpu_memory_usage",
|
||||
"GPU Memory Usage via ROCm-SMI", "Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_power,
|
||||
"sampling_gpu_power", "GPU Power Usage via ROCm-SMI",
|
||||
"Derived from sampling")
|
||||
TIMEMORY_METADATA_SPECIALIZATION(omnitrace::component::sampling_gpu_temp,
|
||||
"sampling_gpu_temp", "GPU Temperature via ROCm-SMI",
|
||||
"Derived from sampling")
|
||||
|
||||
// statistics type
|
||||
TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_wall_clock, double)
|
||||
TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_cpu_clock, double)
|
||||
TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_busy, double)
|
||||
TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_temp, double)
|
||||
TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_power, double)
|
||||
TIMEMORY_STATISTICS_TYPE(omnitrace::component::sampling_gpu_memory, double)
|
||||
|
||||
// enable timing units
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category,
|
||||
omnitrace::component::sampling_wall_clock, true_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category,
|
||||
omnitrace::component::sampling_cpu_clock, true_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, omnitrace::component::sampling_percent,
|
||||
true_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units,
|
||||
omnitrace::component::sampling_wall_clock, true_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units,
|
||||
omnitrace::component::sampling_cpu_clock, true_type)
|
||||
|
||||
// enable percent units
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_percent_units,
|
||||
omnitrace::component::sampling_gpu_busy, true_type)
|
||||
|
||||
// enable memory units
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_memory_category,
|
||||
omnitrace::component::sampling_gpu_memory, true_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_memory_units,
|
||||
omnitrace::component::sampling_gpu_memory, true_type)
|
||||
|
||||
// reporting categories (sum)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_busy,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_temp,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_power,
|
||||
false_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(report_sum, omnitrace::component::sampling_gpu_memory,
|
||||
false_type)
|
||||
|
||||
// reporting categories (mean)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(report_mean, omnitrace::component::sampling_percent,
|
||||
false_type)
|
||||
|
||||
// reporting categories (stats)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(report_statistics, omnitrace::component::sampling_percent,
|
||||
false_type)
|
||||
@@ -0,0 +1,69 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/common.hpp"
|
||||
#include "library/defines.hpp"
|
||||
#include "library/timemory.hpp"
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
// this is used to wrap MPI_Init and MPI_Init_thread
|
||||
struct mpi_gotcha : comp::base<mpi_gotcha, void>
|
||||
{
|
||||
using comm_t = tim::mpi::comm_t;
|
||||
using gotcha_data_t = comp::gotcha_data;
|
||||
|
||||
TIMEMORY_DEFAULT_OBJECT(mpi_gotcha)
|
||||
|
||||
// string id for component
|
||||
static std::string label() { return "mpi_gotcha"; }
|
||||
|
||||
// generate the gotcha wrappers
|
||||
static void configure();
|
||||
|
||||
// called right before MPI_Init with that functions arguments
|
||||
static void audit(const gotcha_data_t& _data, audit::incoming, int*, char***);
|
||||
|
||||
// called right before MPI_Init_thread with that functions arguments
|
||||
static void audit(const gotcha_data_t& _data, audit::incoming, int*, char***, int,
|
||||
int*);
|
||||
|
||||
// called right before MPI_Finalize
|
||||
static void audit(const gotcha_data_t& _data, audit::incoming);
|
||||
|
||||
// called right before MPI_Comm_{rank,size} with that functions arguments
|
||||
void audit(const gotcha_data_t& _data, audit::incoming, comm_t, int*);
|
||||
|
||||
// called right after MPI_{Init,Init_thread,Comm_rank,Comm_size} with the return value
|
||||
void audit(const gotcha_data_t& _data, audit::outgoing, int _retval);
|
||||
|
||||
private:
|
||||
int* m_rank_ptr = nullptr;
|
||||
int* m_size_ptr = nullptr;
|
||||
int m_rank = 0;
|
||||
int m_size = 1;
|
||||
};
|
||||
|
||||
using mpi_gotcha_t = comp::gotcha<5, tim::component_tuple<mpi_gotcha>, api::omnitrace>;
|
||||
} // namespace omnitrace
|
||||
@@ -0,0 +1,50 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/defines.hpp"
|
||||
#include "library/timemory.hpp"
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
namespace component
|
||||
{
|
||||
// timemory component which calls omnitrace functions
|
||||
// (used in gotcha wrappers)
|
||||
struct omnitrace : comp::base<omnitrace, void>
|
||||
{
|
||||
static std::string label() { return "omnitrace"; }
|
||||
void start();
|
||||
void stop();
|
||||
void set_prefix(const char*);
|
||||
|
||||
private:
|
||||
const char* m_prefix = nullptr;
|
||||
};
|
||||
} // namespace component
|
||||
} // namespace omnitrace
|
||||
|
||||
TIMEMORY_METADATA_SPECIALIZATION(
|
||||
omnitrace::component::omnitrace, "omnitrace",
|
||||
"Invokes instrumentation functions 'omnitrace_push_trace' and 'omnitrace_pop_trace'",
|
||||
"Used by gotcha wrappers")
|
||||
@@ -0,0 +1,70 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/common.hpp"
|
||||
#include "library/defines.hpp"
|
||||
#include "library/timemory.hpp"
|
||||
|
||||
#include <future>
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
struct pthread_gotcha : tim::component::base<pthread_gotcha, void>
|
||||
{
|
||||
struct wrapper
|
||||
{
|
||||
using routine_t = void* (*) (void*);
|
||||
using promise_t = std::promise<void>;
|
||||
|
||||
wrapper(routine_t _routine, void* _arg, bool, promise_t*);
|
||||
void* operator()() const;
|
||||
|
||||
static void* wrap(void* _arg);
|
||||
|
||||
private:
|
||||
bool m_enable_sampling = false;
|
||||
routine_t m_routine = nullptr;
|
||||
void* m_arg = nullptr;
|
||||
promise_t* m_promise = nullptr;
|
||||
};
|
||||
|
||||
TIMEMORY_DEFAULT_OBJECT(pthread_gotcha)
|
||||
|
||||
// string id for component
|
||||
static std::string label() { return "pthread_gotcha"; }
|
||||
|
||||
// generate the gotcha wrappers
|
||||
static void configure();
|
||||
static void shutdown();
|
||||
|
||||
// threads can set this to avoid starting sampling on child threads
|
||||
static bool& enable_sampling_on_child_threads();
|
||||
|
||||
// pthread_create
|
||||
int operator()(pthread_t* thread, const pthread_attr_t* attr,
|
||||
void* (*start_routine)(void*), void* arg) const;
|
||||
};
|
||||
|
||||
using pthread_gotcha_t = tim::component::gotcha<2, std::tuple<>, pthread_gotcha>;
|
||||
} // namespace omnitrace
|
||||
@@ -0,0 +1,173 @@
|
||||
// Copyright (c) 2018 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// with the Software without restriction, including without limitation the
|
||||
// rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
// sell copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimers.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above copyright
|
||||
// notice, this list of conditions and the following disclaimers in the
|
||||
// documentation and/or other materials provided with the distribution.
|
||||
//
|
||||
// * Neither the names of Advanced Micro Devices, Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this Software without specific prior written permission.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
|
||||
// THE SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/common.hpp"
|
||||
#include "library/components/fwd.hpp"
|
||||
#include "library/defines.hpp"
|
||||
#include "library/thread_data.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <future>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <ratio>
|
||||
#include <thread>
|
||||
#include <tuple>
|
||||
#include <type_traits>
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
namespace rocm_smi
|
||||
{
|
||||
void
|
||||
setup();
|
||||
|
||||
void
|
||||
config();
|
||||
|
||||
void
|
||||
sample();
|
||||
|
||||
void
|
||||
shutdown();
|
||||
|
||||
void
|
||||
post_process();
|
||||
|
||||
void set_state(State);
|
||||
|
||||
uint32_t
|
||||
device_count();
|
||||
|
||||
struct data
|
||||
{
|
||||
using msec_t = std::chrono::milliseconds;
|
||||
using usec_t = std::chrono::microseconds;
|
||||
using nsec_t = std::chrono::nanoseconds;
|
||||
using promise_t = std::promise<void>;
|
||||
|
||||
using timestamp_t = int64_t;
|
||||
using power_t = uint64_t;
|
||||
using busy_perc_t = uint32_t;
|
||||
using mem_usage_t = uint64_t;
|
||||
using temp_t = int64_t;
|
||||
|
||||
TIMEMORY_DEFAULT_OBJECT(data)
|
||||
|
||||
explicit data(uint32_t _dev_id);
|
||||
|
||||
void sample(uint32_t _dev_id);
|
||||
void print(std::ostream& _os) const;
|
||||
|
||||
static void post_process(uint32_t _dev_id);
|
||||
|
||||
uint32_t m_dev_id = std::numeric_limits<uint32_t>::max();
|
||||
timestamp_t m_ts = 0;
|
||||
busy_perc_t m_busy_perc = 0;
|
||||
temp_t m_temp = 0;
|
||||
power_t m_power = 0;
|
||||
mem_usage_t m_mem_usage = 0;
|
||||
|
||||
friend std::ostream& operator<<(std::ostream& _os, const data& _v)
|
||||
{
|
||||
_v.print(_os);
|
||||
return _os;
|
||||
}
|
||||
|
||||
private:
|
||||
friend void omnitrace::rocm_smi::setup();
|
||||
friend void omnitrace::rocm_smi::config();
|
||||
friend void omnitrace::rocm_smi::sample();
|
||||
friend void omnitrace::rocm_smi::shutdown();
|
||||
friend void omnitrace::rocm_smi::post_process();
|
||||
|
||||
static size_t device_count;
|
||||
static std::set<uint32_t> device_list;
|
||||
static std::unique_ptr<promise_t> polling_finished;
|
||||
static std::vector<data>& get_initial();
|
||||
static std::unique_ptr<std::thread>& get_thread();
|
||||
static bool setup();
|
||||
static bool shutdown();
|
||||
};
|
||||
|
||||
#if !defined(OMNITRACE_USE_ROCM_SMI)
|
||||
inline void
|
||||
setup()
|
||||
{}
|
||||
|
||||
inline void
|
||||
config()
|
||||
{}
|
||||
|
||||
inline void
|
||||
sample()
|
||||
{}
|
||||
|
||||
inline void
|
||||
shutdown()
|
||||
{}
|
||||
|
||||
inline void
|
||||
post_process()
|
||||
{}
|
||||
|
||||
inline void set_state(State) {}
|
||||
#endif
|
||||
} // namespace rocm_smi
|
||||
} // namespace omnitrace
|
||||
|
||||
#if defined(OMNITRACE_USE_ROCM_SMI)
|
||||
# if !defined(OMNITRACE_EXTERN_COMPONENTS) || \
|
||||
(defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0)
|
||||
|
||||
# include <timemory/components/base.hpp>
|
||||
# include <timemory/components/data_tracker/components.hpp>
|
||||
# include <timemory/operations.hpp>
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(
|
||||
TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_busy>), true,
|
||||
double)
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(
|
||||
TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_temp>), true,
|
||||
double)
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(
|
||||
TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_power>), true,
|
||||
double)
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(
|
||||
TIMEMORY_ESC(data_tracker<double, omnitrace::component::backtrace_gpu_memory>), true,
|
||||
double)
|
||||
|
||||
# endif
|
||||
#endif
|
||||
@@ -0,0 +1,104 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/components/fwd.hpp"
|
||||
#include "library/defines.hpp"
|
||||
|
||||
#include <timemory/api.hpp>
|
||||
#include <timemory/components/base.hpp>
|
||||
#include <timemory/components/data_tracker/components.hpp>
|
||||
#include <timemory/components/macros.hpp>
|
||||
#include <timemory/enum.h>
|
||||
#include <timemory/macros/os.hpp>
|
||||
#include <timemory/mpl/type_traits.hpp>
|
||||
#include <timemory/mpl/types.hpp>
|
||||
|
||||
namespace tim
|
||||
{
|
||||
namespace component
|
||||
{
|
||||
using roctracer_data = data_tracker<double, roctracer>;
|
||||
|
||||
struct roctracer
|
||||
: base<roctracer, void>
|
||||
, private policy::instance_tracker<roctracer, false>
|
||||
{
|
||||
using value_type = void;
|
||||
using base_type = base<roctracer, void>;
|
||||
using tracker_type = policy::instance_tracker<roctracer, false>;
|
||||
|
||||
TIMEMORY_DEFAULT_OBJECT(roctracer)
|
||||
|
||||
static void preinit();
|
||||
static void global_init() { setup(); }
|
||||
static void global_finalize() { shutdown(); }
|
||||
|
||||
static bool is_setup();
|
||||
static void setup();
|
||||
static void shutdown();
|
||||
static void add_setup(const std::string&, std::function<void()>&&);
|
||||
static void add_shutdown(const std::string&, std::function<void()>&&);
|
||||
static void remove_setup(const std::string&);
|
||||
static void remove_shutdown(const std::string&);
|
||||
|
||||
void start();
|
||||
void stop();
|
||||
};
|
||||
|
||||
#if !defined(OMNITRACE_USE_ROCTRACER)
|
||||
inline void
|
||||
roctracer::setup()
|
||||
{}
|
||||
|
||||
inline void
|
||||
roctracer::shutdown()
|
||||
{}
|
||||
|
||||
inline bool
|
||||
roctracer::is_setup()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
} // namespace component
|
||||
} // namespace tim
|
||||
|
||||
#if !defined(OMNITRACE_USE_ROCTRACER)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_available, component::roctracer_data, false_type)
|
||||
#endif
|
||||
|
||||
TIMEMORY_SET_COMPONENT_API(component::roctracer_data, project::timemory, category::timing,
|
||||
os::supports_unix)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(is_timing_category, component::roctracer_data, true_type)
|
||||
TIMEMORY_DEFINE_CONCRETE_TRAIT(uses_timing_units, component::roctracer_data, true_type)
|
||||
|
||||
#if !defined(OMNITRACE_EXTERN_COMPONENTS) || \
|
||||
(defined(OMNITRACE_EXTERN_COMPONENTS) && OMNITRACE_EXTERN_COMPONENTS > 0)
|
||||
|
||||
# include <timemory/operations.hpp>
|
||||
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(roctracer, false, void)
|
||||
TIMEMORY_DECLARE_EXTERN_COMPONENT(roctracer_data, true, double)
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,89 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "library/components/roctracer.hpp"
|
||||
#include "library/config.hpp"
|
||||
#include "library/debug.hpp"
|
||||
#include "library/dynamic_library.hpp"
|
||||
#include "library/perfetto.hpp"
|
||||
#include "library/ptl.hpp"
|
||||
|
||||
#include <roctracer.h>
|
||||
#include <roctracer_ext.h>
|
||||
#include <roctracer_hcc.h>
|
||||
#include <roctracer_hip.h>
|
||||
|
||||
#define AMD_INTERNAL_BUILD 1
|
||||
#include <ext/hsa_rt_utils.hpp>
|
||||
#include <roctracer_hsa.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
|
||||
// Macro to check ROC-tracer calls status
|
||||
#define ROCTRACER_CALL(call) \
|
||||
do \
|
||||
{ \
|
||||
int err = call; \
|
||||
if(err != 0) \
|
||||
{ \
|
||||
std::cerr << roctracer_error_string() << " in: " << #call << std::flush; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
namespace omnitrace
|
||||
{
|
||||
using roctracer_bundle_t =
|
||||
tim::component_bundle<api::omnitrace, comp::roctracer_data, comp::wall_clock>;
|
||||
using roctracer_hsa_bundle_t =
|
||||
tim::component_bundle<api::omnitrace, comp::roctracer_data>;
|
||||
using roctracer_functions_t = std::vector<std::pair<std::string, std::function<void()>>>;
|
||||
|
||||
// HSA API callback function
|
||||
void
|
||||
hsa_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
|
||||
|
||||
void
|
||||
hsa_activity_callback(uint32_t op, activity_record_t* record, void* arg);
|
||||
|
||||
void
|
||||
hip_exec_activity_callbacks(int64_t _tid);
|
||||
|
||||
// HIP API callback function
|
||||
void
|
||||
hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg);
|
||||
|
||||
// Activity tracing callback
|
||||
void
|
||||
hip_activity_callback(const char* begin, const char* end, void*);
|
||||
|
||||
bool&
|
||||
roctracer_is_setup();
|
||||
|
||||
roctracer_functions_t&
|
||||
roctracer_setup_routines();
|
||||
|
||||
roctracer_functions_t&
|
||||
roctracer_shutdown_routines();
|
||||
} // namespace omnitrace
|
||||
Reference in New Issue
Block a user