AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Roctracer flush activity fix + perfetto.cfg (#317)

Browse Source 
* Fix roctracer_flush_activity

- invoke roctracer_flush_activity() before disabling domains

* create comp::roctracer::flush()

- real issue was the global state when roctracer_flush_activity() was called

* formatting

* Update lib/omnitrace/library/components/roctracer.hpp

- provide definition of comp::roctracer::flush when OMNITRACE_USE_ROCTRACER is not defined

* omnitrace.cfg -> perfetto.cfg

- rename provided perfetto config file (omnitrace.cfg) to perfetto.cfg to avoid confusion

* Update lib/core

- gpu.hpp: defines for OMNITRACE_USE_{HIP,ROCTRACER,ROCPROFILER,ROCM_SMI}
- gpu.cpp
  - include core/hip_runtime.hpp
  - fix serialization of hipDeviceProp_t
- add hip_runtime.hpp
  -  ensure proper inclusion of hip_runtime.h
- add rccl.hpp
  - ensure proper inclusion of rccl.h

* Update lib/omnitrace/library

- rcclp.cpp
  - update includes for rccl
- roctracer.hpp
  - update includes for hip_runtime
- components/comm_data.hpp
  - update includes for rccl
- components/rcclp.hpp
  - update includes for rccl

* Update bin/omnitrace-avail/avail.cpp

- update includes for hip_runtime

* Update examples/rccl/CMakeLists.txt

- fix find_package for rccl when CI enabled

* Update CMakeLists.txt

- set cmake policy CMP0135 to NEW for cmake >= 3.24
  - Enable DOWNLOAD_EXTRACT_TIMESTAMP with ExternalProject_Add + URL download method

* Update timemory submodule

* Update pybind11 submodule

* Update pybind11 submodule

* Update lib/core/rccl.hpp

- include rccl.h only if OMNITRACE_USE_RCCL > 0

* Update lib/core/{gpu,hip_runtime}.hpp

* Update lib/core/gpu.cpp

- reintroduce some ppdefs

* Update lib/core/gpu.cpp

- fix ifdef on OMNITRACE_HIP_VERSION

* Update lib/core/gpu.cpp

- fix static assert for OMNITRACE_HIP_VERSION_MINOR when HIP version 4.x or older (unreliable minor versions)

* Update lib/core/gpu.cpp

- fix ifdef on OMNITRACE_HIP_VERSION

* Update lib/core/config.cpp

- disable OMNITRACE_PERFETTO_COMBINE_TRACES by default

* Update lib/core/perfetto.cpp

- if unable to open perfetto temp file, return the ReadTraceBlocking()

* Update lib/core/config.*

- flush tmpfile before closing
This commit is contained in:
Jonathan R. Madsen
2024-01-10 05:02:22 -06:00
committed by GitHub
parent aeb346b6d6
commit 7bc50f5a0a
24 changed files with 441 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lib/omnitrace/library.cpp
+7
View File
@@ -726,6 +726,13 @@ omnitrace_finalize_hidden(void)
}
}
if(get_use_roctracer())
{
OMNITRACE_VERBOSE_F(1, "Flushing roctracer...\n");
// ensure that roctracer is flushed before setting the state to finalized
comp::roctracer::flush();
}
set_state(State::Finalized);
push_enable_sampling_on_child_threads(false);
source/lib/omnitrace/library/components/comm_data.hpp
+1 -8
View File
@@ -26,6 +26,7 @@
#include "core/common.hpp"
#include "core/components/fwd.hpp"
#include "core/defines.hpp"
#include "core/rccl.hpp"
#include "core/timemory.hpp"
#include "library/components/category_region.hpp"
@@ -37,14 +38,6 @@
#include <optional>
#if defined(OMNITRACE_USE_RCCL)
# if OMNITRACE_HIP_VERSION == 0 || OMNITRACE_HIP_VERSION >= 50200
# include <rccl/rccl.h>
# else
# include <rccl.h>
# endif
#endif
#if defined(OMNITRACE_USE_MPI)
# include <mpi.h>
#endif
source/lib/omnitrace/library/components/rcclp.hpp
+1 -6
View File
@@ -25,6 +25,7 @@
#include "core/common.hpp"
#include "core/components/fwd.hpp"
#include "core/defines.hpp"
#include "core/rccl.hpp"
#include "core/timemory.hpp"
#include "library/components/category_region.hpp"
#include "library/components/comm_data.hpp"
@@ -32,12 +33,6 @@
#include <timemory/api/macros.hpp>
#include <timemory/components/macros.hpp>
#if OMNITRACE_HIP_VERSION == 0 || OMNITRACE_HIP_VERSION >= 50200
# include <rccl/rccl.h>
#else
# include <rccl.h>
#endif
#include <atomic>
#include <functional>
#include <memory>
source/lib/omnitrace/library/components/roctracer.cpp
+37 -19
View File
@@ -32,6 +32,7 @@
#include "library/thread_data.hpp"
#include "library/thread_info.hpp"
#include <chrono>
#include <roctracer.h>
#define HIP_PROF_HIP_API_STRING 1
@@ -272,6 +273,41 @@ roctracer::setup(void* table, bool on_load_trace)
OMNITRACE_VERBOSE_F(1, "roctracer is setup\n");
}
void
roctracer::flush()
{
auto wait_for_activity_flush_completion = []() {
uint16_t nitr = 0;
while(roctracer_activity_count() > 0 && nitr++ < 10)
std::this_thread::sleep_for(std::chrono::milliseconds{ 100 });
};
// a flush may already be happening
wait_for_activity_flush_completion();
if(roctracer_activity_count() == 0)
{
OMNITRACE_VERBOSE_F(2, "executing roctracer_flush_activity()...\n");
OMNITRACE_ROCTRACER_CALL(roctracer_flush_activity());
// wait to make sure flush completes
std::this_thread::sleep_for(std::chrono::milliseconds{ 100 });
wait_for_activity_flush_completion();
}
else
{
OMNITRACE_CI_FAIL(true,
"roctracer_activity_count() != 0 (== %li). "
"roctracer::shutdown() most likely called during abort",
roctracer_activity_count().load());
}
OMNITRACE_VERBOSE_F(2, "executing hip_exec_activity_callbacks(0..%zu)\n",
thread_info::get_peak_num_threads());
// make sure all async operations are executed
for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
hip_exec_activity_callbacks(i);
}
void
roctracer::shutdown()
{
@@ -282,16 +318,11 @@ roctracer::shutdown()
tim::storage<comp::roctracer_data>::instance()->reset();
return;
}
roctracer_is_setup() = false;
OMNITRACE_VERBOSE_F(1, "shutting down roctracer...\n");
OMNITRACE_VERBOSE_F(2, "executing hip_exec_activity_callbacks(0..%zu)\n",
thread_info::get_peak_num_threads());
// make sure all async operations are executed
for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
hip_exec_activity_callbacks(i);
// callback for hsa
OMNITRACE_VERBOSE_F(2, "executing %zu roctracer_shutdown_routines...\n",
roctracer_shutdown_routines().size());
@@ -352,19 +383,6 @@ roctracer::shutdown()
roctracer_disable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_COPY));
}
if(roctracer_activity_count() == 0)
{
OMNITRACE_VERBOSE_F(2, "executing roctracer_flush_activity()...\n");
OMNITRACE_ROCTRACER_CALL(roctracer_flush_activity());
}
else
{
OMNITRACE_CI_FAIL(true,
"roctracer_activity_count() != 0 (== %li). "
"roctracer::shutdown() most likely called during abort",
roctracer_activity_count().load());
}
OMNITRACE_VERBOSE_F(1, "roctracer is shutdown\n");
}
source/lib/omnitrace/library/components/roctracer.hpp
+5
View File
@@ -58,6 +58,7 @@ struct roctracer
static bool is_setup();
static void setup(void* hsa_api_table, bool on_load_trace = false);
static void flush();
static void shutdown();
static void add_setup(const std::string&, std::function<void()>&&);
static void add_shutdown(const std::string&, std::function<void()>&&);
@@ -77,6 +78,10 @@ inline void
roctracer::setup(void*, bool)
{}
inline void
roctracer::flush()
{}
inline void
roctracer::shutdown()
{}
source/lib/omnitrace/library/rcclp.cpp
+1 -6
View File
@@ -26,17 +26,12 @@
#include "core/components/fwd.hpp"
#include "core/defines.hpp"
#include "core/dynamic_library.hpp"
#include "core/rccl.hpp"
#include "core/timemory.hpp"
#include "library/components/category_region.hpp"
#include <timemory/timemory.hpp>
#if OMNITRACE_HIP_VERSION == 0 || OMNITRACE_HIP_VERSION >= 50200
# include <rccl/rccl.h>
#else
# include <rccl.h>
#endif
#include <dlfcn.h>
#include <limits>
#include <memory>
source/lib/omnitrace/library/roctracer.cpp
-2
View File
@@ -44,8 +44,6 @@
#include <cstdint>
#include <tuple>
#define HIP_PROF_HIP_API_STRING 1
#include <roctracer_ext.h>
#include <roctracer_hip.h>
#include <roctracer_roctx.h>
source/lib/omnitrace/library/roctracer.hpp
+1
View File
@@ -24,6 +24,7 @@
#include "core/config.hpp"
#include "core/debug.hpp"
#include "core/hip_runtime.hpp"
#include "core/perfetto.hpp"
#include "library/components/roctracer.hpp"
#include "library/ptl.hpp"