AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SDK: create CMake option for strict checks on CPU vs. GPU timestamps (#1159)

Browse Source 
* SDK: create CMake option for strict checks on CPU vs. GPU timestamps

- Configurating CMake with `ROCPROFILER_BUILD_CI_STRICT_TIMESTAMPS=ON` will enable fatal errors if dispatch/memcpy timestamps on GPU are outside of the start/end time from the CPU
- `ROCPROFIELR_BUILD_CI_STRICT_TIMESTAMPS` defaults to the value of `ROCPROFILER_BUILD_CI`

* Formatting

* Disable async_copy frequency scaling

* Disable profiling dispatch time frequency scaling

* Support runtime configuration via env variables

- ROCPROFILER_CI_FREQ_SCALE_TIMESTAMPS env variable will enable scaling the timestamps based on the hsa timestamp period
- ROCPROFILER_CI_STRICT_TIMESTAMPS env variable will enable strict timestamp checks
  - when cmake is configured with ROCPROFILER_BUILD_CI_STRICT_TIMESTAMPS=ON, this env variable defaults to true

* ROCPROFILER_BUILD_CI_STRICT_TIMESTAMPS defaults to OFF

* Update cmake-target

* Common tracing::adjust_profiling_time

---------

Co-authored-by: Gopesh Bhardwaj <gopesh.bhardwaj@amd.com>
This commit is contained in:
Jonathan R. Madsen
2024-11-01 23:12:51 -05:00
committed by GitHub
parent 4204042ac6
commit ad48201912
9 changed files with 282 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lib/rocprofiler-sdk/kernel_dispatch/profiling_time.cpp
+17 -73
View File
@@ -21,10 +21,13 @@
// THE SOFTWARE.
#include "lib/rocprofiler-sdk/kernel_dispatch/profiling_time.hpp"
#include "lib/common/defines.hpp"
#include "lib/common/environment.hpp"
#include "lib/common/logging.hpp"
#include "lib/common/utility.hpp"
#include "lib/rocprofiler-sdk/agent.hpp"
#include "lib/rocprofiler-sdk/hsa/hsa.hpp"
#include "lib/rocprofiler-sdk/tracing/profiling_time.hpp"
#include <rocprofiler-sdk/fwd.h>
@@ -36,93 +39,35 @@ namespace rocprofiler
{
namespace kernel_dispatch
{
namespace
{
hsa_amd_profiling_dispatch_time_t&
operator+=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs)
{
lhs.start += rhs;
lhs.end += rhs;
return lhs;
}
hsa_amd_profiling_dispatch_time_t&
operator-=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs)
{
lhs.start -= rhs;
lhs.end -= rhs;
return lhs;
}
hsa_amd_profiling_dispatch_time_t&
operator*=(hsa_amd_profiling_dispatch_time_t& lhs, uint64_t rhs)
{
lhs.start *= rhs;
lhs.end *= rhs;
return lhs;
}
} // namespace
profiling_time&
profiling_time::operator+=(uint64_t offset)
{
start += offset;
end += offset;
return *this;
}
profiling_time&
profiling_time::operator-=(uint64_t offset)
{
start -= offset;
end -= offset;
return *this;
}
profiling_time&
profiling_time::operator*=(uint64_t scale)
{
start *= scale;
end *= scale;
return *this;
}
profiling_time
get_dispatch_time(hsa_agent_t _hsa_agent,
hsa_signal_t _signal,
rocprofiler_kernel_id_t _kernel_id,
std::optional<uint64_t> _baseline)
{
static auto sysclock_period = hsa::get_hsa_timestamp_period();
auto ts = common::timestamp_ns();
auto dispatch_time = hsa_amd_profiling_dispatch_time_t{};
auto dispatch_time_status = hsa::get_amd_ext_table()->hsa_amd_profiling_get_dispatch_time_fn(
_hsa_agent, _signal, &dispatch_time);
if(dispatch_time_status == HSA_STATUS_SUCCESS)
auto _profile_time =
tracing::profiling_time{dispatch_time_status, dispatch_time.start, dispatch_time.end};
if(_profile_time.status == HSA_STATUS_SUCCESS)
{
// if we encounter this in CI, it will cause test to fail
ROCP_CI_LOG_IF(ERROR, dispatch_time.end < dispatch_time.start)
ROCP_CI_LOG_IF(ERROR, _profile_time.end < _profile_time.start)
<< "hsa_amd_profiling_get_dispatch_time for kernel_id=" << _kernel_id
<< " on rocprofiler_agent="
<< CHECK_NOTNULL(agent::get_rocprofiler_agent(_hsa_agent))->id.handle
<< " returned dispatch times where the end time (" << dispatch_time.end
<< ") was less than the start time (" << dispatch_time.start << ")";
<< CHECK_NOTNULL(agent::get_rocprofiler_agent(_hsa_agent))->node_id
<< " returned dispatch times where the end time (" << _profile_time.end
<< ") was less than the start time (" << _profile_time.start << ")";
// normalize
dispatch_time *= sysclock_period;
// below is a hack for clock skew issues:
// the timestamp of this handler for the kernel dispatch will always be after when the
// kernel completed
if(ts < dispatch_time.end) dispatch_time -= (dispatch_time.end - ts);
// below is a hack for clock skew issues:
// the timestamp of the packet rewriter for the kernel packet will always be before when the
// kernel started
if(_baseline && dispatch_time.start < *_baseline)
dispatch_time += (*_baseline - dispatch_time.start);
_profile_time = tracing::adjust_profiling_time(
"dispatch",
_profile_time,
tracing::profiling_time{
HSA_STATUS_SUCCESS, _baseline.value_or(dispatch_time.start), ts});
}
else
{
@@ -133,8 +78,7 @@ get_dispatch_time(hsa_agent_t _hsa_agent,
<< " :: " << hsa::get_hsa_status_string(dispatch_time_status);
}
return profiling_time{
.status = dispatch_time_status, .start = dispatch_time.start, .end = dispatch_time.end};
return _profile_time;
}
} // namespace kernel_dispatch
} // namespace rocprofiler