AI/rocm-systems
2
0
انشعاب 0
کد مسائل تقاضاهای واکشی Actions Packages پروژه‌ها انتشارها دانشنامه فعالیت

Fix agent node id + randomize offset id (#625)

فهرست منبع 
* Fix agent node id + randomize offset id

- fixes the node_id value
- randomizes a constant offset for the id.handle values
- switch to using node ids in rocprofiler-sdk-tool library
- update tests related to agents

* Logical node id

- sequential node id values from 0 to (N-1) where N is the number of agents
This commit is contained in:
Jonathan R. Madsen
2024-03-21 20:04:21 -05:00
کامیت شده توسط GitHub
والد 2f9b1767e9
کامیت 1addfed9f6
8فایلهای تغییر یافته به همراه170 افزوده شده و 31 حذف شده
دانلود پرونده وصله دانلود فایل تغییرات diff Expand all files Collapse all files
source/include/rocprofiler-sdk/agent.h
+12 -4
مشاهده پرونده
@@ -98,7 +98,14 @@ typedef struct rocprofiler_agent_mem_bank_t
} rocprofiler_agent_mem_bank_t;
/**
* @brief Agent (rocprofiler_agent_t stores the properties of an Agent)
* @brief Stores the properties of an agent (CPU, GPU, etc.)
*
* The `node_id` member is the KFD topology node id. It should be considered the "universal"
* indexing number. It is equivalent to the HSA-runtime HSA_AMD_AGENT_INFO_DRIVER_NODE_ID property
* of a `hsa_agent_t`. The `const char*` fields (`name`, `vendor_name`, etc.) are guaranteed to be
* valid pointers to null-terminated strings during tool finalization. Pointers to the agents via
* @see ::rocprofiler_query_available_agents are constant and will not be deallocated until after
* tool finalization. Making copies of the agent struct is also valid.
*/
typedef struct rocprofiler_agent_v0_t
{
@@ -187,9 +194,10 @@ typedef struct rocprofiler_agent_v0_t
///< based on the device type.
const rocprofiler_pc_sampling_configuration_t*
pc_sampling_configs; ///< GPU only. Array of PC sampling configuration types.
uint32_t node_id; ///< Node sequence number. This will be equivalent to the HSA-runtime
///< HSA_AMD_AGENT_INFO_DRIVER_NODE_ID property
uint32_t reserved0; ///< reserved padding
uint32_t node_id; ///< Node sequence number. This will be equivalent to the HSA-runtime
///< HSA_AMD_AGENT_INFO_DRIVER_NODE_ID property
int32_t logical_node_id; ///< Logical sequence number. This will always be [0..N) where N is
///< the total number of agents
} rocprofiler_agent_v0_t;
typedef rocprofiler_agent_v0_t rocprofiler_agent_t;
source/lib/rocprofiler-sdk-tool/tool.cpp
+21 -4
مشاهده پرونده
@@ -311,6 +311,7 @@ using kernel_symbol_data_map_t = std::unordered_map<rocprofiler_kernel_id_t, ker
using targeted_kernels_set_t = std::unordered_set<rocprofiler_kernel_id_t>;
using counter_dimension_info_map_t =
std::unordered_map<uint64_t, std::vector<rocprofiler_record_dimension_info_t>>;
using agent_info_map_t = std::unordered_map<rocprofiler_agent_id_t, const rocprofiler_agent_t*>;
auto code_obj_data = common::Synchronized<code_object_data_map_t, true>{};
auto kernel_data = common::Synchronized<kernel_symbol_data_map_t, true>{};
@@ -319,6 +320,7 @@ auto target_kernels = common::Synchronized<targeted_kernels_set_t>{};
auto dispatch_index = std::atomic<uint64_t>{0};
auto* buffered_name_info = as_pointer(get_buffer_id_names());
auto* callback_name_info = as_pointer(get_callback_id_names());
auto* agent_info = as_pointer(agent_info_map_t{});
bool
add_kernel_target(uint64_t _kern_id)
@@ -661,7 +663,7 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
tool::csv::kernel_trace_csv_encoder::write_row(
kernel_trace_ss,
CHECK_NOTNULL(buffered_name_info)->kind_names.at(record->kind),
record->agent_id.handle,
agent_info->at(record->agent_id)->node_id,
record->queue_id.handle,
record->kernel_id,
std::move(kernel_name),
@@ -714,8 +716,8 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
CHECK_NOTNULL(buffered_name_info)
->operation_names.at(record->kind)
.at(record->operation),
record->src_agent_id.handle,
record->dst_agent_id.handle,
agent_info->at(record->src_agent_id)->node_id,
agent_info->at(record->dst_agent_id)->node_id,
record->correlation_id.internal,
record->start_timestamp,
record->end_timestamp);
@@ -931,7 +933,7 @@ counter_record_callback(rocprofiler_profile_counting_dispatch_data_t dispatch_da
csv_encoder::write_row(counter_collection_ss,
correlation_id.internal,
cnt_dispatch_data_v->dispatch_index,
dispatch_data.agent_id.handle,
agent_info->at(dispatch_data.agent_id)->node_id,
dispatch_data.queue_id.handle,
getpid(),
cnt_dispatch_data_v->thread_id,
@@ -1321,6 +1323,21 @@ rocprofiler_configure(uint32_t version,
return nullptr;
}
ROCPROFILER_CALL(
rocprofiler_query_available_agents(
ROCPROFILER_AGENT_INFO_VERSION_0,
[](rocprofiler_agent_version_t, const void** agents, size_t num_agents, void*) {
for(size_t i = 0; i < num_agents; ++i)
{
auto* agent = static_cast<const rocprofiler_agent_v0_t*>(agents[i]);
agent_info->emplace(agent->id, agent);
}
return ROCPROFILER_STATUS_SUCCESS;
},
sizeof(rocprofiler_agent_t),
nullptr),
"Iterate rocporfiler agents")
LOG(INFO) << id->name << " is using rocprofiler-sdk v" << major << "." << minor << "." << patch
<< " (" << runtime_version << ")";
source/lib/rocprofiler-sdk/agent.cpp
+80 -12
مشاهده پرونده
@@ -27,18 +27,24 @@
#include "lib/common/filesystem.hpp"
#include "lib/common/scope_destructor.hpp"
#include "lib/common/static_object.hpp"
#include "lib/common/utility.hpp"
#include "lib/rocprofiler-sdk/agent.hpp"
#include "lib/rocprofiler-sdk/hsa/agent_cache.hpp"
#include <fmt/core.h>
#include <fmt/format.h>
#include <fmt/ranges.h>
#include <glog/logging.h>
#include <hsa/hsa.h>
#include <hsa/hsa_api_trace.h>
#include <libdrm/amdgpu.h>
#include <xf86drm.h>
#include <fstream>
#include <limits>
#include <random>
#include <regex>
#include <set>
#include <shared_mutex>
#include <sstream>
#include <string>
@@ -92,6 +98,18 @@ get_string_entry(std::string_view name)
.second.get();
}
uint64_t
get_agent_offset()
{
static uint64_t _v = []() {
auto gen = std::mt19937{std::random_device{}()};
auto rng = std::uniform_int_distribution<uint64_t>{std::numeric_limits<uint8_t>::max(),
std::numeric_limits<uint16_t>::max()};
return rng(gen);
}();
return _v;
}
struct cpu_info
{
long processor = -1;
@@ -377,8 +395,8 @@ read_topology()
while(true)
{
auto idx = idcount++;
auto node_path = sysfs_nodes_path / std::to_string(idx);
auto node_id = nodecount++;
auto node_path = sysfs_nodes_path / std::to_string(node_id);
// assumes that nodes are monotonically increasing and thus once we are missing a node
// folder for a number, there are no more nodes
if(!fs::exists(node_path)) break;
@@ -403,13 +421,11 @@ read_topology()
// we may have been able to open the properties file but if it was empty, we ignore it
if(properties.empty()) continue;
auto agent_info = rocprofiler_agent_t{};
memset(&agent_info, 0, sizeof(agent_info));
agent_info.size = sizeof(rocprofiler_agent_t);
agent_info.id.handle = idx;
agent_info.type = ROCPROFILER_AGENT_TYPE_NONE;
agent_info.node_id = nodecount++;
auto agent_info = common::init_public_api_struct(rocprofiler_agent_t{});
agent_info.type = ROCPROFILER_AGENT_TYPE_NONE;
agent_info.logical_node_id = idcount++;
agent_info.node_id = node_id;
agent_info.id.handle = (agent_info.logical_node_id) + get_agent_offset();
if(!name_prop.empty())
agent_info.model_name =
@@ -705,9 +721,57 @@ construct_agent_cache(::HsaApiTable* table)
},
&hsa_agents);
ROCP_CI_LOG_IF(ERROR, rocp_agents.size() != hsa_agents.size())
auto get_hsa_status_string = [table](hsa_status_t _status) -> std::string_view {
const char* _status_msg = nullptr;
return (table->core_->hsa_status_string_fn(_status, &_status_msg) == HSA_STATUS_SUCCESS &&
_status_msg)
? std::string_view{_status_msg}
: std::string_view{"(unknown HSA error)"};
};
auto rocp_hsa_agent_node_ids = std::set<uint32_t>{};
if(rocp_agents.size() != hsa_agents.size())
{
for(auto hitr : hsa_agents)
{
auto internal_node_id = std::numeric_limits<uint32_t>::max();
auto ret = table->core_->hsa_agent_get_info_fn(
hitr,
static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_DRIVER_NODE_ID),
&internal_node_id);
LOG_IF(ERROR, ret != HSA_STATUS_SUCCESS)
<< "hsa_agent_get_info(hsa_agent_t=" << hitr.handle
<< ", HSA_AMD_AGENT_INFO_DRIVER_NODE_ID, ...) returned " << ret
<< " :: " << get_hsa_status_string(ret);
if(ret == HSA_STATUS_SUCCESS)
{
{
auto ret_emplace = rocp_hsa_agent_node_ids.emplace(internal_node_id).second;
LOG_IF(WARNING, !ret_emplace)
<< "duplicate internal node id " << internal_node_id;
}
for(const auto* ritr : rocp_agents)
{
if(ritr->node_id == internal_node_id)
{
rocp_hsa_agent_node_ids.erase(internal_node_id);
break;
}
}
}
}
}
LOG_IF(FATAL, !rocp_hsa_agent_node_ids.empty())
<< "Found " << rocp_agents.size() << " rocprofiler agents and " << hsa_agents.size()
<< " HSA agents";
<< " HSA agents. HSA agents contained " << rocp_hsa_agent_node_ids.size()
<< " internal node ids not found by rocprofiler: "
<< fmt::format(
"{}",
fmt::join(rocp_hsa_agent_node_ids.begin(), rocp_hsa_agent_node_ids.end(), ", "));
get_agent_mapping().reserve(get_agent_mapping().size() + rocp_agents.size());
@@ -745,7 +809,9 @@ construct_agent_cache(::HsaApiTable* table)
}
}
LOG_IF(ERROR, agent_map.size() != hsa_agents.size())
LOG(ERROR) << "# agent node maps: " << hsa_agent_node_map.size();
LOG_IF(FATAL, agent_map.size() != hsa_agents.size())
<< "rocprofiler was only able to map " << agent_map.size()
<< " rocprofiler agents to HSA agents, expected " << hsa_agents.size();
@@ -833,8 +899,10 @@ construct_agent_cache(::HsaApiTable* table)
std::optional<hsa_agent_t>
get_hsa_agent(const rocprofiler_agent_t* agent)
{
LOG(ERROR) << "# of agent mappings: " << get_agent_mapping().size();
for(const auto& itr : get_agent_mapping())
{
LOG(ERROR) << "checking " << itr.rocp_agent->id.handle << " vs. " << agent->id.handle;
if(itr.rocp_agent->id.handle == agent->id.handle) return itr.hsa_agent;
}
source/lib/rocprofiler-sdk/tests/agent.cpp
+50 -9
مشاهده پرونده
@@ -24,11 +24,14 @@
#include <rocprofiler-sdk/fwd.h>
#include <rocprofiler-sdk/registration.h>
#include "lib/rocprofiler-sdk/agent.hpp"
#include "lib/rocprofiler-sdk/registration.hpp"
#include "lib/rocprofiler-sdk/tests/details/agent.hpp"
#include <fmt/core.h>
#include <gtest/gtest.h>
#include <hsa/hsa.h>
#include <hsa/hsa_api_trace.h>
#include <pthread.h>
#include <cstdint>
@@ -101,7 +104,7 @@ TEST(rocprofiler_lib, agent_abi)
EXPECT_EQ(offsetof(rocprofiler_agent_t, num_pc_sampling_configs), 280) << msg;
EXPECT_EQ(offsetof(rocprofiler_agent_t, pc_sampling_configs), 288) << msg;
EXPECT_EQ(offsetof(rocprofiler_agent_t, node_id), 296) << msg;
EXPECT_EQ(offsetof(rocprofiler_agent_t, reserved0), 300) << msg;
EXPECT_EQ(offsetof(rocprofiler_agent_t, logical_node_id), 300) << msg;
// Add test for offset of new field above this. Do NOT change any existing values!
constexpr auto expected_rocp_agent_size = 304;
@@ -145,7 +148,6 @@ TEST(rocprofiler_lib, agent)
if(agents_ver != ROCPROFILER_AGENT_INFO_VERSION_0) return ROCPROFILER_STATUS_ERROR;
auto* agents_v = static_cast<std::vector<const rocprofiler_agent_t*>*>(user_data);
// EXPECT_EQ(num_agents, hsa_agents_v.size());
for(size_t i = 0; i < num_agents; ++i)
{
const auto* agent = static_cast<const rocprofiler_agent_t*>(agents_arr[i]);
@@ -154,6 +156,26 @@ TEST(rocprofiler_lib, agent)
return ROCPROFILER_STATUS_SUCCESS;
};
hsa_init();
{
auto table = ::HsaApiTable{};
auto core_table = ::CoreApiTable{};
auto amd_ext_table = ::AmdExtTable{};
memset(&table, 0, sizeof(table));
memset(&core_table, 0, sizeof(core_table));
memset(&amd_ext_table, 0, sizeof(amd_ext_table));
core_table.hsa_iterate_agents_fn = &hsa_iterate_agents;
core_table.hsa_status_string_fn = &hsa_status_string;
core_table.hsa_agent_get_info_fn = &hsa_agent_get_info;
amd_ext_table.hsa_amd_agent_iterate_memory_pools_fn = &hsa_amd_agent_iterate_memory_pools;
amd_ext_table.hsa_amd_memory_pool_get_info_fn = &hsa_amd_memory_pool_get_info;
table.core_ = &core_table;
table.amd_ext_ = &amd_ext_table;
rocprofiler::agent::construct_agent_cache(&table);
}
std::cout << "# querying available agents...\n" << std::flush;
auto status =
rocprofiler_query_available_agents(ROCPROFILER_AGENT_INFO_VERSION_0,
@@ -168,10 +190,12 @@ TEST(rocprofiler_lib, agent)
auto& hsa_agents_v = _rocm_info.agents;
ASSERT_EQ(agents.size(), hsa_agents_v.size());
for(size_t i = 0; i < agents.size(); ++i)
EXPECT_GE(agents.size(), hsa_agents_v.size());
uint64_t skipped = 0;
for(const auto* agent : agents)
{
const auto* agent = agents.at(i);
ASSERT_NE(agent, nullptr);
auto msg = fmt::format("name={}, model={}, gfx version={}, id={}, type={}",
agent->name,
@@ -180,11 +204,26 @@ TEST(rocprofiler_lib, agent)
agent->node_id,
agent->type == ROCPROFILER_AGENT_TYPE_CPU ? "CPU" : "GPU");
// std::cout << msg << std::endl;
EXPECT_LT(i, hsa_agents_v.size()) << msg;
if(i >= hsa_agents_v.size()) continue;
rocprofiler::test::agent_info_t* hsa_agent = nullptr;
{
auto _hsa_agent = rocprofiler::agent::get_hsa_agent(agent);
auto* hsa_agent = &hsa_agents_v.at(i);
if(!_hsa_agent)
{
++skipped;
continue;
}
for(auto& hitr : hsa_agents_v)
{
if(_hsa_agent && _hsa_agent->handle == hitr.hsa_agent.handle)
{
hsa_agent = &hitr;
break;
}
}
ASSERT_NE(hsa_agent, nullptr) << msg;
}
if(agent->type == ROCPROFILER_AGENT_TYPE_CPU)
{
@@ -238,6 +277,8 @@ TEST(rocprofiler_lib, agent)
}
}
EXPECT_EQ(skipped, (agents.size() - hsa_agents_v.size()));
// clean up memory leak
for(auto& itr : _rocm_info.isas)
delete[] itr.name_str;
source/lib/rocprofiler-sdk/tests/details/agent.cpp
+3
مشاهده پرونده
@@ -103,6 +103,9 @@ AcquireSystemInfo(system_info_t* sys_info)
hsa_status_t
AcquireAgentInfoEntry(hsa_agent_t agent, agent_info_t* agent_i)
{
// store the hsa_agent_t value
agent_i->hsa_agent = agent;
hsa_status_t err;
// Get agent name and vendor
err = hsa_agent_get_info(agent, HSA_AGENT_INFO_NAME, agent_i->name);
source/lib/rocprofiler-sdk/tests/details/agent.hpp
+1
مشاهده پرونده
@@ -56,6 +56,7 @@ struct system_info_t
// calls, and is later used for reference when displaying the information.
struct agent_info_t
{
hsa_agent_t hsa_agent = {.handle = 0};
char name[64] = {'\0'};
char vendor_name[64] = {'\0'};
char device_mkt_name[64] = {'\0'};
tests/common/serialization.hpp
+1
مشاهده پرونده
@@ -487,6 +487,7 @@ save(ArchiveT& ar, const rocprofiler_agent_t& data)
SAVE_DATA_CSTR(model_name);
SAVE_DATA_FIELD(num_pc_sampling_configs);
SAVE_DATA_FIELD(node_id);
SAVE_DATA_FIELD(logical_node_id);
auto generate = [&](auto name, const auto* value, uint64_t size) {
using value_type = std::remove_const_t<std::remove_pointer_t<decltype(value)>>;
tests/rocprofv3/tracing/validate.py
+2 -2
مشاهده پرونده
@@ -68,13 +68,13 @@ def test_memory_copy_trace(memory_copy_input_data):
row = memory_copy_input_data[0]
assert row["Direction"] == "HOST_TO_DEVICE"
assert int(row["Source_Agent_Id"]) == 0
assert int(row["Destination_Agent_Id"]) == 1
assert int(row["Destination_Agent_Id"]) >= 1
assert int(row["Correlation_Id"]) > 0
assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
row = memory_copy_input_data[1]
assert row["Direction"] == "DEVICE_TO_HOST"
assert int(row["Source_Agent_Id"]) == 1
assert int(row["Source_Agent_Id"]) >= 1
assert int(row["Destination_Agent_Id"]) == 0
assert int(row["Correlation_Id"]) > 0
assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])