Gbaraldi/att tool (#766)
* Enabling codeobj and thread trace samples * Updating aqlprofile_v2 header * Codeobj and thread trace samples with output log files * Fixing clang format * Cmake formatting * Adding coverage to codeobj * Comment trace sample * Adding ATT Parser API * Fixing forwarding to aqlprofile * Clang formatting * Clang tidy * Adding option to print memory kernels * Clang format * Remove default from switch case * Separating client/main on codeobj sample for ASAn * Formatting * Gbaraldi/att tool rebase (#801) * Enabling codeobj and thread trace samples * Updating aqlprofile_v2 header * Codeobj and thread trace samples with output log files * Fixing clang format * Cmake formatting * Adding coverage to codeobj * Comment trace sample * Removing python from workflow * Adding ATT Parser API * Fixing forwarding to aqlprofile * Clang formatting * Clang tidy * Adding option to print memory kernels * Clang format * Remove default from switch case * Separating client/main on codeobj sample for ASAn * Formatting * Enabling codeobj and thread trace samples * Updating aqlprofile_v2 header * Codeobj and thread trace samples with output log files * Fixing clang format * Cmake formatting * Adding coverage to codeobj * Comment trace sample * Adding ATT Parser API * Fixing forwarding to aqlprofile * Clang formatting * Clang tidy * Adding option to print memory kernels * Clang format * Remove default from switch case * Separating client/main on codeobj sample for ASAn * Formatting * Fix codeobj library * Allow thread trace in parallel with other service * Zeroing the HSA signals * Adding exception wrappers in ATT sample * Removed force configure * Remove force configure from ISA decode * Removing codecov flag * Gbaraldi/att tool tests (#828) * Adding tests for codeobj ISA decode * Adding ATT tests * Adding ATT integration tests * Formatting * Changing codeobj binary extension * Renaming codeobj library spaces * Fixing samples * Formatting * Formatting * Fixing int test * Fixing linker error * Fixing memory fault * Moving kernel ot inside namespace * ASAN linking fix * Removing unecessary headers * Formatting * Fixing target_cu * Remove codeobj binary * Revert "Remove codeobj binary" This reverts commit 7d286f89d8096bc36925cd79cd742a5e6d10d179. * Enable memory snapshot * adding comgr --------- Co-authored-by: Ammar ELWazir <ammar.elwazir@amd.com>
Dieser Commit ist enthalten in:
Giovanni Lenzi Baraldi
committet von
GitHub
GitHub
Ursprung
6d3fbcffad
Commit
099ac7c72d
@@ -36,12 +36,11 @@
|
||||
#include <rocprofiler-sdk/fwd.h>
|
||||
#include <rocprofiler-sdk/registration.h>
|
||||
#include <rocprofiler-sdk/rocprofiler.h>
|
||||
#include "lib/rocprofiler-sdk/aql/aql_profile_v2.h"
|
||||
#include <rocprofiler-sdk-codeobj/code_printing.hpp>
|
||||
#include <shared_mutex>
|
||||
|
||||
#include "code_object_track.hpp"
|
||||
#include "common/defines.hpp"
|
||||
#include "common/filesystem.hpp"
|
||||
#include "lib/rocprofiler-sdk-codeobj/code_printing.hpp"
|
||||
|
||||
#include <cxxabi.h>
|
||||
#include <atomic>
|
||||
@@ -64,19 +63,28 @@
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#define TARGET_CU 1
|
||||
#define SIMD_SELECT 0x3
|
||||
#define BUFFER_SIZE 0x6000000
|
||||
#define NUM_SE 2
|
||||
constexpr bool COPY_MEMORY_CODEOBJ = false;
|
||||
#define OUTPUT_OFSTREAM "advanced_thread_trace.log"
|
||||
#define TARGET_CU 1
|
||||
#define SIMD_SELECT 0x3
|
||||
#define BUFFER_SIZE 0x6000000
|
||||
#define SE_MASK 0x11
|
||||
constexpr bool COPY_MEMORY_CODEOBJ = true;
|
||||
|
||||
template <>
|
||||
struct std::hash<pcinfo_t>
|
||||
{
|
||||
uint64_t operator()(const pcinfo_t& info) const
|
||||
#define C_API_BEGIN \
|
||||
try \
|
||||
{
|
||||
return info.addr ^ (info.marker_id << 32ul) ^ (info.marker_id >> 32ul);
|
||||
}
|
||||
#define C_API_END \
|
||||
} \
|
||||
catch(std::exception & e) \
|
||||
{ \
|
||||
std::cerr << "Error in " << __FILE__ << ':' << __LINE__ << ' ' << e.what() << std::endl; \
|
||||
} \
|
||||
catch(...) { std::cerr << "Error in " << __FILE__ << ':' << __LINE__ << std::endl; }
|
||||
|
||||
struct pcinfo_t
|
||||
{
|
||||
uint64_t marker_id;
|
||||
uint64_t addr;
|
||||
};
|
||||
|
||||
bool
|
||||
@@ -97,6 +105,10 @@ namespace client
|
||||
using code_obj_load_data_t = rocprofiler_callback_tracing_code_object_load_data_t;
|
||||
using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;
|
||||
|
||||
using Instruction = rocprofiler::codeobj::disassembly::Instruction;
|
||||
using CodeobjAddressTranslate = rocprofiler::codeobj::disassembly::CodeobjAddressTranslate;
|
||||
using SymbolInfo = rocprofiler::codeobj::disassembly::SymbolInfo;
|
||||
|
||||
rocprofiler_client_id_t* client_id = nullptr;
|
||||
rocprofiler_context_id_t client_ctx = {};
|
||||
|
||||
@@ -109,13 +121,35 @@ struct isa_map_elem_t
|
||||
|
||||
struct ToolData
|
||||
{
|
||||
ToolData()
|
||||
{
|
||||
try
|
||||
{
|
||||
output_file.open(OUTPUT_OFSTREAM);
|
||||
} catch(...)
|
||||
{}
|
||||
|
||||
if(output_file.is_open())
|
||||
std::cout << "Writing code-object-isa-decode log to: " << OUTPUT_OFSTREAM << std::endl;
|
||||
else
|
||||
std::cout << "Could not open log file: " << OUTPUT_OFSTREAM << ", writing to stdout\n";
|
||||
};
|
||||
|
||||
std::shared_mutex isa_map_mut;
|
||||
std::mutex output_mut;
|
||||
CodeobjAddressTranslate codeobjTranslate;
|
||||
std::map<pcinfo_t, std::unique_ptr<isa_map_elem_t>> isa_map;
|
||||
std::unordered_map<uint64_t, SymbolInfo> kernels_in_codeobj = {};
|
||||
std::unordered_map<uint64_t, std::string> kernel_object_to_kernel_name = {};
|
||||
std::stringstream output;
|
||||
int num_waves = 0;
|
||||
|
||||
std::ostream& output()
|
||||
{
|
||||
if(output_file.is_open())
|
||||
return output_file;
|
||||
else
|
||||
return std::cout;
|
||||
}
|
||||
|
||||
std::stringstream printKernel(uint64_t vaddr)
|
||||
{
|
||||
@@ -133,6 +167,9 @@ struct ToolData
|
||||
}
|
||||
return ss;
|
||||
}
|
||||
|
||||
private:
|
||||
std::ofstream output_file;
|
||||
};
|
||||
|
||||
struct source_location
|
||||
@@ -151,15 +188,12 @@ struct trace_data_t
|
||||
ToolData* tool;
|
||||
};
|
||||
|
||||
std::atomic<int> TRACE_DATA_ID{-1};
|
||||
std::atomic<int> KERNEL_ADDR_ID{-1};
|
||||
std::atomic<int> OCCUPANCY_ID{-1};
|
||||
|
||||
void
|
||||
tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
|
||||
rocprofiler_user_data_t* user_data,
|
||||
void* callback_data)
|
||||
{
|
||||
C_API_BEGIN
|
||||
if(record.kind != ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT) return;
|
||||
if(record.phase != ROCPROFILER_CALLBACK_PHASE_LOAD) return;
|
||||
|
||||
@@ -200,19 +234,18 @@ tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
|
||||
|
||||
(void) user_data;
|
||||
(void) callback_data;
|
||||
C_API_END
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
|
||||
rocprofiler_att_control_flags_t
|
||||
dispatch_callback(rocprofiler_queue_id_t queue_id,
|
||||
const rocprofiler_agent_t* agent,
|
||||
rocprofiler_correlation_id_t correlation_id,
|
||||
dispatch_callback(rocprofiler_queue_id_t /* queue_id */,
|
||||
const rocprofiler_agent_t* /* agent */,
|
||||
rocprofiler_correlation_id_t /* correlation_id */,
|
||||
const hsa_kernel_dispatch_packet_t* dispatch_packet,
|
||||
uint64_t kernel_id,
|
||||
void* userdata)
|
||||
uint64_t /* kernel_id */,
|
||||
void* userdata)
|
||||
{
|
||||
C_API_BEGIN
|
||||
assert(userdata && "Dispatch callback passed null!");
|
||||
ToolData& tool = *reinterpret_cast<ToolData*>(userdata);
|
||||
|
||||
@@ -236,82 +269,55 @@ dispatch_callback(rocprofiler_queue_id_t queue_id,
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
C_API_END
|
||||
return ROCPROFILER_ATT_CONTROL_NONE;
|
||||
}
|
||||
|
||||
void
|
||||
iterate_trace_types(int id, const char* metadata, void*)
|
||||
{
|
||||
if(std::string_view(metadata).find("occupancy") == 0)
|
||||
OCCUPANCY_ID.store(id);
|
||||
else if(std::string_view(metadata).find("kernel_ids_addr") == 0)
|
||||
KERNEL_ADDR_ID.store(id);
|
||||
else if(std::string_view(metadata).find("tracedata") == 0)
|
||||
TRACE_DATA_ID.store(id);
|
||||
}
|
||||
|
||||
hsa_status_t
|
||||
get_trace_data(int trace_type_id,
|
||||
int correlation_id,
|
||||
void* trace_events,
|
||||
uint64_t trace_size,
|
||||
void* userdata)
|
||||
get_trace_data(rocprofiler_att_parser_data_type_t type, void* att_data, void* userdata)
|
||||
{
|
||||
C_API_BEGIN
|
||||
assert(userdata && "ISA callback passed null!");
|
||||
trace_data_t& trace_data = *reinterpret_cast<trace_data_t*>(userdata);
|
||||
assert(trace_data.tool && "ISA callback passed null!");
|
||||
ToolData& tool = *reinterpret_cast<ToolData*>(trace_data.tool);
|
||||
|
||||
std::stringstream ss;
|
||||
std::shared_lock<std::shared_mutex> shared_lock(tool.isa_map_mut);
|
||||
|
||||
if(trace_type_id == OCCUPANCY_ID)
|
||||
{
|
||||
ss << "Num waves: " << trace_size / 2 << '\n';
|
||||
// auto* occ = reinterpret_cast<att_occupancy_info_t*>(trace_events);
|
||||
}
|
||||
else if(trace_type_id == KERNEL_ADDR_ID)
|
||||
{
|
||||
ss << "Num KRN events: " << trace_size << std::hex << '\n';
|
||||
auto* kaddr = reinterpret_cast<pcinfo_t*>(trace_events);
|
||||
for(size_t i = 0; i < trace_size; i++)
|
||||
if(kaddr[i].addr != 0)
|
||||
{
|
||||
ss << " - ADDR: " << kaddr[i].addr << ' ' << tool.printKernel(kaddr[i].addr).str()
|
||||
<< '\n';
|
||||
}
|
||||
ss << std::dec;
|
||||
}
|
||||
else if(trace_type_id == TRACE_DATA_ID)
|
||||
{
|
||||
ss << "Trace Length: " << trace_size << '\n';
|
||||
auto* tracedata = reinterpret_cast<att_trace_event_t*>(trace_events);
|
||||
if(type == ROCPROFILER_ATT_PARSER_DATA_TYPE_OCCUPANCY) tool.num_waves++;
|
||||
|
||||
for(size_t i = 0; i < trace_size; i++)
|
||||
if(type != ROCPROFILER_ATT_PARSER_DATA_TYPE_ISA) return;
|
||||
|
||||
auto& event = *reinterpret_cast<rocprofiler_att_data_type_isa_t*>(att_data);
|
||||
|
||||
pcinfo_t pc{event.marker_id, event.offset};
|
||||
auto it = tool.isa_map.find(pc);
|
||||
if(it == tool.isa_map.end())
|
||||
{
|
||||
shared_lock.unlock();
|
||||
{
|
||||
pcinfo_t pc = tracedata[i].pc;
|
||||
auto it = tool.isa_map.find(pc);
|
||||
if(it == tool.isa_map.end())
|
||||
std::unique_lock<std::shared_mutex> unique_lock(tool.isa_map_mut);
|
||||
auto ptr = std::make_unique<isa_map_elem_t>();
|
||||
try
|
||||
{
|
||||
shared_lock.unlock();
|
||||
{
|
||||
std::unique_lock<std::shared_mutex> unique_lock(tool.isa_map_mut);
|
||||
auto ptr = std::make_unique<isa_map_elem_t>();
|
||||
ptr->code_line = tool.codeobjTranslate.get(pc.marker_id, pc.addr);
|
||||
it = tool.isa_map.emplace(pc, std::move(ptr)).first;
|
||||
}
|
||||
shared_lock.lock();
|
||||
ptr->code_line = tool.codeobjTranslate.get(pc.marker_id, pc.addr);
|
||||
} catch(std::exception& e)
|
||||
{
|
||||
std::cerr << pc.marker_id << ":" << pc.addr << ' ' << e.what() << std::endl;
|
||||
return;
|
||||
} catch(...)
|
||||
{
|
||||
std::cerr << "Could not fetch: " << pc.marker_id << ':' << pc.addr << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
it->second->hitcount.fetch_add(tracedata[i].hitcount, std::memory_order_relaxed);
|
||||
it->second->latency.fetch_add(tracedata[i].latency, std::memory_order_relaxed);
|
||||
it = tool.isa_map.emplace(pc, std::move(ptr)).first;
|
||||
}
|
||||
shared_lock.lock();
|
||||
}
|
||||
|
||||
std::unique_lock<std::mutex> lk(tool.output_mut);
|
||||
tool.output << ss.str();
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
it->second->hitcount.fetch_add(event.hitcount, std::memory_order_relaxed);
|
||||
it->second->latency.fetch_add(event.latency, std::memory_order_relaxed);
|
||||
C_API_END
|
||||
}
|
||||
|
||||
uint64_t
|
||||
@@ -325,16 +331,15 @@ copy_trace_data(int* seid, uint8_t** buffer, uint64_t* buffer_size, void* userda
|
||||
return *buffer_size;
|
||||
}
|
||||
|
||||
hsa_status_t
|
||||
rocprofiler_status_t
|
||||
isa_callback(char* isa_instruction,
|
||||
char* source_reference,
|
||||
uint64_t* isa_memory_size,
|
||||
uint64_t* isa_size,
|
||||
uint64_t* source_size,
|
||||
uint64_t marker_id,
|
||||
uint64_t offset,
|
||||
void* userdata)
|
||||
{
|
||||
C_API_BEGIN
|
||||
assert(userdata && "ISA callback passed null!");
|
||||
trace_data_t& trace_data = *reinterpret_cast<trace_data_t*>(userdata);
|
||||
assert(trace_data.tool && "ISA callback passed null!");
|
||||
@@ -347,56 +352,47 @@ isa_callback(char* isa_instruction,
|
||||
instruction = tool.codeobjTranslate.get(marker_id, offset);
|
||||
}
|
||||
|
||||
if(!instruction.get()) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
|
||||
if(!instruction.get()) return ROCPROFILER_STATUS_ERROR_INVALID_ARGUMENT;
|
||||
|
||||
{
|
||||
size_t tmp_isa_size = *isa_size;
|
||||
size_t tmp_source_size = *source_size;
|
||||
*isa_size = instruction->inst.size();
|
||||
*source_size = instruction->comment.size();
|
||||
size_t tmp_isa_size = *isa_size;
|
||||
*isa_size = instruction->inst.size();
|
||||
|
||||
if(*isa_size > tmp_isa_size || *source_size > tmp_source_size)
|
||||
return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
|
||||
if(*isa_size > tmp_isa_size) return ROCPROFILER_STATUS_ERROR_OUT_OF_RESOURCES;
|
||||
}
|
||||
|
||||
memcpy(isa_instruction, instruction->inst.data(), *isa_size);
|
||||
memcpy(source_reference, instruction->comment.data(), *source_size);
|
||||
*isa_memory_size = instruction->size;
|
||||
|
||||
auto ptr = std::make_unique<isa_map_elem_t>();
|
||||
ptr->code_line = std::move(instruction);
|
||||
tool.isa_map.emplace(pcinfo_t{offset, marker_id}, std::move(ptr));
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
tool.isa_map.emplace(pcinfo_t{marker_id, offset}, std::move(ptr));
|
||||
C_API_END
|
||||
return ROCPROFILER_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void
|
||||
shader_data_callback(int64_t se_id,
|
||||
int64_t data_type_id,
|
||||
const char* data_type_name,
|
||||
void* se_data,
|
||||
size_t data_size,
|
||||
void* userdata)
|
||||
shader_data_callback(int64_t se_id, void* se_data, size_t data_size, void* userdata)
|
||||
{
|
||||
C_API_BEGIN
|
||||
assert(userdata && "Shader callback passed null!");
|
||||
ToolData& tool = *reinterpret_cast<ToolData*>(userdata);
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(tool.output_mut);
|
||||
tool.output << "SE ID: " << se_id << " with size " << data_size << std::hex << '\n';
|
||||
tool.output() << "SE ID: " << se_id << " with size " << data_size << std::hex << '\n';
|
||||
}
|
||||
trace_data_t data{.id = se_id, .data = (uint8_t*) se_data, .size = data_size, .tool = &tool};
|
||||
auto status = aqlprofile_att_parse_data(copy_trace_data, get_trace_data, isa_callback, &data);
|
||||
(void) status;
|
||||
auto status = rocprofiler_att_parse_data(copy_trace_data, get_trace_data, isa_callback, &data);
|
||||
if(status != ROCPROFILER_STATUS_SUCCESS)
|
||||
std::cerr << "shader_data_callback failed with status " << status << std::endl;
|
||||
C_API_END
|
||||
}
|
||||
|
||||
#pragma GCC diagnostic pop
|
||||
|
||||
int
|
||||
tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
|
||||
{
|
||||
(void) fini_func;
|
||||
aqlprofile_att_parser_iterate_event_list(iterate_trace_types, nullptr);
|
||||
ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "context creation");
|
||||
|
||||
ROCPROFILER_CALL(
|
||||
@@ -408,22 +404,19 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
|
||||
tool_data),
|
||||
"code object tracing service configure");
|
||||
|
||||
rocprofiler_att_parameters_t parameters{};
|
||||
parameters.target_cu = TARGET_CU;
|
||||
parameters.simd_select = SIMD_SELECT;
|
||||
parameters.buffer_size = BUFFER_SIZE;
|
||||
std::vector<rocprofiler_att_parameter_t> parameters;
|
||||
parameters.push_back({ROCPROFILER_ATT_PARAMETER_TARGET_CU, TARGET_CU});
|
||||
parameters.push_back({ROCPROFILER_ATT_PARAMETER_SIMD_SELECT, SIMD_SELECT});
|
||||
parameters.push_back({ROCPROFILER_ATT_PARAMETER_BUFFER_SIZE, BUFFER_SIZE});
|
||||
parameters.push_back({ROCPROFILER_ATT_PARAMETER_SHADER_ENGINE_MASK, SE_MASK});
|
||||
|
||||
std::vector<int> shaders;
|
||||
for(size_t i = 0; i < NUM_SE; i++)
|
||||
shaders.push_back(2 * i); // use shader engines 0, 2
|
||||
|
||||
parameters.shader_ids = shaders.data();
|
||||
parameters.shader_num = shaders.size();
|
||||
|
||||
ROCPROFILER_CALL(
|
||||
rocprofiler_configure_thread_trace_service(
|
||||
client_ctx, parameters, dispatch_callback, shader_data_callback, tool_data),
|
||||
"thread trace service configure");
|
||||
ROCPROFILER_CALL(rocprofiler_configure_thread_trace_service(client_ctx,
|
||||
parameters.data(),
|
||||
parameters.size(),
|
||||
dispatch_callback,
|
||||
shader_data_callback,
|
||||
tool_data),
|
||||
"thread trace service configure");
|
||||
|
||||
int valid_ctx = 0;
|
||||
ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
|
||||
@@ -476,9 +469,9 @@ tool_fini(void* tool_data)
|
||||
size_t latency = line->latency.load(std::memory_order_relaxed);
|
||||
auto& code_line = line->code_line->inst;
|
||||
|
||||
tool.output << std::hex << "0x" << addr.addr << std::dec << ' ' << code_line
|
||||
<< empty_space.substr(0, max_inst_size - code_line.size())
|
||||
<< " Hit: " << hitcount << " - Latency: " << latency << '\n';
|
||||
tool.output() << std::hex << "0x" << addr.addr << std::dec << ' ' << code_line
|
||||
<< empty_space.substr(0, max_inst_size - code_line.size())
|
||||
<< " Hit: " << hitcount << " - Latency: " << latency << '\n';
|
||||
|
||||
if(code_line.find("s_waitcnt") == 0)
|
||||
{
|
||||
@@ -511,30 +504,16 @@ tool_fini(void* tool_data)
|
||||
float vmc_fraction = 100 * vmc_latency / float(total_latency);
|
||||
float lgk_fraction = 100 * lgk_latency / float(total_latency);
|
||||
|
||||
tool.output << "Total executed instructions: " << total_exec << '\n'
|
||||
<< "Total executed vector instructions: " << vector_exec << " with average "
|
||||
<< vector_latency / float(vector_exec) << " cycles.\n"
|
||||
<< "Total executed scalar instructions: " << scalar_exec << " with average "
|
||||
<< scalar_latency / float(scalar_exec) << " cycles.\n"
|
||||
<< "Vector memory ops occupied: " << vmc_fraction << "% of cycles.\n"
|
||||
<< "Scalar and LDS memory ops occupied: " << lgk_fraction << "% of cycles.\n";
|
||||
|
||||
std::cout << tool.output.str();
|
||||
tool.output() << "Total executed instructions: " << total_exec << '\n'
|
||||
<< "Total executed vector instructions: " << vector_exec << " with average "
|
||||
<< vector_latency / float(vector_exec) << " cycles.\n"
|
||||
<< "Total executed scalar instructions: " << scalar_exec << " with average "
|
||||
<< scalar_latency / float(scalar_exec) << " cycles.\n"
|
||||
<< "Vector memory ops occupied: " << vmc_fraction << "% of cycles.\n"
|
||||
<< "Scalar and LDS memory ops occupied: " << lgk_fraction << "% of cycles.\n"
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
void
|
||||
setup()
|
||||
{
|
||||
if(int status = 0;
|
||||
rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
|
||||
{
|
||||
ROCPROFILER_CALL(rocprofiler_force_configure(&rocprofiler_configure),
|
||||
"force configuration");
|
||||
}
|
||||
}
|
||||
|
||||
// force configuration when library is loaded
|
||||
bool cfg_on_load = (client::setup(), true);
|
||||
} // namespace client
|
||||
|
||||
extern "C" rocprofiler_tool_configure_result_t*
|
||||
|
||||
In neuem Issue referenzieren
Einen Benutzer sperren