[rocprofiler-compute] Fix for multi process workload profiling (#2418)
* Fix for multi process workload profiling
Native counter collection tool updates:
* Do not dump empty counter data for a process
* Use PID instead of UUID for dumped csv files to facilitate correlation
* Handle merging multiple pairs of rocpd (from sdk tool) and csv (from
native tool) files
* Handle merging multiple pairs of csv (from sdk tool) and csv (from
native tool) files
Rocpd output format updates:
* Merge multiple rocpd databases into a single csv
* Reset dispatch id and kernel id for unique dispatches and unique
kernels respectively
* Retain multiple rocpd databases per run for multi process workloads
* Add test case for multiprocess profiling using rocflop workload
* Add rocflop
* Fix native counter csv to rocprofv3 csv conversion
* Use kernel_id instead of dispatch_id to correlate native counter csv
and kernel trace csv
* python formatting using ruff 0.14 instead of 0.13
This commit is contained in:
vedithal-amd
GitHub
@@ -77,11 +77,11 @@ for the agent and returns a pointer to it.
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <set>
|
||||
#include <shared_mutex>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <unistd.h>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
@@ -148,7 +148,7 @@ struct counter_info_record_t {
|
||||
// Tool data struct, now includes a vector of counter_info_record_t
|
||||
struct tool_data_t {
|
||||
std::mutex mut{};
|
||||
std::unique_ptr<std::ostream> output_stream{nullptr};
|
||||
std::string output_filename{};
|
||||
std::unordered_map<uint64_t, std::string> counter_id_name_map{};
|
||||
std::string requested_counters{};
|
||||
std::string kernel_filter_include_regex{};
|
||||
@@ -614,14 +614,28 @@ void generate_output(tool_data_t *tool_data) {
|
||||
}),
|
||||
tool_data->counter_records.end());
|
||||
}
|
||||
|
||||
if (tool_data->counter_records.empty()) {
|
||||
return;
|
||||
}
|
||||
// Write collected counter records and clean up
|
||||
if (auto &os = tool_data->output_stream) {
|
||||
if (!tool_data->output_filename.empty()) {
|
||||
std::ofstream ofs(tool_data->output_filename);
|
||||
if (!ofs.is_open()) {
|
||||
std::cerr << "Failed to open output file: " << tool_data->output_filename
|
||||
<< std::endl;
|
||||
return;
|
||||
}
|
||||
// Write header at the beginning of the file
|
||||
ofs << "dispatch_id,gpu_id,kernel_id,lds_per_workgroup,"
|
||||
"counter_id,counter_name,counter_value\n";
|
||||
for (const auto &r : tool_data->counter_records)
|
||||
*os << r.dispatch_id << ',' << r.agent_id << "," << r.kernel_id << ','
|
||||
ofs << r.dispatch_id << ',' << r.agent_id << "," << r.kernel_id << ','
|
||||
<< r.LDS_memory_size << ',' << r.counter_id << ',' << r.counter_name
|
||||
<< ',' << r.counter_value << '\n';
|
||||
os->flush();
|
||||
ofs.flush();
|
||||
std::clog << "[rocprofiler-compute] [" << __FUNCTION__
|
||||
<< "] Counter collection data has been written to: "
|
||||
<< tool_data->output_filename << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -638,18 +652,13 @@ void tool_fini(void *user_data) {
|
||||
|
||||
} // namespace
|
||||
|
||||
std::unique_ptr<tool_data_t> create_tool_data(rocprofiler_client_id_t *id) {
|
||||
std::unique_ptr<tool_data_t>
|
||||
create_tool_data(rocprofiler_client_id_t * /*id*/) {
|
||||
auto tool_data = std::make_unique<tool_data_t>();
|
||||
|
||||
// Generate a unique output filename using a random hex string (no libuuid
|
||||
// dependency)
|
||||
std::random_device rd;
|
||||
std::mt19937 gen(rd());
|
||||
std::uniform_int_distribution<uint32_t> dis(0, 0xFFFFFFFF);
|
||||
std::stringstream filename_ss;
|
||||
filename_ss << std::hex << dis(gen);
|
||||
// Generate a unique output filename using the process ID
|
||||
std::string base_filename =
|
||||
"counter_collection_" + filename_ss.str().substr(0, 8) + ".csv";
|
||||
std::to_string(getpid()) + "_native_counter_collection.csv";
|
||||
|
||||
// Require ROCPROF_OUTPUT_PATH to be set, otherwise error out
|
||||
std::string filename;
|
||||
@@ -664,20 +673,7 @@ std::unique_ptr<tool_data_t> create_tool_data(rocprofiler_client_id_t *id) {
|
||||
// Use the generated base filename along with ROCPROF_OUTPUT_PATH
|
||||
filename += base_filename;
|
||||
|
||||
// Set output stream to file
|
||||
auto ofs = std::make_unique<std::ofstream>(filename);
|
||||
if (!ofs->is_open()) {
|
||||
throw std::runtime_error("Failed to open output file: " + filename);
|
||||
}
|
||||
tool_data->output_stream = std::move(ofs);
|
||||
// Write header at the beginning of the file
|
||||
*tool_data->output_stream << "dispatch_id,gpu_id,kernel_id,lds_per_workgroup,"
|
||||
"counter_id,counter_name,counter_value\n";
|
||||
tool_data->output_stream->flush();
|
||||
|
||||
// Write to clog the path of the logging file
|
||||
std::clog << id->name << " [" << __FUNCTION__
|
||||
<< "] Logging counter collection to: " << filename << std::endl;
|
||||
tool_data->output_filename = filename;
|
||||
|
||||
// Store ROCPROF env. vars. in tool_data
|
||||
|
||||
|
||||
@@ -61,7 +61,8 @@ def simple_bar(df: pd.DataFrame, title: Optional[str] = None) -> Optional[str]:
|
||||
|
||||
if "Metric" in df.columns and "Avg" in df.columns:
|
||||
metric_dict = (
|
||||
pd.DataFrame([df["Metric"], df["Avg"]])
|
||||
pd
|
||||
.DataFrame([df["Metric"], df["Avg"]])
|
||||
.replace("", 0)
|
||||
.replace(float("inf"), -1) # It should not happen
|
||||
.replace(float("-inf"), -1)
|
||||
@@ -258,7 +259,8 @@ def px_simple_multi_bar(
|
||||
|
||||
for group, metric in nested_bar.items():
|
||||
dfigs.append(
|
||||
px.bar(
|
||||
px
|
||||
.bar(
|
||||
title=group,
|
||||
x=metric.values(),
|
||||
y=metric.keys(),
|
||||
|
||||
@@ -219,7 +219,8 @@ def get_views() -> list[TextClause]:
|
||||
select(
|
||||
Kernel.kernel_name,
|
||||
(Dispatch.end_timestamp - Dispatch.start_timestamp).label("duration"),
|
||||
func.row_number()
|
||||
func
|
||||
.row_number()
|
||||
.over(
|
||||
partition_by=Kernel.kernel_name,
|
||||
order_by=Dispatch.end_timestamp - Dispatch.start_timestamp,
|
||||
|
||||
@@ -132,7 +132,8 @@ class MIGPUSpecs:
|
||||
cls._all_gpu_models.append(curr_gpu_model)
|
||||
cls._gpu_model_dict[curr_gpu_arch].append(curr_gpu_model)
|
||||
cls._num_xcds_dict[curr_gpu_model] = (
|
||||
models.get("partition_mode", {})
|
||||
models
|
||||
.get("partition_mode", {})
|
||||
.get("compute_partition_mode", {})
|
||||
.get("num_xcds", {})
|
||||
)
|
||||
|
||||
@@ -580,7 +580,8 @@ def gen_counter_list(formula: str) -> tuple[bool, list[str]]:
|
||||
return visited, counters
|
||||
try:
|
||||
tree = ast.parse(
|
||||
formula.replace("$normUnit", "SQ_WAVES")
|
||||
formula
|
||||
.replace("$normUnit", "SQ_WAVES")
|
||||
.replace("$denom", "SQ_WAVES")
|
||||
.replace(
|
||||
"$numActiveCUs",
|
||||
@@ -1606,9 +1607,9 @@ def load_pc_sampling_data_per_kernel(
|
||||
pc_sample_instructions = search_key_in_json(file_name, "pc_sample_instructions")
|
||||
df["instruction"] = (
|
||||
df["inst_index"].apply(
|
||||
lambda x: pc_sample_instructions[x]
|
||||
if x < len(pc_sample_instructions)
|
||||
else None
|
||||
lambda x: (
|
||||
pc_sample_instructions[x] if x < len(pc_sample_instructions) else None
|
||||
)
|
||||
)
|
||||
if pc_sample_instructions
|
||||
else None
|
||||
@@ -1618,9 +1619,11 @@ def load_pc_sampling_data_per_kernel(
|
||||
pc_sample_comments = search_key_in_json(file_name, "pc_sample_comments")
|
||||
df["source_line"] = (
|
||||
df["inst_index"].apply(
|
||||
lambda x: f".../{Path(pc_sample_comments[x]).name}"
|
||||
if x < len(pc_sample_comments)
|
||||
else None
|
||||
lambda x: (
|
||||
f".../{Path(pc_sample_comments[x]).name}"
|
||||
if x < len(pc_sample_comments)
|
||||
else None
|
||||
)
|
||||
)
|
||||
if pc_sample_comments
|
||||
else None
|
||||
@@ -1719,7 +1722,8 @@ def load_pc_sampling_data(
|
||||
|
||||
# Group by Instruction_Comment and aggregate
|
||||
grouped_counts = (
|
||||
merged_df.groupby("Instruction_Comment")
|
||||
merged_df
|
||||
.groupby("Instruction_Comment")
|
||||
.agg(
|
||||
count=("Instruction_Comment", "count"),
|
||||
instruction=("Instruction", "first"),
|
||||
|
||||
@@ -38,6 +38,7 @@ COUNTERS_COLLECTION_QUERY = """
|
||||
SELECT
|
||||
agent_id as GPU_ID,
|
||||
dispatch_id as Dispatch_ID,
|
||||
pid as PID,
|
||||
grid_size as Grid_Size,
|
||||
workgroup_size as Workgroup_Size,
|
||||
lds_block_size as LDS_Per_Workgroup,
|
||||
@@ -61,24 +62,28 @@ TABLE_NAME_PREFIX_QUERY = (
|
||||
INSERT_QUERY = "INSERT INTO {table_name} ({columns}) VALUES ({placeholders})"
|
||||
|
||||
|
||||
def convert_db_to_csv(
|
||||
db_path: str,
|
||||
def convert_dbs_to_csv(
|
||||
db_paths: list[str],
|
||||
csv_file_path: str,
|
||||
) -> None:
|
||||
"""
|
||||
Read rocpd database and write to CSV file
|
||||
Read rocpd databases and write to CSV file
|
||||
"""
|
||||
# Read counters_collection view from the database and write to CSV
|
||||
# Read counters_collection view from the databases and write to CSV
|
||||
try:
|
||||
with closing(sqlite3.connect(db_path)) as conn:
|
||||
with closing(conn.execute(COUNTERS_COLLECTION_QUERY)) as cursor:
|
||||
with open(csv_file_path, "w", newline="") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
writer.writerow([
|
||||
description[0] for description in cursor.description
|
||||
])
|
||||
for row in cursor:
|
||||
writer.writerow(row)
|
||||
with open(csv_file_path, "w", newline="") as csvfile:
|
||||
writer = csv.writer(csvfile)
|
||||
header_written = False
|
||||
for db_path in db_paths:
|
||||
with closing(sqlite3.connect(db_path)) as conn:
|
||||
with closing(conn.execute(COUNTERS_COLLECTION_QUERY)) as cursor:
|
||||
if not header_written:
|
||||
writer.writerow([
|
||||
description[0] for description in cursor.description
|
||||
])
|
||||
header_written = True
|
||||
for row in cursor:
|
||||
writer.writerow(row)
|
||||
except OSError as e:
|
||||
console_error(f"Database error while converting to CSV: {e}")
|
||||
except Exception as e:
|
||||
|
||||
@@ -426,7 +426,8 @@ def format_table_output(
|
||||
and "Value" in df.columns
|
||||
):
|
||||
mem_data = (
|
||||
pd.DataFrame([df["Metric"], df["Value"]])
|
||||
pd
|
||||
.DataFrame([df["Metric"], df["Value"]])
|
||||
.transpose()
|
||||
.set_index("Metric")
|
||||
.to_dict()["Value"]
|
||||
|
||||
@@ -885,24 +885,48 @@ def run_prof(
|
||||
rocprof_cmd == "rocprofiler-sdk"
|
||||
and options["ROCPROF_COUNTER_COLLECTION"] == "0"
|
||||
):
|
||||
# Update rocpd database with counter csv created by native tool
|
||||
rocpd_data.update_rocpd_pmc_events(
|
||||
pd.read_csv(glob.glob(workload_dir + "/out/pmc_1/*.csv")[0]),
|
||||
glob.glob(workload_dir + "/out/pmc_1/*/*.db")[0],
|
||||
)
|
||||
for db_name in glob.glob(workload_dir + "/out/pmc_1/*/*.db"):
|
||||
pid = Path(db_name).stem.split("_")[0]
|
||||
rocpd_data.update_rocpd_pmc_events(
|
||||
pd.read_csv(
|
||||
f"{workload_dir}/out/pmc_1/{pid}_native_counter_collection.csv"
|
||||
),
|
||||
db_name,
|
||||
)
|
||||
console_debug(f"Updated rocpd db {db_name} with native tool counters.")
|
||||
# Write results_fbase.csv
|
||||
rocpd_data.convert_db_to_csv(
|
||||
glob.glob(workload_dir + "/out/pmc_1/*/*.db")[0],
|
||||
rocpd_data.convert_dbs_to_csv(
|
||||
glob.glob(workload_dir + "/out/pmc_1/*/*.db"),
|
||||
workload_dir + f"/results_{fbase}.csv",
|
||||
)
|
||||
combined_df = pd.read_csv(workload_dir + f"/results_{fbase}.csv")
|
||||
# Reset Dispatch_ID based on PID, Kernel_Name, Grid_Size,
|
||||
# Workgroup_Size, LDS_Per_Workgroup
|
||||
combined_df["Dispatch_ID"] = combined_df.groupby(
|
||||
["PID", "Kernel_Name", "Grid_Size", "Workgroup_Size", "LDS_Per_Workgroup"],
|
||||
sort=False,
|
||||
).ngroup()
|
||||
# Reset Kernel_ID based on Kernel_Name, Grid_Size,
|
||||
# Workgroup_Size, LDS_Per_Workgroup
|
||||
combined_df["Kernel_ID"] = combined_df.groupby(
|
||||
["Kernel_Name", "Grid_Size", "Workgroup_Size", "LDS_Per_Workgroup"],
|
||||
sort=False,
|
||||
).ngroup()
|
||||
# Drop PID since its not required
|
||||
combined_df = combined_df.drop(columns=["PID"])
|
||||
combined_df.to_csv(workload_dir + f"/results_{fbase}.csv", index=False)
|
||||
|
||||
if retain_rocpd_output:
|
||||
shutil.copyfile(
|
||||
glob.glob(workload_dir + "/out/pmc_1/*/*.db")[0],
|
||||
workload_dir + "/" + fbase + ".db",
|
||||
)
|
||||
console_warning(
|
||||
f"Retaining large raw rocpd database: {workload_dir}/{fbase}.db"
|
||||
)
|
||||
for db_path in glob.glob(workload_dir + "/out/pmc_1/*/*.db"):
|
||||
pid = Path(db_path).stem.split("_")[0]
|
||||
shutil.copyfile(
|
||||
db_path,
|
||||
workload_dir + f"/{fbase}_{pid}.db",
|
||||
)
|
||||
console_warning(
|
||||
f"Retaining large raw rocpd database: "
|
||||
f"{workload_dir}/{fbase}_{pid}.db"
|
||||
)
|
||||
# Remove temp directory
|
||||
shutil.rmtree(workload_dir + "/" + "out")
|
||||
return
|
||||
@@ -1064,81 +1088,66 @@ def convert_native_counter_collection_csv(workload_dir: str) -> None:
|
||||
trace to write counter collection csv in rocprofiler-sdk format
|
||||
for further processing to pmc_perf.csv file
|
||||
"""
|
||||
counter_data = pd.read_csv(
|
||||
glob.glob(f"{workload_dir}/out/pmc_1/*.csv")[0], index_col=False
|
||||
)
|
||||
# Group by on counter_data based on dispatch_id and
|
||||
# counter_id and sum the counter_value
|
||||
counter_data = counter_data.groupby(
|
||||
["dispatch_id", "counter_name"], as_index=False
|
||||
).agg({"counter_value": "sum"})
|
||||
kernel_data_filename = glob.glob(f"{workload_dir}/out/pmc_1/*/*_kernel_trace.csv")[
|
||||
0
|
||||
]
|
||||
kernel_data = pd.read_csv(kernel_data_filename)
|
||||
rocprofv3_counter_data = pd.DataFrame({
|
||||
"Correlation_Id": counter_data["dispatch_id"],
|
||||
"Dispatch_Id": counter_data["dispatch_id"],
|
||||
"Agent_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Agent_Id"
|
||||
].values,
|
||||
"Queue_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Queue_Id"
|
||||
].values,
|
||||
"Process_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Thread_Id"
|
||||
].values,
|
||||
"Thread_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Thread_Id"
|
||||
].values,
|
||||
"Grid_Size": (
|
||||
kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
["Grid_Size_X", "Grid_Size_Y", "Grid_Size_Z"]
|
||||
]
|
||||
.prod(axis=1)
|
||||
.values
|
||||
),
|
||||
"Kernel_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Kernel_Id"
|
||||
].values,
|
||||
"Kernel_Name": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Kernel_Name"
|
||||
].values,
|
||||
"Workgroup_Size": (
|
||||
kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
["Workgroup_Size_X", "Workgroup_Size_Y", "Workgroup_Size_Z"]
|
||||
]
|
||||
.prod(axis=1)
|
||||
.values
|
||||
),
|
||||
"LDS_Block_Size": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"LDS_Block_Size"
|
||||
].values,
|
||||
"Scratch_Size": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Scratch_Size"
|
||||
].values,
|
||||
"VGPR_Count": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"VGPR_Count"
|
||||
].values,
|
||||
"Accum_VGPR_Count": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Accum_VGPR_Count"
|
||||
].values,
|
||||
"SGPR_Count": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"SGPR_Count"
|
||||
].values,
|
||||
"Counter_Name": counter_data["counter_name"],
|
||||
"Counter_Value": counter_data["counter_value"],
|
||||
"Start_Timestamp": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"Start_Timestamp"
|
||||
].values,
|
||||
"End_Timestamp": kernel_data.iloc[counter_data["dispatch_id"] - 1][
|
||||
"End_Timestamp"
|
||||
].values,
|
||||
})
|
||||
rocprofv3_counter_data.to_csv(
|
||||
kernel_data_filename.replace("kernel_trace", "counter_collection"),
|
||||
index=False,
|
||||
)
|
||||
for native_filename in glob.glob(
|
||||
f"{workload_dir}/out/pmc_1/*_native_counter_collection.csv"
|
||||
):
|
||||
counter_data = pd.read_csv(native_filename, index_col=False)
|
||||
# Group by on dispatch_id and counter_id and sum the counter_value,
|
||||
# Other rows in group have the same value, so take the first one
|
||||
groupby_cols = ["dispatch_id", "counter_name"]
|
||||
agg_dict = {
|
||||
col: "first" for col in counter_data.columns if col not in groupby_cols
|
||||
}
|
||||
# Overwrite counter_value aggregation to sum
|
||||
agg_dict["counter_value"] = "sum"
|
||||
counter_data = counter_data.groupby(groupby_cols, as_index=False).agg(agg_dict)
|
||||
|
||||
pid = Path(native_filename).stem.split("_")[0]
|
||||
kernel_data_filename = glob.glob(
|
||||
f"{workload_dir}/out/pmc_1/*/{pid}_kernel_trace.csv"
|
||||
)[0]
|
||||
kernel_data = pd.read_csv(kernel_data_filename)
|
||||
|
||||
# Merge counter_data with kernel_data on kernel_id
|
||||
merged_data = pd.merge(
|
||||
counter_data,
|
||||
kernel_data,
|
||||
left_on="kernel_id",
|
||||
right_on="Kernel_Id",
|
||||
how="left",
|
||||
)
|
||||
|
||||
rocprofv3_counter_data = pd.DataFrame({
|
||||
"Correlation_Id": merged_data["dispatch_id"],
|
||||
"Dispatch_Id": merged_data["dispatch_id"],
|
||||
"Agent_Id": merged_data["Agent_Id"],
|
||||
"Queue_Id": merged_data["Queue_Id"],
|
||||
"Process_Id": merged_data["Thread_Id"],
|
||||
"Thread_Id": merged_data["Thread_Id"],
|
||||
"Grid_Size": (
|
||||
merged_data[["Grid_Size_X", "Grid_Size_Y", "Grid_Size_Z"]].prod(axis=1)
|
||||
),
|
||||
"Kernel_Id": merged_data["Kernel_Id"],
|
||||
"Kernel_Name": merged_data["Kernel_Name"],
|
||||
"Workgroup_Size": (
|
||||
merged_data[
|
||||
["Workgroup_Size_X", "Workgroup_Size_Y", "Workgroup_Size_Z"]
|
||||
].prod(axis=1)
|
||||
),
|
||||
"LDS_Block_Size": merged_data["LDS_Block_Size"],
|
||||
"Scratch_Size": merged_data["Scratch_Size"],
|
||||
"VGPR_Count": merged_data["VGPR_Count"],
|
||||
"Accum_VGPR_Count": merged_data["Accum_VGPR_Count"],
|
||||
"SGPR_Count": merged_data["SGPR_Count"],
|
||||
"Counter_Name": merged_data["counter_name"],
|
||||
"Counter_Value": merged_data["counter_value"],
|
||||
"Start_Timestamp": merged_data["Start_Timestamp"],
|
||||
"End_Timestamp": merged_data["End_Timestamp"],
|
||||
})
|
||||
rocprofv3_counter_data.to_csv(
|
||||
kernel_data_filename.replace("kernel_trace", "counter_collection"),
|
||||
index=False,
|
||||
)
|
||||
|
||||
|
||||
def process_rocprofv3_output(workload_dir: str, using_native_tool: bool) -> list[str]:
|
||||
|
||||
مرجع در شماره جدید
Block a user