AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[rocprofiler-compute] Fix for multi process workload profiling (#2418)

Browse Source 
* Fix for multi process workload profiling

Native counter collection tool updates:
    * Do not dump empty counter data for a process
    * Use PID instead of UUID for dumped csv files to facilitate correlation
    * Handle merging multiple pairs of rocpd (from sdk tool) and csv (from
      native tool) files
    * Handle merging multiple pairs of csv (from sdk tool) and csv (from
      native tool) files

Rocpd output format updates:
    * Merge multiple rocpd databases into a single csv
    * Reset dispatch id and kernel id for unique dispatches and unique
      kernels respectively
    * Retain multiple rocpd databases per run for multi process workloads

* Add test case for multiprocess profiling using rocflop workload

* Add rocflop

* Fix native counter csv to rocprofv3 csv conversion

* Use kernel_id instead of dispatch_id to correlate native counter csv
  and kernel trace csv

* python formatting using ruff 0.14 instead of 0.13
This commit is contained in:
vedithal-amd
2025-12-23 13:12:18 -05:00
committed by GitHub
parent 3e49440495
commit 588773f9bf
13 changed files with 880 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocprofiler-compute/src/utils/utils.py
+98 -89
View File
@@ -885,24 +885,48 @@ def run_prof(
rocprof_cmd == "rocprofiler-sdk"
and options["ROCPROF_COUNTER_COLLECTION"] == "0"
):
# Update rocpd database with counter csv created by native tool
rocpd_data.update_rocpd_pmc_events(
pd.read_csv(glob.glob(workload_dir + "/out/pmc_1/*.csv")[0]),
glob.glob(workload_dir + "/out/pmc_1/*/*.db")[0],
)
for db_name in glob.glob(workload_dir + "/out/pmc_1/*/*.db"):
pid = Path(db_name).stem.split("_")[0]
rocpd_data.update_rocpd_pmc_events(
pd.read_csv(
f"{workload_dir}/out/pmc_1/{pid}_native_counter_collection.csv"
),
db_name,
)
console_debug(f"Updated rocpd db {db_name} with native tool counters.")
# Write results_fbase.csv
rocpd_data.convert_db_to_csv(
glob.glob(workload_dir + "/out/pmc_1/*/*.db")[0],
rocpd_data.convert_dbs_to_csv(
glob.glob(workload_dir + "/out/pmc_1/*/*.db"),
workload_dir + f"/results_{fbase}.csv",
)
combined_df = pd.read_csv(workload_dir + f"/results_{fbase}.csv")
# Reset Dispatch_ID based on PID, Kernel_Name, Grid_Size,
# Workgroup_Size, LDS_Per_Workgroup
combined_df["Dispatch_ID"] = combined_df.groupby(
["PID", "Kernel_Name", "Grid_Size", "Workgroup_Size", "LDS_Per_Workgroup"],
sort=False,
).ngroup()
# Reset Kernel_ID based on Kernel_Name, Grid_Size,
# Workgroup_Size, LDS_Per_Workgroup
combined_df["Kernel_ID"] = combined_df.groupby(
["Kernel_Name", "Grid_Size", "Workgroup_Size", "LDS_Per_Workgroup"],
sort=False,
).ngroup()
# Drop PID since its not required
combined_df = combined_df.drop(columns=["PID"])
combined_df.to_csv(workload_dir + f"/results_{fbase}.csv", index=False)
if retain_rocpd_output:
shutil.copyfile(
glob.glob(workload_dir + "/out/pmc_1/*/*.db")[0],
workload_dir + "/" + fbase + ".db",
)
console_warning(
f"Retaining large raw rocpd database: {workload_dir}/{fbase}.db"
)
for db_path in glob.glob(workload_dir + "/out/pmc_1/*/*.db"):
pid = Path(db_path).stem.split("_")[0]
shutil.copyfile(
db_path,
workload_dir + f"/{fbase}_{pid}.db",
)
console_warning(
f"Retaining large raw rocpd database: "
f"{workload_dir}/{fbase}_{pid}.db"
)
# Remove temp directory
shutil.rmtree(workload_dir + "/" + "out")
return
@@ -1064,81 +1088,66 @@ def convert_native_counter_collection_csv(workload_dir: str) -> None:
trace to write counter collection csv in rocprofiler-sdk format
for further processing to pmc_perf.csv file
"""
counter_data = pd.read_csv(
glob.glob(f"{workload_dir}/out/pmc_1/*.csv")[0], index_col=False
)
# Group by on counter_data based on dispatch_id and
# counter_id and sum the counter_value
counter_data = counter_data.groupby(
["dispatch_id", "counter_name"], as_index=False
).agg({"counter_value": "sum"})
kernel_data_filename = glob.glob(f"{workload_dir}/out/pmc_1/*/*_kernel_trace.csv")[
0
]
kernel_data = pd.read_csv(kernel_data_filename)
rocprofv3_counter_data = pd.DataFrame({
"Correlation_Id": counter_data["dispatch_id"],
"Dispatch_Id": counter_data["dispatch_id"],
"Agent_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Agent_Id"
].values,
"Queue_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Queue_Id"
].values,
"Process_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Thread_Id"
].values,
"Thread_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Thread_Id"
].values,
"Grid_Size": (
kernel_data.iloc[counter_data["dispatch_id"] - 1][
["Grid_Size_X", "Grid_Size_Y", "Grid_Size_Z"]
]
.prod(axis=1)
.values
),
"Kernel_Id": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Kernel_Id"
].values,
"Kernel_Name": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Kernel_Name"
].values,
"Workgroup_Size": (
kernel_data.iloc[counter_data["dispatch_id"] - 1][
["Workgroup_Size_X", "Workgroup_Size_Y", "Workgroup_Size_Z"]
]
.prod(axis=1)
.values
),
"LDS_Block_Size": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"LDS_Block_Size"
].values,
"Scratch_Size": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Scratch_Size"
].values,
"VGPR_Count": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"VGPR_Count"
].values,
"Accum_VGPR_Count": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Accum_VGPR_Count"
].values,
"SGPR_Count": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"SGPR_Count"
].values,
"Counter_Name": counter_data["counter_name"],
"Counter_Value": counter_data["counter_value"],
"Start_Timestamp": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"Start_Timestamp"
].values,
"End_Timestamp": kernel_data.iloc[counter_data["dispatch_id"] - 1][
"End_Timestamp"
].values,
})
rocprofv3_counter_data.to_csv(
kernel_data_filename.replace("kernel_trace", "counter_collection"),
index=False,
)
for native_filename in glob.glob(
f"{workload_dir}/out/pmc_1/*_native_counter_collection.csv"
):
counter_data = pd.read_csv(native_filename, index_col=False)
# Group by on dispatch_id and counter_id and sum the counter_value,
# Other rows in group have the same value, so take the first one
groupby_cols = ["dispatch_id", "counter_name"]
agg_dict = {
col: "first" for col in counter_data.columns if col not in groupby_cols
}
# Overwrite counter_value aggregation to sum
agg_dict["counter_value"] = "sum"
counter_data = counter_data.groupby(groupby_cols, as_index=False).agg(agg_dict)
pid = Path(native_filename).stem.split("_")[0]
kernel_data_filename = glob.glob(
f"{workload_dir}/out/pmc_1/*/{pid}_kernel_trace.csv"
)[0]
kernel_data = pd.read_csv(kernel_data_filename)
# Merge counter_data with kernel_data on kernel_id
merged_data = pd.merge(
counter_data,
kernel_data,
left_on="kernel_id",
right_on="Kernel_Id",
how="left",
)
rocprofv3_counter_data = pd.DataFrame({
"Correlation_Id": merged_data["dispatch_id"],
"Dispatch_Id": merged_data["dispatch_id"],
"Agent_Id": merged_data["Agent_Id"],
"Queue_Id": merged_data["Queue_Id"],
"Process_Id": merged_data["Thread_Id"],
"Thread_Id": merged_data["Thread_Id"],
"Grid_Size": (
merged_data[["Grid_Size_X", "Grid_Size_Y", "Grid_Size_Z"]].prod(axis=1)
),
"Kernel_Id": merged_data["Kernel_Id"],
"Kernel_Name": merged_data["Kernel_Name"],
"Workgroup_Size": (
merged_data[
["Workgroup_Size_X", "Workgroup_Size_Y", "Workgroup_Size_Z"]
].prod(axis=1)
),
"LDS_Block_Size": merged_data["LDS_Block_Size"],
"Scratch_Size": merged_data["Scratch_Size"],
"VGPR_Count": merged_data["VGPR_Count"],
"Accum_VGPR_Count": merged_data["Accum_VGPR_Count"],
"SGPR_Count": merged_data["SGPR_Count"],
"Counter_Name": merged_data["counter_name"],
"Counter_Value": merged_data["counter_value"],
"Start_Timestamp": merged_data["Start_Timestamp"],
"End_Timestamp": merged_data["End_Timestamp"],
})
rocprofv3_counter_data.to_csv(
kernel_data_filename.replace("kernel_trace", "counter_collection"),
index=False,
)
def process_rocprofv3_output(workload_dir: str, using_native_tool: bool) -> list[str]: