Implement interface to rocprofiler sdk (#695)
* Setting ROCPROF=rocprofiler-sdk environment variable will use rocprofiler-sdk C++ library instead of rocprofv3 python script
* Add runtime option --rocprofiler-sdk-library-path to use custom version of rocprofiler sdk library
* Add --rocprofiler-sdk-library-path conftest option for tests
* Setup appropriate environment variables to inject rocprofiler sdk code to user command
* Add env. vars. for counter collection and filtering
* Add env. vars. for pc sampling
* Use python bindings to list counters supported by rocprofiler sdk
This commit is contained in:
vedithal-amd
committad av
GitHub
GitHub
förälder
d527d77337
incheckning
5cb86e31fc
+2
-2
@@ -1181,7 +1181,7 @@ def load_pc_sampling_data(workload, dir, file_prefix):
|
||||
|
||||
# No kernel filter, return grouped and sorted csv directly
|
||||
if not workload.filter_kernel_ids:
|
||||
# NB: the default file name is subject to changes from rocprofv3
|
||||
# NB: the default file name is subject to changes from rocprofv3/rocprofiler_sdk
|
||||
csv_file_path = Path.joinpath(
|
||||
Path(dir), file_prefix + "_pc_sampling_host_trap.csv"
|
||||
)
|
||||
@@ -1221,7 +1221,7 @@ def load_pc_sampling_data(workload, dir, file_prefix):
|
||||
|
||||
elif len(workload.filter_kernel_ids) == 1:
|
||||
# print("kernel id", workload.filter_kernel_ids[0])
|
||||
# NB: the default file name is subject to changes from rocprofv3
|
||||
# NB: the default file name is subject to changes from rocprofv3/rocprofiler_sdk
|
||||
json_file_path = Path.joinpath(Path(dir), file_prefix + "_results.json")
|
||||
if not json_file_path.exists():
|
||||
console_error("PC sampling: can not read %s " % json_file_path)
|
||||
|
||||
+149
-40
@@ -81,7 +81,11 @@ def using_v1():
|
||||
|
||||
def using_v3():
|
||||
return "ROCPROF" not in os.environ.keys() or (
|
||||
"ROCPROF" in os.environ.keys() and os.environ["ROCPROF"].endswith("rocprofv3")
|
||||
"ROCPROF" in os.environ.keys()
|
||||
and (
|
||||
os.environ["ROCPROF"].endswith("rocprofv3")
|
||||
or os.environ["ROCPROF"] == "rocprofiler-sdk"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -141,9 +145,23 @@ def get_version_display(version, sha, mode):
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def detect_rocprof():
|
||||
def detect_rocprof(args):
|
||||
"""Detect loaded rocprof version. Resolve path and set cmd globally."""
|
||||
global rocprof_cmd
|
||||
|
||||
if os.environ.get("ROCPROF") == "rocprofiler-sdk":
|
||||
if not path(args.rocprofiler_sdk_library_path).exists():
|
||||
console_error(
|
||||
"Could not find rocprofiler-sdk library at "
|
||||
+ args.rocprofiler_sdk_library_path
|
||||
)
|
||||
rocprof_cmd = "rocprofiler-sdk"
|
||||
console_debug("rocprof_cmd is {}".format(rocprof_cmd))
|
||||
console_debug(
|
||||
"rocprofiler_sdk_path is {}".format(args.rocprofiler_sdk_library_path)
|
||||
)
|
||||
return rocprof_cmd
|
||||
|
||||
# detect rocprof
|
||||
if not "ROCPROF" in os.environ.keys():
|
||||
rocprof_cmd = "rocprofv3"
|
||||
@@ -575,6 +593,35 @@ def v3_counter_csv_to_v2_csv(counter_file, agent_info_filepath, converted_csv_fi
|
||||
result.to_csv(converted_csv_file, index=False)
|
||||
|
||||
|
||||
def parse_text(text_file):
|
||||
"""
|
||||
Parse the text file to get the pmc counters.
|
||||
"""
|
||||
|
||||
def process_line(line):
|
||||
if "pmc:" not in line:
|
||||
return ""
|
||||
line = line.strip()
|
||||
pos = line.find("#")
|
||||
if pos >= 0:
|
||||
line = line[0:pos]
|
||||
|
||||
def _dedup(_line, _sep):
|
||||
for itr in _sep:
|
||||
_line = " ".join(_line.split(itr))
|
||||
return _line.strip()
|
||||
|
||||
# remove tabs and duplicate spaces
|
||||
return _dedup(line.replace("pmc:", ""), ["\n", "\t", " "]).split(" ")
|
||||
|
||||
with open(text_file, "r") as file:
|
||||
return [
|
||||
counter
|
||||
for litr in [process_line(itr) for itr in file.readlines()]
|
||||
for counter in litr
|
||||
]
|
||||
|
||||
|
||||
def run_prof(
|
||||
fname, profiler_options, workload_dir, mspec, loglevel, format_rocprof_output
|
||||
):
|
||||
@@ -585,11 +632,25 @@ def run_prof(
|
||||
|
||||
path_counter_config_yaml = path(fname).with_suffix(".yaml")
|
||||
# standard rocprof options
|
||||
default_options = ["-i", fname]
|
||||
options = default_options + profiler_options
|
||||
if rocprof_cmd == "rocprofiler-sdk":
|
||||
options = profiler_options
|
||||
options["ROCPROF_COUNTER_COLLECTION"] = "1"
|
||||
options["ROCPROF_COUNTERS"] = "pmc: " + " ".join(parse_text(fname))
|
||||
else:
|
||||
default_options = ["-i", fname]
|
||||
options = default_options + profiler_options
|
||||
|
||||
if using_v3():
|
||||
options = ["-A", "absolute"] + options
|
||||
if path_counter_config_yaml.exists():
|
||||
if rocprof_cmd == "rocprofiler-sdk":
|
||||
options["ROCPROF_AGENT_INDEX"] = "absolute"
|
||||
else:
|
||||
options = ["-A", "absolute"] + options
|
||||
|
||||
if using_v3() and path_counter_config_yaml.exists():
|
||||
if rocprof_cmd == "rocprofiler-sdk":
|
||||
with open(path_counter_config_yaml, "r") as file:
|
||||
options["ROCPROF_EXTRA_COUNTERS_CONTENTS"] = file.read()
|
||||
else:
|
||||
options = ["-E", str(path_counter_config_yaml)] + options
|
||||
|
||||
# set required env var for mi300
|
||||
@@ -603,16 +664,26 @@ def run_prof(
|
||||
is_timestamps = True
|
||||
time_1 = time.time()
|
||||
|
||||
console_debug("rocprof command: {}".format([rocprof_cmd] + options))
|
||||
# profile the app
|
||||
if new_env:
|
||||
if rocprof_cmd == "rocprofiler-sdk":
|
||||
app_cmd = options.pop("APP_CMD")
|
||||
for key, value in options.items():
|
||||
new_env[key] = value
|
||||
console_debug("rocprof sdk env vars: {}".format(new_env))
|
||||
console_debug("rocprof sdk user provided command: {}".format(app_cmd))
|
||||
success, output = capture_subprocess_output(
|
||||
[rocprof_cmd] + options, new_env=new_env, profileMode=True
|
||||
app_cmd, new_env=new_env, profileMode=True
|
||||
)
|
||||
else:
|
||||
success, output = capture_subprocess_output(
|
||||
[rocprof_cmd] + options, profileMode=True
|
||||
)
|
||||
console_debug("rocprof command: {}".format([rocprof_cmd] + options))
|
||||
# profile the app
|
||||
if new_env:
|
||||
success, output = capture_subprocess_output(
|
||||
[rocprof_cmd] + options, new_env=new_env, profileMode=True
|
||||
)
|
||||
else:
|
||||
success, output = capture_subprocess_output(
|
||||
[rocprof_cmd] + options, profileMode=True
|
||||
)
|
||||
|
||||
time_2 = time.time()
|
||||
console_debug(
|
||||
@@ -647,17 +718,22 @@ def run_prof(
|
||||
combined_results.to_csv(
|
||||
workload_dir + "/out/pmc_1/results_" + fbase + ".csv", index=False
|
||||
)
|
||||
elif rocprof_cmd.endswith("v3"):
|
||||
elif rocprof_cmd.endswith("v3") or rocprof_cmd == "rocprofiler-sdk":
|
||||
# rocprofv3 requires additional processing for each process
|
||||
results_files = process_rocprofv3_output(
|
||||
format_rocprof_output, workload_dir, is_timestamps
|
||||
)
|
||||
|
||||
if "--kokkos-trace" in options:
|
||||
if rocprof_cmd == "rocprofiler-sdk":
|
||||
# TODO: as rocprofv3 --kokkos-trace feature improves, rocprof-compute should make updates accordingly
|
||||
process_kokkos_trace_output(workload_dir, fbase)
|
||||
elif "--hip-trace" in options:
|
||||
process_hip_trace_output(workload_dir, fbase)
|
||||
if "ROCPROF_HIP_RUNTIME_API_TRACE" in options:
|
||||
process_hip_trace_output(workload_dir, fbase)
|
||||
else:
|
||||
if "--kokkos-trace" in options:
|
||||
# TODO: as rocprofv3 --kokkos-trace feature improves, rocprof-compute should make updates accordingly
|
||||
process_kokkos_trace_output(workload_dir, fbase)
|
||||
elif "--hip-trace" in options:
|
||||
process_hip_trace_output(workload_dir, fbase)
|
||||
|
||||
# Combine results into single CSV file
|
||||
if results_files:
|
||||
@@ -721,33 +797,66 @@ def run_prof(
|
||||
df.to_csv(workload_dir + "/" + fbase + ".csv", index=False)
|
||||
|
||||
|
||||
def pc_sampling_prof(interval, workload_dir, appcmd):
|
||||
def pc_sampling_prof(interval, workload_dir, appcmd, rocprofiler_sdk_library_path):
|
||||
"""
|
||||
Run rocprof with pc sampling. Current support v3 only.
|
||||
"""
|
||||
# Todo:
|
||||
# - precheck with rocprofv3 –-list-avail
|
||||
options = [
|
||||
"--pc-sampling-beta-enable",
|
||||
"--pc-sampling-method",
|
||||
"host_trap",
|
||||
"--pc-sampling-unit",
|
||||
"time",
|
||||
"--output-format",
|
||||
"csv",
|
||||
"json",
|
||||
"--pc-sampling-interval",
|
||||
str(interval),
|
||||
"-d",
|
||||
workload_dir,
|
||||
"-o",
|
||||
"ps_file", # todo: sync up with the name from source in 2100_.yaml
|
||||
"--",
|
||||
appcmd,
|
||||
]
|
||||
success, output = capture_subprocess_output(
|
||||
[rocprof_cmd] + options, new_env=os.environ.copy(), profileMode=True
|
||||
)
|
||||
if rocprof_cmd == "rocprofiler-sdk":
|
||||
rocm_libdir = str(pathlib.Path(rocprofiler_sdk_library_path).parent)
|
||||
rocprofiler_sdk_tool_path = str(
|
||||
pathlib.Path(rocm_libdir).joinpath(
|
||||
"rocprofiler-sdk/librocprofiler-sdk-tool.so"
|
||||
)
|
||||
)
|
||||
ld_preload = [
|
||||
rocprofiler_sdk_tool_path,
|
||||
rocprofiler_sdk_library_path,
|
||||
]
|
||||
options = {
|
||||
"ROCPROFILER_LIBRARY_CTOR": "1",
|
||||
"LD_PRELOAD": ":".join(ld_preload),
|
||||
"ROCP_TOOL_LIBRARIES": rocprofiler_sdk_tool_path,
|
||||
"LD_LIBRARY_PATH": rocm_libdir,
|
||||
"ROCPROF_OUTPUT_FORMAT": "csv,json",
|
||||
"ROCPROF_OUTPUT_PATH": workload_dir,
|
||||
"ROCPROF_OUTPUT_FILE_NAME": "ps_file",
|
||||
"ROCPROFILER_PC_SAMPLING_BETA_ENABLED": "1",
|
||||
"ROCPROF_PC_SAMPLING_UNIT": "time",
|
||||
"ROCPROF_PC_SAMPLING_INTERVAL": str(interval),
|
||||
"ROCPROF_PC_SAMPLING_METHOD": "host_trap",
|
||||
}
|
||||
new_env = os.environ.copy()
|
||||
for key, value in options.items():
|
||||
new_env[key] = value
|
||||
console_debug("pc sampling rocprof sdk env vars: {}".format(new_env))
|
||||
console_debug("pc sampling rocprof sdk user provided command: {}".format(appcmd))
|
||||
success, output = capture_subprocess_output(
|
||||
appcmd, new_env=new_env, profileMode=True
|
||||
)
|
||||
else:
|
||||
options = [
|
||||
"--pc-sampling-beta-enabled",
|
||||
"--pc-sampling-method",
|
||||
"host_trap",
|
||||
"--pc-sampling-unit",
|
||||
"time",
|
||||
"--output-format",
|
||||
"csv",
|
||||
"json",
|
||||
"--pc-sampling-interval",
|
||||
str(interval),
|
||||
"-d",
|
||||
workload_dir,
|
||||
"-o",
|
||||
"ps_file", # todo: sync up with the name from source in 2100_.yaml
|
||||
"--",
|
||||
appcmd,
|
||||
]
|
||||
success, output = capture_subprocess_output(
|
||||
[rocprof_cmd] + options, new_env=os.environ.copy(), profileMode=True
|
||||
)
|
||||
|
||||
if not success:
|
||||
console_error("PC sampling failed.")
|
||||
|
||||
Referens i nytt ärende
Block a user