AI/rocm-systems
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance logging and warning reporting

Browse Source 
Signed-off-by: coleramos425 <colramos@amd.com>
This commit is contained in:
coleramos425 committed 2023-05-16 15:39:45 -05:00
1 parent 444102d927
commit 44edef4635
2 files changed
+73 -31
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/omniperf
+24 -14
View File
@@ -160,7 +160,7 @@ def isWorkloadEmpty(my_parser, path):
) )
def replace_timestamps(workload_dir): def replace_timestamps(workload_dir, log_file):
df_stamps = pd.read_csv(workload_dir + "/timestamps.csv") df_stamps = pd.read_csv(workload_dir + "/timestamps.csv")
if "BeginNs" in df_stamps.columns and "EndNs" in df_stamps.columns: if "BeginNs" in df_stamps.columns and "EndNs" in df_stamps.columns:
# Update timestamps for all *.csv output files # Update timestamps for all *.csv output files
@@ -171,9 +171,11 @@ def replace_timestamps(workload_dir):
df_pmc_perf["EndNs"] = df_stamps["EndNs"] df_pmc_perf["EndNs"] = df_stamps["EndNs"]
df_pmc_perf.to_csv(fname, index=False) df_pmc_perf.to_csv(fname, index=False)
else: else:
warning = "WARNING: Incomplete profiling data detected. Unable to update timestamps."
warnings.warn( warnings.warn(
"WARNING: Incomplete profiling data detected. Unable to update timestamps." warning
) )
log_file.write(warning + "\n")
def gen_sysinfo(workload_name, workload_dir, ip_blocks, app_cmd, skip_roof): def gen_sysinfo(workload_name, workload_dir, ip_blocks, app_cmd, skip_roof):
@@ -408,7 +410,7 @@ def characterize_app(args, VER):
for fname in glob.glob(workload_dir + "/perfmon/*.txt"): for fname in glob.glob(workload_dir + "/perfmon/*.txt"):
# Kernel filtering (in-place replacement) # Kernel filtering (in-place replacement)
if not args.kernel == None: if not args.kernel == None:
run_subprocess( success, output = capture_subprocess_output(
[ [
"sed", "sed",
"-i", "-i",
@@ -417,10 +419,11 @@ def characterize_app(args, VER):
fname, fname,
] ]
) )
log.write(output)
# Dispatch filtering (inplace replacement) # Dispatch filtering (inplace replacement)
if not args.dispatch == None: if not args.dispatch == None:
run_subprocess( success, output = capture_subprocess_output(
[ [
"sed", "sed",
"-i", "-i",
@@ -429,17 +432,17 @@ def characterize_app(args, VER):
fname, fname,
] ]
) )
log.write(output)
print(fname) print(fname)
if args.use_rocscope == True: if args.use_rocscope == True:
run_rocscope(args, fname) run_rocscope(args, fname)
else: else:
run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, log, args.verbose) run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, log, args.verbose)
# Close log
log.close()
# run again with timestamps # run again with timestamps
run_subprocess( success, output = capture_subprocess_output(
[ [
rocprof_cmd, rocprof_cmd,
# "-i", fname, # "-i", fname,
@@ -451,12 +454,16 @@ def characterize_app(args, VER):
'"' + app_cmd + '"', '"' + app_cmd + '"',
] ]
) )
log.write(output)
# Update pmc_perf.csv timestamps # Update pmc_perf.csv timestamps
replace_timestamps(workload_dir) replace_timestamps(workload_dir, log)
# Manually join each pmc_perf*.csv output # Manually join each pmc_perf*.csv output
if args.use_rocscope == False: if args.use_rocscope == False:
join_prof(workload_dir, args.join_type) join_prof(workload_dir, args.join_type, log, args.verbose)
# Close log
log.close()
################################################ ################################################
@@ -640,7 +647,7 @@ def omniperf_profile(args, VER):
for fname in glob.glob(workload_dir + "/perfmon/*.txt"): for fname in glob.glob(workload_dir + "/perfmon/*.txt"):
# Kernel filtering (in-place replacement) # Kernel filtering (in-place replacement)
if not args.kernel == None: if not args.kernel == None:
run_subprocess( success, output = capture_subprocess_output(
[ [
"sed", "sed",
"-i", "-i",
@@ -649,10 +656,11 @@ def omniperf_profile(args, VER):
fname, fname,
] ]
) )
log.write(output)
# Dispatch filtering (inplace replacement) # Dispatch filtering (inplace replacement)
if not args.dispatch == None: if not args.dispatch == None:
run_subprocess( success, output = capture_subprocess_output(
[ [
"sed", "sed",
"-i", "-i",
@@ -661,6 +669,7 @@ def omniperf_profile(args, VER):
fname, fname,
] ]
) )
log.write(output)
print(fname) print(fname)
if args.use_rocscope == True: if args.use_rocscope == True:
run_rocscope(args, fname) run_rocscope(args, fname)
@@ -668,7 +677,7 @@ def omniperf_profile(args, VER):
run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose) run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose)
# run again with timestamps # run again with timestamps
run_subprocess( success, output = capture_subprocess_output(
[ [
rocprof_cmd, rocprof_cmd,
# "-i", fname, # "-i", fname,
@@ -680,12 +689,13 @@ def omniperf_profile(args, VER):
'"' + args.remaining + '"', '"' + args.remaining + '"',
] ]
) )
log.write(output)
# Update pmc_perf.csv timestamps # Update pmc_perf.csv timestamps
replace_timestamps(workload_dir) replace_timestamps(workload_dir, log)
# Manually join each pmc_perf*.csv output # Manually join each pmc_perf*.csv output
if args.use_rocscope == False: if args.use_rocscope == False:
join_prof(workload_dir, args.join_type) join_prof(workload_dir, args.join_type, log, args.verbose)
# Generate sysinfo # Generate sysinfo
gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof) gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof)
src/utils/perfagg.py
+49 -17
View File
@@ -25,6 +25,7 @@
import sys, os, pathlib, shutil, subprocess, argparse, glob, re import sys, os, pathlib, shutil, subprocess, argparse, glob, re
import numpy as np import numpy as np
import math import math
import warnings
import pandas as pd import pandas as pd
prog = "omniperf" prog = "omniperf"
@@ -87,8 +88,12 @@ perfmon_config = {
} }
def test_df_column_equality(df):
return df.eq(df.iloc[:, 0], axis=0).all(1).all()
# joins disparate runs less dumbly than rocprof # joins disparate runs less dumbly than rocprof
def join_prof(workload_dir, join_type, out=None): def join_prof(workload_dir, join_type, log_file, verbose, out=None):
# Set default output directory if not specified # Set default output directory if not specified
if out == None: if out == None:
out = workload_dir + "/pmc_perf.csv" out = workload_dir + "/pmc_perf.csv"
@@ -96,25 +101,48 @@ def join_prof(workload_dir, join_type, out=None):
df = None df = None
for i, file in enumerate(files): for i, file in enumerate(files):
# _df = parse_rocprof_kernels(file)
_df = pd.read_csv(file) _df = pd.read_csv(file)
key = _df.groupby("KernelName").cumcount()
if join_type == "kernel": if join_type == "kernel":
_df["key"] = _df.KernelName + " - " + key.astype(str) key = _df.groupby("KernelName").cumcount()
elif join_type == "grid": elif join_type == "grid":
_df["key"] = ( key = _df.groupby(["KernelName", "grd"]).cumcount()
_df.KernelName + " - " + key.astype(str) + " - " + _df.grd.astype(str)
)
else: else:
print("ERROR: Unrecognized --join-type") print("ERROR: Unrecognized --join-type")
sys.exit(1) sys.exit(1)
_df["key"] = _df.KernelName + " - " + key.astype(str)
if df is None: if df is None:
df = _df df = _df
else: else:
# join by unique index of kernel # join by unique index of kernel
df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}")) df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}"))
# TODO: check for any mismatch in joins
duplicate_cols = {
"gpu": [col for col in df.columns if "gpu" in col],
"grd": [col for col in df.columns if "grd" in col],
"wpr": [col for col in df.columns if "wgr" in col],
"lds": [col for col in df.columns if "lds" in col],
"scr": [col for col in df.columns if "scr" in col],
"arch_vgpr": [col for col in df.columns if "arch_vgpr" in col],
"accum_vgpr": [col for col in df.columns if "accum_vgpr" in col],
"spgr": [col for col in df.columns if "sgpr" in col],
}
for key, cols in duplicate_cols.items():
_df = df[cols]
if not test_df_column_equality(_df):
msg = (
"WARNING: Detected differing {} values while joining pmc_perf.csv".format(
key
)
)
warnings.warn(msg)
log_file.write(msg + "\n")
if test_df_column_equality(_df) and verbose:
msg = "Successfully joined {} in pmc_perf.csv".format(key)
print(msg)
log_file.write(msg + "\n")
# now, we can: # now, we can:
#   A) throw away any of the "boring" duplicats #   A) throw away any of the "boring" duplicats
df = df[ df = df[
@@ -124,17 +152,20 @@ def join_prof(workload_dir, join_type, out=None):
if not any( if not any(
check in k check in k
for check in [ for check in [
# "gpu", # removed merged counters, keep original
"gpu-id_",
"grd_",
"wgr_",
"lds_",
"scr_",
"vgpr_",
"sgpr_",
"Index_",
# un-mergable, remove all
"queue-id", "queue-id",
"queue-index", "queue-index",
"pid", "pid",
"tid", "tid",
# "grd",
# "wgr",
# "lds",
# "scr",
# "vgpr",
# "sgpr",
"fbar", "fbar",
"sig", "sig",
"obj", "obj",
@@ -178,8 +209,9 @@ def join_prof(workload_dir, join_type, out=None):
# and save to file # and save to file
df.to_csv(out, index=False) df.to_csv(out, index=False)
# and delete old file(s) # and delete old file(s)
for file in files: if not verbose:
os.remove(file) for file in files:
os.remove(file)
def pmc_perf_split(workload_dir): def pmc_perf_split(workload_dir):