Enhance logging and warning reporting
Signed-off-by: coleramos425 <colramos@amd.com>
Dieser Commit ist enthalten in:
coleramos425
+24
-14
@@ -160,7 +160,7 @@ def isWorkloadEmpty(my_parser, path):
|
||||
)
|
||||
|
||||
|
||||
def replace_timestamps(workload_dir):
|
||||
def replace_timestamps(workload_dir, log_file):
|
||||
df_stamps = pd.read_csv(workload_dir + "/timestamps.csv")
|
||||
if "BeginNs" in df_stamps.columns and "EndNs" in df_stamps.columns:
|
||||
# Update timestamps for all *.csv output files
|
||||
@@ -171,9 +171,11 @@ def replace_timestamps(workload_dir):
|
||||
df_pmc_perf["EndNs"] = df_stamps["EndNs"]
|
||||
df_pmc_perf.to_csv(fname, index=False)
|
||||
else:
|
||||
warning = "WARNING: Incomplete profiling data detected. Unable to update timestamps."
|
||||
warnings.warn(
|
||||
"WARNING: Incomplete profiling data detected. Unable to update timestamps."
|
||||
warning
|
||||
)
|
||||
log_file.write(warning + "\n")
|
||||
|
||||
|
||||
def gen_sysinfo(workload_name, workload_dir, ip_blocks, app_cmd, skip_roof):
|
||||
@@ -408,7 +410,7 @@ def characterize_app(args, VER):
|
||||
for fname in glob.glob(workload_dir + "/perfmon/*.txt"):
|
||||
# Kernel filtering (in-place replacement)
|
||||
if not args.kernel == None:
|
||||
run_subprocess(
|
||||
success, output = capture_subprocess_output(
|
||||
[
|
||||
"sed",
|
||||
"-i",
|
||||
@@ -417,10 +419,11 @@ def characterize_app(args, VER):
|
||||
fname,
|
||||
]
|
||||
)
|
||||
log.write(output)
|
||||
|
||||
# Dispatch filtering (inplace replacement)
|
||||
if not args.dispatch == None:
|
||||
run_subprocess(
|
||||
success, output = capture_subprocess_output(
|
||||
[
|
||||
"sed",
|
||||
"-i",
|
||||
@@ -429,17 +432,17 @@ def characterize_app(args, VER):
|
||||
fname,
|
||||
]
|
||||
)
|
||||
log.write(output)
|
||||
print(fname)
|
||||
if args.use_rocscope == True:
|
||||
run_rocscope(args, fname)
|
||||
else:
|
||||
run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, log, args.verbose)
|
||||
|
||||
# Close log
|
||||
log.close()
|
||||
|
||||
|
||||
# run again with timestamps
|
||||
run_subprocess(
|
||||
success, output = capture_subprocess_output(
|
||||
[
|
||||
rocprof_cmd,
|
||||
# "-i", fname,
|
||||
@@ -451,12 +454,16 @@ def characterize_app(args, VER):
|
||||
'"' + app_cmd + '"',
|
||||
]
|
||||
)
|
||||
log.write(output)
|
||||
# Update pmc_perf.csv timestamps
|
||||
replace_timestamps(workload_dir)
|
||||
replace_timestamps(workload_dir, log)
|
||||
|
||||
# Manually join each pmc_perf*.csv output
|
||||
if args.use_rocscope == False:
|
||||
join_prof(workload_dir, args.join_type)
|
||||
join_prof(workload_dir, args.join_type, log, args.verbose)
|
||||
|
||||
# Close log
|
||||
log.close()
|
||||
|
||||
|
||||
################################################
|
||||
@@ -640,7 +647,7 @@ def omniperf_profile(args, VER):
|
||||
for fname in glob.glob(workload_dir + "/perfmon/*.txt"):
|
||||
# Kernel filtering (in-place replacement)
|
||||
if not args.kernel == None:
|
||||
run_subprocess(
|
||||
success, output = capture_subprocess_output(
|
||||
[
|
||||
"sed",
|
||||
"-i",
|
||||
@@ -649,10 +656,11 @@ def omniperf_profile(args, VER):
|
||||
fname,
|
||||
]
|
||||
)
|
||||
log.write(output)
|
||||
|
||||
# Dispatch filtering (inplace replacement)
|
||||
if not args.dispatch == None:
|
||||
run_subprocess(
|
||||
success, output = capture_subprocess_output(
|
||||
[
|
||||
"sed",
|
||||
"-i",
|
||||
@@ -661,6 +669,7 @@ def omniperf_profile(args, VER):
|
||||
fname,
|
||||
]
|
||||
)
|
||||
log.write(output)
|
||||
print(fname)
|
||||
if args.use_rocscope == True:
|
||||
run_rocscope(args, fname)
|
||||
@@ -668,7 +677,7 @@ def omniperf_profile(args, VER):
|
||||
run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose)
|
||||
|
||||
# run again with timestamps
|
||||
run_subprocess(
|
||||
success, output = capture_subprocess_output(
|
||||
[
|
||||
rocprof_cmd,
|
||||
# "-i", fname,
|
||||
@@ -680,12 +689,13 @@ def omniperf_profile(args, VER):
|
||||
'"' + args.remaining + '"',
|
||||
]
|
||||
)
|
||||
log.write(output)
|
||||
# Update pmc_perf.csv timestamps
|
||||
replace_timestamps(workload_dir)
|
||||
replace_timestamps(workload_dir, log)
|
||||
|
||||
# Manually join each pmc_perf*.csv output
|
||||
if args.use_rocscope == False:
|
||||
join_prof(workload_dir, args.join_type)
|
||||
join_prof(workload_dir, args.join_type, log, args.verbose)
|
||||
|
||||
# Generate sysinfo
|
||||
gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof)
|
||||
|
||||
+49
-17
@@ -25,6 +25,7 @@
|
||||
import sys, os, pathlib, shutil, subprocess, argparse, glob, re
|
||||
import numpy as np
|
||||
import math
|
||||
import warnings
|
||||
import pandas as pd
|
||||
|
||||
prog = "omniperf"
|
||||
@@ -87,8 +88,12 @@ perfmon_config = {
|
||||
}
|
||||
|
||||
|
||||
def test_df_column_equality(df):
|
||||
return df.eq(df.iloc[:, 0], axis=0).all(1).all()
|
||||
|
||||
|
||||
# joins disparate runs less dumbly than rocprof
|
||||
def join_prof(workload_dir, join_type, out=None):
|
||||
def join_prof(workload_dir, join_type, log_file, verbose, out=None):
|
||||
# Set default output directory if not specified
|
||||
if out == None:
|
||||
out = workload_dir + "/pmc_perf.csv"
|
||||
@@ -96,25 +101,48 @@ def join_prof(workload_dir, join_type, out=None):
|
||||
df = None
|
||||
|
||||
for i, file in enumerate(files):
|
||||
# _df = parse_rocprof_kernels(file)
|
||||
|
||||
_df = pd.read_csv(file)
|
||||
key = _df.groupby("KernelName").cumcount()
|
||||
if join_type == "kernel":
|
||||
_df["key"] = _df.KernelName + " - " + key.astype(str)
|
||||
key = _df.groupby("KernelName").cumcount()
|
||||
elif join_type == "grid":
|
||||
_df["key"] = (
|
||||
_df.KernelName + " - " + key.astype(str) + " - " + _df.grd.astype(str)
|
||||
)
|
||||
key = _df.groupby(["KernelName", "grd"]).cumcount()
|
||||
else:
|
||||
print("ERROR: Unrecognized --join-type")
|
||||
sys.exit(1)
|
||||
|
||||
_df["key"] = _df.KernelName + " - " + key.astype(str)
|
||||
if df is None:
|
||||
df = _df
|
||||
else:
|
||||
# join by unique index of kernel
|
||||
df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}"))
|
||||
|
||||
# TODO: check for any mismatch in joins
|
||||
duplicate_cols = {
|
||||
"gpu": [col for col in df.columns if "gpu" in col],
|
||||
"grd": [col for col in df.columns if "grd" in col],
|
||||
"wpr": [col for col in df.columns if "wgr" in col],
|
||||
"lds": [col for col in df.columns if "lds" in col],
|
||||
"scr": [col for col in df.columns if "scr" in col],
|
||||
"arch_vgpr": [col for col in df.columns if "arch_vgpr" in col],
|
||||
"accum_vgpr": [col for col in df.columns if "accum_vgpr" in col],
|
||||
"spgr": [col for col in df.columns if "sgpr" in col],
|
||||
}
|
||||
for key, cols in duplicate_cols.items():
|
||||
_df = df[cols]
|
||||
if not test_df_column_equality(_df):
|
||||
msg = (
|
||||
"WARNING: Detected differing {} values while joining pmc_perf.csv".format(
|
||||
key
|
||||
)
|
||||
)
|
||||
warnings.warn(msg)
|
||||
log_file.write(msg + "\n")
|
||||
if test_df_column_equality(_df) and verbose:
|
||||
msg = "Successfully joined {} in pmc_perf.csv".format(key)
|
||||
print(msg)
|
||||
log_file.write(msg + "\n")
|
||||
|
||||
# now, we can:
|
||||
# A) throw away any of the "boring" duplicats
|
||||
df = df[
|
||||
@@ -124,17 +152,20 @@ def join_prof(workload_dir, join_type, out=None):
|
||||
if not any(
|
||||
check in k
|
||||
for check in [
|
||||
# "gpu",
|
||||
# removed merged counters, keep original
|
||||
"gpu-id_",
|
||||
"grd_",
|
||||
"wgr_",
|
||||
"lds_",
|
||||
"scr_",
|
||||
"vgpr_",
|
||||
"sgpr_",
|
||||
"Index_",
|
||||
# un-mergable, remove all
|
||||
"queue-id",
|
||||
"queue-index",
|
||||
"pid",
|
||||
"tid",
|
||||
# "grd",
|
||||
# "wgr",
|
||||
# "lds",
|
||||
# "scr",
|
||||
# "vgpr",
|
||||
# "sgpr",
|
||||
"fbar",
|
||||
"sig",
|
||||
"obj",
|
||||
@@ -178,8 +209,9 @@ def join_prof(workload_dir, join_type, out=None):
|
||||
# and save to file
|
||||
df.to_csv(out, index=False)
|
||||
# and delete old file(s)
|
||||
for file in files:
|
||||
os.remove(file)
|
||||
if not verbose:
|
||||
for file in files:
|
||||
os.remove(file)
|
||||
|
||||
|
||||
def pmc_perf_split(workload_dir):
|
||||
|
||||
In neuem Issue referenzieren
Einen Benutzer sperren