diff --git a/projects/rocprofiler-compute/src/omniperf b/projects/rocprofiler-compute/src/omniperf index f3388bdf42..41fce2fbbe 100755 --- a/projects/rocprofiler-compute/src/omniperf +++ b/projects/rocprofiler-compute/src/omniperf @@ -38,7 +38,7 @@ import warnings from parser import parse from utils import specs -from utils.perfagg import perfmon_filter, pmc_filter +from utils.perfagg import perfmon_filter, pmc_filter, pmc_perf_split, join_prof from utils import remove_workload from utils import csv_converter # Import workload from omniperf_analyze.omniperf_analyze import roofline_only # Standalone roofline @@ -163,11 +163,13 @@ def isWorkloadEmpty(my_parser, path): def replace_timestamps(workload_dir): df_stamps = pd.read_csv(workload_dir + "/timestamps.csv") if "BeginNs" in df_stamps.columns and "EndNs" in df_stamps.columns: - df_pmc_perf = pd.read_csv(workload_dir + "/pmc_perf.csv") + # Update timestamps for all *.csv output files + for fname in glob.glob(workload_dir + "/" + "*.csv"): + df_pmc_perf = pd.read_csv(fname) - df_pmc_perf["BeginNs"] = df_stamps["BeginNs"] - df_pmc_perf["EndNs"] = df_stamps["EndNs"] - df_pmc_perf.to_csv(workload_dir + "/pmc_perf.csv", index=False) + df_pmc_perf["BeginNs"] = df_stamps["BeginNs"] + df_pmc_perf["EndNs"] = df_stamps["EndNs"] + df_pmc_perf.to_csv(fname, index=False) else: warnings.warn( "WARNING: Incomplete profiling data detected. Unable to update timestamps." @@ -395,6 +397,9 @@ def characterize_app(args, VER): # Perfmon filtering pmc_filter(workload_dir, perfmon_dir, args.target) + # Separate pmc_perf runs + pmc_perf_split(workload_dir, perfmon_dir) + # Set up a log file log = open(workload_dir + "/log.txt", "w") print("Log: ", workload_dir + "/log.txt\n") @@ -449,6 +454,10 @@ def characterize_app(args, VER): # Update pmc_perf.csv timestamps replace_timestamps(workload_dir) + # Manually join each pmc_perf*.csv output + if args.use_rocscope == False: + join_prof(workload_dir, workload_dir + "/pmc_perf_NEW.csv") + ################################################ # Profiling Helpers @@ -551,6 +560,9 @@ def omniperf_profile(args, VER): # Perfmon filtering perfmon_filter(workload_dir, perfmon_dir, args) + # Separate pmc_perf runs + pmc_perf_split(workload_dir) + # Set up a log file log = open(workload_dir + "/log.txt", "w") print("Log: ", workload_dir + "/log.txt\n") @@ -670,6 +682,10 @@ def omniperf_profile(args, VER): ) # Update pmc_perf.csv timestamps replace_timestamps(workload_dir) + + # Manually join each pmc_perf*.csv output + if args.use_rocscope == False: + join_prof(workload_dir, workload_dir + "/pmc_perf.csv") # Generate sysinfo gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof) diff --git a/projects/rocprofiler-compute/src/utils/perfagg.py b/projects/rocprofiler-compute/src/utils/perfagg.py index 0606b4dc4b..6dd5eb4f47 100755 --- a/projects/rocprofiler-compute/src/utils/perfagg.py +++ b/projects/rocprofiler-compute/src/utils/perfagg.py @@ -25,6 +25,7 @@ import sys, os, pathlib, shutil, subprocess, argparse, glob, re import numpy as np import math +import pandas as pd prog = "omniperf" @@ -85,6 +86,94 @@ perfmon_config = { }, } +# joins disparate runs less dumbly than rocprof +def join_prof(workload_dir, out): + files = glob.glob(workload_dir + "/" + "pmc_perf_*.csv") + df = None + + for i, file in enumerate(files): + #_df = parse_rocprof_kernels(file) + _df = pd.read_csv(file) + key = _df.groupby("KernelName").cumcount() + _df['key'] = _df.KernelName + ' - ' + key.astype(str) + + if df is None: + df = _df + else: + # join by unique index of kernel + df = pd.merge(df, _df, how='inner', on='key', suffixes=('', f'_{i}')) + # now, we can: + #   A) throw away any of the "boring" duplicats + df = df[[k for k in df.keys() if not any( + check in k for check in [ + 'gpu', 'queue-id', 'queue-index', 'pid', 'tid', 'grd', 'wgr', + 'lds', 'scr', 'vgpr', 'sgpr', 'fbar', 'sig', 'obj'])]] + #   B) any timestamps that are _not_ the duration, which is the one we care + #   about + df = df[[k for k in df.keys() if not any( + check in k for check in [ + 'stop', 'start', 'DispatchNs', 'CompleteNs'])]] + #   C) sanity check the name and key + namekeys = [k for k in df.keys() if 'KernelName' in k] + assert len(namekeys) + for k in namekeys[1:]: + assert (df[namekeys[0]] == df[k]).all() + df = df.drop(columns=namekeys[1:]) + # now take the median of the durations + dkeys = [k for k in df.keys() if 'duration' in k] + duration = df[dkeys].median(axis=1) + # compute min and max, just for sanity + min_duration = df[dkeys].min(axis=1) + max_duration = df[dkeys].max(axis=1) + std_duration = df[dkeys].std(axis=1) + mean_duration = df[dkeys].mean(axis=1) + # and replace + df = df.drop(columns=dkeys) + df['duration'] = duration + df['duration[max]'] = max_duration + df['duration[min]'] = min_duration + df['duration[std]'] = std_duration + df['duration[mean]'] = mean_duration + # finally, join the drop key + df = df.drop(columns=['key']) + # and save to file + df.to_csv(out, index=False) + # and delete old file(s) + for file in files: + os.remove(file) + +def pmc_perf_split(workload_dir): + workload_perfmon_dir = workload_dir + "/perfmon" + lines = open(workload_perfmon_dir + "/pmc_perf.txt", "r").read().splitlines() + + # Iterate over each line in pmc_perf.txt + mpattern = r"^pmc:(.*)" + i = 0 + for line in lines: + # Verify no comments + stext = line.split("#")[0].strip() + if not stext: + continue + + # all pmc counters start with "pmc:" + m = re.match(mpattern, stext) + if m is None: + continue + + # Create separate file for each line + fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w") + fd.write(stext + "\n\n") + fd.write("gpu:\n") + fd.write("range:\n") + fd.write("kernel:\n") + fd.close() + + i += 1 + + # Remove old pmc_perf.txt input from perfmon dir + os.remove(workload_perfmon_dir + "/pmc_perf.txt") + + def perfmon_coalesce(pmc_files_list, workload_dir, soc): workload_perfmon_dir = workload_dir + "/perfmon"