From cd0dc014014550bbd13561f7a1caf6fbed701bcf Mon Sep 17 00:00:00 2001 From: josantos Date: Thu, 10 Aug 2023 11:13:27 -0500 Subject: [PATCH] Names shortened/demangled after join_prof - Added kernelVerbose flag in profile_group - Added KernelVerbose flag in analyze_group - Analyze replaces csv with shortened/demangled name - csv_converter uses llvm-cxxfilt Signed-off-by: josantos [ROCm/rocprofiler-compute commit: 59d77f9d8127ac6734ab11736023dbeb169e12c2] --- projects/rocprofiler-compute/src/omniperf | 67 ++++++++++++------- .../src/omniperf_analyze/omniperf_analyze.py | 10 +++ projects/rocprofiler-compute/src/parser.py | 18 +++++ .../src/utils/csv_converter.py | 26 ++++--- 4 files changed, 86 insertions(+), 35 deletions(-) diff --git a/projects/rocprofiler-compute/src/omniperf b/projects/rocprofiler-compute/src/omniperf index acdfbc3bfe..4195ca64a7 100755 --- a/projects/rocprofiler-compute/src/omniperf +++ b/projects/rocprofiler-compute/src/omniperf @@ -446,6 +446,21 @@ def characterize_app(args, VER): # Manually join each pmc_perf*.csv output if args.use_rocscope == False: join_prof(workload_dir, args.join_type, log, args.verbose) + #demangle + for filename in os.listdir(workload_dir): + try: + # fileName = file[0 : file.find(".")] + # Only shorten KernelNames if instructed to + if args.kernelVerbose < 5: + t1 = pd.read_csv( + os.path.join(workload_dir, filename), + on_bad_lines="skip", + engine="python", + ) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) + t2.to_csv(fname, index=False) + except pd.errors.EmptyDataError: + print("Skipping empty csv " + filename) # Close log log.close() @@ -660,6 +675,7 @@ def omniperf_profile(args, VER): run_rocscope(args, fname) else: run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose) + # Update timestamps replace_timestamps(workload_dir, log) @@ -667,6 +683,22 @@ def omniperf_profile(args, VER): # Manually join each pmc_perf*.csv output if args.use_rocscope == False: join_prof(workload_dir, args.join_type, log, args.verbose) + #demangle + for filename in os.listdir(workload_dir): + if filename.endswith('.csv'): + try: + # fileName = file[0 : file.find(".")] + # Only shorten KernelNames if instructed to + if args.kernelVerbose < 5: + t1 = pd.read_csv( + os.path.join(workload_dir, filename), + on_bad_lines="skip", + engine="python", + ) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) + t2.to_csv(os.path.join(workload_dir, filename), index=False) + except pd.errors.EmptyDataError: + print("Skipping empty csv " + filename) # Generate sysinfo gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof) @@ -739,7 +771,7 @@ def main(): # PROFILE MODE ############## if args.mode == "profile": - Extractionlvl = 3 #args.extraction_level + Extractionlvl = args.kernelVerbose print("Resolving rocprof") resolve_rocprof() # Cannot access parent directories @@ -777,43 +809,28 @@ def main(): roof_setup(args, my_parser, VER) # Generate roofline roofline_only(args.path, args.device, args.sort, args.mem_level, args.kernel_names, args.verbose) - #demangle - for file in os.listdir(args.path): - if file.endswith(".csv"): - try: - fileName = file[0 : file.find(".")] - # Only shorten KernelNames if instructed to - if Extractionlvl < 5: - t1 = pd.read_csv( - os.listdir(args.path) + "/" + file, - on_bad_lines="skip", - engine="python", - ) - - t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl) - except pd.errors.EmptyDataError: - print("Skipping empty csv " + file) # Profile only else: print("\n-------------\nProfile only\n-------------\n") omniperf_profile(args, VER) + workload_dir = args.path #demangle - for file in os.listdir(args.path): - if file.endswith(".csv"): + for filename in os.listdir(workload_dir): + if filename.endswith('.csv'): try: - fileName = file[0 : file.find(".")] + # fileName = file[0 : file.find(".")] # Only shorten KernelNames if instructed to - if Extractionlvl < 5: + if args.kernelVerbose < 5: t1 = pd.read_csv( - os.listdir(args.path) + "/" + file, + os.path.join(workload_dir, filename), on_bad_lines="skip", engine="python", ) - - t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) + t2.to_csv(os.path.join(workload_dir, filename), index=False) except pd.errors.EmptyDataError: - print("Skipping empty csv " + file) + print("Skipping empty csv " + filename) ############## # DATABASE MODE diff --git a/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py b/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py index 87fac064db..2a657d3d30 100644 --- a/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py +++ b/projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py @@ -45,6 +45,8 @@ import os.path from pathlib import Path from omniperf_analyze.utils import parser, file_io from omniperf_analyze.utils.gui_components.roofline import get_roofline +from utils import csv_converter +import pandas as pd archConfigs = {} @@ -220,7 +222,15 @@ def run_cli(args, runs): # If we assume the panel layout for all archs are similar, it doesn't matter # which archConfig passed into show_all function. # After decide to how to manage kernels display patterns, we can revisit it. + cache =dict() for d in args.path: + #demangle + for filename in os.listdir(d[0]): + if filename.endswith('.csv'): + df = pd.read_csv(os.path.join(d[0],filename)) + new_df = csv_converter.kernel_name_shortener(df, cache, args.kernelVerbose) + new_df.to_csv(os.path.join(d[0],filename), index=False) + file_io.create_df_kernel_top_stats( d[0], runs[d[0]].filter_gpu_ids, diff --git a/projects/rocprofiler-compute/src/parser.py b/projects/rocprofiler-compute/src/parser.py index 9d6dd8f6f2..8de09542f1 100644 --- a/projects/rocprofiler-compute/src/parser.py +++ b/projects/rocprofiler-compute/src/parser.py @@ -204,6 +204,15 @@ def parse(my_parser): nargs=argparse.REMAINDER, help="\t\t\tProvide command for profiling after double dash.", ) + profile_group.add_argument( + "-f", + "--kernelVerbose", + required=False, + metavar="", + help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", + default=2, + type=int, + ) ## Roofline Command Line Options roofline_group.add_argument( @@ -514,3 +523,12 @@ def parse(my_parser): action="store_true", help="\t\tRandomly generate a port to launch GUI application.\n\t\tRegistered Ports range inclusive (1024-49151).", ) + analyze_group.add_argument( + "-f", + "--kernelVerbose", + required=False, + metavar="", + help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", + default=2, + type=int, + ) diff --git a/projects/rocprofiler-compute/src/utils/csv_converter.py b/projects/rocprofiler-compute/src/utils/csv_converter.py index 34f2e82614..35bc5a649f 100644 --- a/projects/rocprofiler-compute/src/utils/csv_converter.py +++ b/projects/rocprofiler-compute/src/utils/csv_converter.py @@ -25,6 +25,7 @@ import argparse import collections import os +import subprocess import sys import re import pandas as pd @@ -33,11 +34,11 @@ from pymongo import MongoClient from tqdm import tqdm import shutil + cache = dict() supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"} MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout - def kernel_name_shortener(df, cache, level): if level >= 5: return df @@ -55,6 +56,13 @@ def kernel_name_shortener(df, cache, level): if original_name in cache: continue + cmd = ["llvm-cxxfilt", original_name] + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + demangled_name, e = proc.communicate() + demangled_name = str(demangled_name, "UTF-8").strip() + # cache miss, add the shortened name to the dictionary new_name = "" matches = "" @@ -62,15 +70,14 @@ def kernel_name_shortener(df, cache, level): names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] - if names_and_args.search(original_name): - matches = names_and_args.findall(original_name) + if names_and_args.search(demangled_name): + matches = names_and_args.findall(demangled_name) else: # Works for first case '__amd_rocclr_fillBuffer.kd' - # remove .kd and then parse through original regex - first_case = re.compile(r"([^\s]+)(.kd)") - Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") - interim_name = first_case.search(original_name).group(1) - matches = Mod_name_and_args.findall(interim_name) + cache[original_name] = new_name + if new_name == None or new_name == "": + cache[original_name] = demangled_name + continue current_level = 0 for name in matches: @@ -103,13 +110,12 @@ def kernel_name_shortener(df, cache, level): cache[original_name] = new_name if new_name == None or new_name == "": - cache[original_name] = original_name + cache[original_name] = demangled_name df[columnName] = df[columnName].map(cache) return df - # Verify target directory and setup connection def parse(args, profileAndExport): host = args.host