From 9360ed8b0ce8aea316543701b53d5fbf2fdab136 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Mon, 31 Jul 2023 14:19:26 -0500 Subject: [PATCH 01/12] Use llvm-cxxfilt to demangle names in kernel_name_shortener Signed-off-by: JoseSantosAMD --- src/utils/csv_converter.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index b178439aca..b3d4db711a 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -32,6 +32,7 @@ import getpass from pymongo import MongoClient from tqdm import tqdm import shutil +import subprocess cache = dict() supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"} @@ -54,6 +55,13 @@ def kernel_name_shortener(df, cache, level): original_name = df.loc[index, columnName] if original_name in cache: continue + + cmd = ["llvm-cxxfilt", original_name] + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + demangled_name, e = proc.communicate() + demangled_name = str(demangled_name, 'UTF-8').strip() # cache miss, add the shortened name to the dictionary new_name = "" @@ -62,14 +70,14 @@ def kernel_name_shortener(df, cache, level): names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] - if names_and_args.search(original_name): - matches = names_and_args.findall(original_name) + if names_and_args.search(demangled_name): + matches = names_and_args.findall(demangled_name) else: # Works for first case '__amd_rocclr_fillBuffer.kd' # remove .kd and then parse through original regex first_case = re.compile(r"([^\s]+)(.kd)") Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") - interim_name = first_case.search(original_name).group(1) + interim_name = first_case.search(demangled_name).group(1) matches = Mod_name_and_args.findall(interim_name) current_level = 0 @@ -103,7 +111,7 @@ def kernel_name_shortener(df, cache, level): cache[original_name] = new_name if new_name == None or new_name == "": - cache[original_name] = original_name + cache[original_name] = demangled_name df[columnName] = df[columnName].map(cache) From b63332a31702b6da293ab0e7b9943598c4a99469 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Wed, 2 Aug 2023 12:38:56 -0500 Subject: [PATCH 02/12] Use llvm-cxxfilt to demangle names Show typed text in in dash-dropdown input box Signed-off-by: JoseSantosAMD --- src/omniperf_analyze/assets/layout.css | 9 +++++++++ src/utils/csv_converter.py | 9 ++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/omniperf_analyze/assets/layout.css b/src/omniperf_analyze/assets/layout.css index 4fcaad24ed..7fdc373f60 100644 --- a/src/omniperf_analyze/assets/layout.css +++ b/src/omniperf_analyze/assets/layout.css @@ -213,6 +213,15 @@ ul#nav li { font-size: 14px; text-align: left; } + +.dash-dropdown input{ + color: black; +} + +.dash-dropdown .Select-placeholder{ + color: black; +} + .VirtualizedSelectOption { overflow: hidden; } diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index b3d4db711a..a3c08c276f 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -74,11 +74,10 @@ def kernel_name_shortener(df, cache, level): matches = names_and_args.findall(demangled_name) else: # Works for first case '__amd_rocclr_fillBuffer.kd' - # remove .kd and then parse through original regex - first_case = re.compile(r"([^\s]+)(.kd)") - Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") - interim_name = first_case.search(demangled_name).group(1) - matches = Mod_name_and_args.findall(interim_name) + cache[original_name] = new_name + if new_name == None or new_name == "": + cache[original_name] = demangled_name + continue current_level = 0 for name in matches: From add68ded6739b02db8c6bec6a73d2e071b8a2683 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Wed, 2 Aug 2023 13:55:08 -0500 Subject: [PATCH 03/12] reformatting Signed-off-by: JoseSantosAMD --- src/utils/csv_converter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index a3c08c276f..4f28d5388e 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -55,13 +55,13 @@ def kernel_name_shortener(df, cache, level): original_name = df.loc[index, columnName] if original_name in cache: continue - + cmd = ["llvm-cxxfilt", original_name] - + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) demangled_name, e = proc.communicate() - demangled_name = str(demangled_name, 'UTF-8').strip() + demangled_name = str(demangled_name, "UTF-8").strip() # cache miss, add the shortened name to the dictionary new_name = "" From 4e51c122d17eb450e48699d99bde712fd5bcfc32 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Tue, 8 Aug 2023 12:06:39 -0500 Subject: [PATCH 04/12] removing calls to kernel_name_shortener in mongo shortening now in profile Signed-off-by: JoseSantosAMD --- src/omniperf | 104 +++++++++++++++++++ src/utils/csv_converter.py | 203 ++++++++++++++++++------------------- 2 files changed, 202 insertions(+), 105 deletions(-) diff --git a/src/omniperf b/src/omniperf index 3b38e419b1..7db92004d7 100755 --- a/src/omniperf +++ b/src/omniperf @@ -53,6 +53,8 @@ from common import ( from common import getVersion +cache = dict() + ################################################ # Helper Functions ################################################ @@ -260,7 +262,76 @@ def mongo_import(args, profileAndImport): csv_converter.convert_folder(connectionInfo, Extractionlvl) print("-- Complete! --") +def kernel_name_shortener(df, cache, level): + if level >= 5: + return df + columnName = "" + if "KernelName" in df: + columnName = "KernelName" + if "Name" in df: + columnName = "Name" + + if columnName == "KernelName" or columnName == "Name": + # loop through all indices + for index in df.index: + original_name = df.loc[index, columnName] + if original_name in cache: + continue + + # cache miss, add the shortened name to the dictionary + new_name = "" + matches = "" + + names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") + + # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] + if names_and_args.search(original_name): + matches = names_and_args.findall(original_name) + else: + # Works for first case '__amd_rocclr_fillBuffer.kd' + # remove .kd and then parse through original regex + first_case = re.compile(r"([^\s]+)(.kd)") + Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") + interim_name = first_case.search(original_name).group(1) + matches = Mod_name_and_args.findall(interim_name) + + current_level = 0 + for name in matches: + ##can cause errors if a function name or argument is equal to 'clone' + if name[0] == "clone": + continue + if len(name) == 3: + if name[2] == "::": + continue + + if current_level < level: + new_name += name[0] + # closing '>' is to be taken account by the while loop + if name[1].count(">") == 0: + if current_level < level: + if not (current_level == level - 1 and name[1].count("<") > 0): + new_name += name[1] + current_level += name[1].count("<") + + curr_index = 0 + # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level + while name[1].count(">") > 0 and curr_index < len(name[1]): + if current_level < level: + new_name += name[1][curr_index:] + current_level -= name[1][curr_index:].count(">") + curr_index = len(name[1]) + elif name[1][curr_index] == (">"): + current_level -= 1 + curr_index += 1 + + cache[original_name] = new_name + if new_name == None or new_name == "": + cache[original_name] = original_name + + df[columnName] = df[columnName].map(cache) + + return df ################################################ # Roofline Helpers ################################################ @@ -768,6 +839,7 @@ def main(): # PROFILE MODE ############## if args.mode == "profile": + Extractionlvl = 3 #args.extraction_level print("Resolving rocprof") resolve_rocprof() # Cannot access parent directories @@ -805,11 +877,43 @@ def main(): roof_setup(args, my_parser, VER) # Generate roofline roofline_only(args.path, args.device, args.sort, args.mem_level, args.kernel_names, args.verbose) + #demangle + for file in os.listdir(args.path): + if file.endswith(".csv"): + try: + fileName = file[0 : file.find(".")] + # Only shorten KernelNames if instructed to + if Extractionlvl < 5: + t1 = pd.read_csv( + os.listdir(args.path) + "/" + file, + on_bad_lines="skip", + engine="python", + ) + + t2 = kernel_name_shortener(t1, cache, level=Extractionlvl) + except pd.errors.EmptyDataError: + print("Skipping empty csv " + file) # Profile only else: print("\n-------------\nProfile only\n-------------\n") omniperf_profile(args, VER) + #demangle + for file in os.listdir(args.path): + if file.endswith(".csv"): + try: + fileName = file[0 : file.find(".")] + # Only shorten KernelNames if instructed to + if Extractionlvl < 5: + t1 = pd.read_csv( + os.listdir(args.path) + "/" + file, + on_bad_lines="skip", + engine="python", + ) + + t2 = kernel_name_shortener(t1, cache, level=Extractionlvl) + except pd.errors.EmptyDataError: + print("Skipping empty csv " + file) ############## # DATABASE MODE diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index 4f28d5388e..9709ab9d87 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -32,89 +32,82 @@ import getpass from pymongo import MongoClient from tqdm import tqdm import shutil -import subprocess -cache = dict() +# cache = dict() supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"} MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout -def kernel_name_shortener(df, cache, level): - if level >= 5: - return df +# def kernel_name_shortener(df, cache, level): +# if level >= 5: +# return df - columnName = "" - if "KernelName" in df: - columnName = "KernelName" - if "Name" in df: - columnName = "Name" +# columnName = "" +# if "KernelName" in df: +# columnName = "KernelName" +# if "Name" in df: +# columnName = "Name" - if columnName == "KernelName" or columnName == "Name": - # loop through all indices - for index in df.index: - original_name = df.loc[index, columnName] - if original_name in cache: - continue +# if columnName == "KernelName" or columnName == "Name": +# # loop through all indices +# for index in df.index: +# original_name = df.loc[index, columnName] +# if original_name in cache: +# continue - cmd = ["llvm-cxxfilt", original_name] +# # cache miss, add the shortened name to the dictionary +# new_name = "" +# matches = "" - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +# names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") - demangled_name, e = proc.communicate() - demangled_name = str(demangled_name, "UTF-8").strip() +# # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] +# if names_and_args.search(original_name): +# matches = names_and_args.findall(original_name) +# else: +# # Works for first case '__amd_rocclr_fillBuffer.kd' +# # remove .kd and then parse through original regex +# first_case = re.compile(r"([^\s]+)(.kd)") +# Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") +# interim_name = first_case.search(original_name).group(1) +# matches = Mod_name_and_args.findall(interim_name) - # cache miss, add the shortened name to the dictionary - new_name = "" - matches = "" +# current_level = 0 +# for name in matches: +# ##can cause errors if a function name or argument is equal to 'clone' +# if name[0] == "clone": +# continue +# if len(name) == 3: +# if name[2] == "::": +# continue - names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") +# if current_level < level: +# new_name += name[0] +# # closing '>' is to be taken account by the while loop +# if name[1].count(">") == 0: +# if current_level < level: +# if not (current_level == level - 1 and name[1].count("<") > 0): +# new_name += name[1] +# current_level += name[1].count("<") - # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] - if names_and_args.search(demangled_name): - matches = names_and_args.findall(demangled_name) - else: - # Works for first case '__amd_rocclr_fillBuffer.kd' - cache[original_name] = new_name - if new_name == None or new_name == "": - cache[original_name] = demangled_name - continue +# curr_index = 0 +# # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level +# while name[1].count(">") > 0 and curr_index < len(name[1]): +# if current_level < level: +# new_name += name[1][curr_index:] +# current_level -= name[1][curr_index:].count(">") +# curr_index = len(name[1]) +# elif name[1][curr_index] == (">"): +# current_level -= 1 +# curr_index += 1 - current_level = 0 - for name in matches: - ##can cause errors if a function name or argument is equal to 'clone' - if name[0] == "clone": - continue - if len(name) == 3: - if name[2] == "::": - continue +# cache[original_name] = new_name +# if new_name == None or new_name == "": +# cache[original_name] = original_name - if current_level < level: - new_name += name[0] - # closing '>' is to be taken account by the while loop - if name[1].count(">") == 0: - if current_level < level: - if not (current_level == level - 1 and name[1].count("<") > 0): - new_name += name[1] - current_level += name[1].count("<") +# df[columnName] = df[columnName].map(cache) - curr_index = 0 - # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level - while name[1].count(">") > 0 and curr_index < len(name[1]): - if current_level < level: - new_name += name[1][curr_index:] - current_level -= name[1][curr_index:].count(">") - curr_index = len(name[1]) - elif name[1][curr_index] == (">"): - current_level -= 1 - curr_index += 1 - - cache[original_name] = new_name - if new_name == None or new_name == "": - cache[original_name] = demangled_name - - df[columnName] = df[columnName].map(cache) - - return df +# return df # Verify target directory and setup connection @@ -151,12 +144,12 @@ def parse(args, profileAndExport): db = "omniperf_" + str(args.team) + "_" + str(name) + "_" + soc - if Extractionlvl >= 5: - print("KernelName shortening disabled") - else: - print("KernelName shortening enabled") + # if Extractionlvl >= 5: + # print("KernelName shortening disabled") + # else: + # print("KernelName shortening enabled") - print("Kernel name verbose level:", Extractionlvl) + # print("Kernel name verbose level:", Extractionlvl) if args.password == "": try: @@ -203,14 +196,14 @@ def convert_folder(connectionInfo, Extractionlvl): print("ERROR: Unable to connect to the server") sys.exit(1) # Set up directories - if Extractionlvl < 5: - newfilepath = connectionInfo["workload"] - newfilepath_h = newfilepath + "/renamedFiles/" - if not os.path.exists(newfilepath_h): - os.mkdir(newfilepath_h) - newfilepath = newfilepath_h + connectionInfo["db"] + "/" - if not os.path.exists(newfilepath): - os.mkdir(newfilepath) + # if Extractionlvl < 5: + # newfilepath = connectionInfo["workload"] + # newfilepath_h = newfilepath + "/renamedFiles/" + # if not os.path.exists(newfilepath_h): + # os.mkdir(newfilepath_h) + # newfilepath = newfilepath_h + connectionInfo["db"] + "/" + # if not os.path.exists(newfilepath): + # os.mkdir(newfilepath) # Upload files i = 0 file = "blank" @@ -220,30 +213,30 @@ def convert_folder(connectionInfo, Extractionlvl): try: fileName = file[0 : file.find(".")] # Only shorten KernelNames if instructed to - if Extractionlvl < 5: - t1 = pd.read_csv( - connectionInfo["workload"] + "/" + file, - on_bad_lines="skip", - engine="python", - ) + # if Extractionlvl < 5: + # t1 = pd.read_csv( + # connectionInfo["workload"] + "/" + file, + # on_bad_lines="skip", + # engine="python", + # ) - t2 = kernel_name_shortener(t1, cache, level=Extractionlvl) - df_saved_file = t2.to_csv(newfilepath + file) + # t2 = kernel_name_shortener(t1, cache, level=Extractionlvl) + # df_saved_file = t2.to_csv(newfilepath + file) - cmd = ( - "mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline" - ).format( - connectionInfo["username"], - connectionInfo["password"], - connectionInfo["host"], - connectionInfo["port"], - connectionInfo["db"], - newfilepath + file, - fileName, - ) - os.system(cmd) - else: - cmd = ( + # cmd = ( + # "mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline" + # ).format( + # connectionInfo["username"], + # connectionInfo["password"], + # connectionInfo["host"], + # connectionInfo["port"], + # connectionInfo["db"], + # newfilepath + file, + # fileName, + # ) + # os.system(cmd) + # else: + cmd = ( "mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline" ).format( connectionInfo["username"], @@ -254,7 +247,7 @@ def convert_folder(connectionInfo, Extractionlvl): connectionInfo["workload"] + "/" + file, fileName, ) - os.system(cmd) + os.system(cmd) i += 1 except pd.errors.EmptyDataError: print("Skipping empty csv " + file) @@ -265,7 +258,7 @@ def convert_folder(connectionInfo, Extractionlvl): newValue = {"name": connectionInfo["db"]} mycol.replace_one(value, newValue, upsert=True) # Remove tmp directory if we shortened KernelNames - if Extractionlvl < 5: - shutil.rmtree(newfilepath_h) + # if Extractionlvl < 5: + # shutil.rmtree(newfilepath_h) print("{} collections added.".format(i)) print("Workload name uploaded") From 2948f73ae889df82bbc78db78ab0cfa77ef45e68 Mon Sep 17 00:00:00 2001 From: josantos Date: Wed, 9 Aug 2023 10:46:57 -0500 Subject: [PATCH 05/12] keep converter in csv_converter.py Signed-off-by: josantos --- src/omniperf | 74 +---------------------- src/utils/csv_converter.py | 120 ++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 132 deletions(-) diff --git a/src/omniperf b/src/omniperf index 6232ed67b1..acdfbc3bfe 100755 --- a/src/omniperf +++ b/src/omniperf @@ -262,76 +262,6 @@ def mongo_import(args, profileAndImport): csv_converter.convert_folder(connectionInfo, Extractionlvl) print("-- Complete! --") -def kernel_name_shortener(df, cache, level): - if level >= 5: - return df - - columnName = "" - if "KernelName" in df: - columnName = "KernelName" - if "Name" in df: - columnName = "Name" - - if columnName == "KernelName" or columnName == "Name": - # loop through all indices - for index in df.index: - original_name = df.loc[index, columnName] - if original_name in cache: - continue - - # cache miss, add the shortened name to the dictionary - new_name = "" - matches = "" - - names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") - - # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] - if names_and_args.search(original_name): - matches = names_and_args.findall(original_name) - else: - # Works for first case '__amd_rocclr_fillBuffer.kd' - # remove .kd and then parse through original regex - first_case = re.compile(r"([^\s]+)(.kd)") - Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") - interim_name = first_case.search(original_name).group(1) - matches = Mod_name_and_args.findall(interim_name) - - current_level = 0 - for name in matches: - ##can cause errors if a function name or argument is equal to 'clone' - if name[0] == "clone": - continue - if len(name) == 3: - if name[2] == "::": - continue - - if current_level < level: - new_name += name[0] - # closing '>' is to be taken account by the while loop - if name[1].count(">") == 0: - if current_level < level: - if not (current_level == level - 1 and name[1].count("<") > 0): - new_name += name[1] - current_level += name[1].count("<") - - curr_index = 0 - # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level - while name[1].count(">") > 0 and curr_index < len(name[1]): - if current_level < level: - new_name += name[1][curr_index:] - current_level -= name[1][curr_index:].count(">") - curr_index = len(name[1]) - elif name[1][curr_index] == (">"): - current_level -= 1 - curr_index += 1 - - cache[original_name] = new_name - if new_name == None or new_name == "": - cache[original_name] = original_name - - df[columnName] = df[columnName].map(cache) - - return df ################################################ # Roofline Helpers ################################################ @@ -860,7 +790,7 @@ def main(): engine="python", ) - t2 = kernel_name_shortener(t1, cache, level=Extractionlvl) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl) except pd.errors.EmptyDataError: print("Skipping empty csv " + file) @@ -881,7 +811,7 @@ def main(): engine="python", ) - t2 = kernel_name_shortener(t1, cache, level=Extractionlvl) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl) except pd.errors.EmptyDataError: print("Skipping empty csv " + file) diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index 9709ab9d87..34f2e82614 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -33,81 +33,81 @@ from pymongo import MongoClient from tqdm import tqdm import shutil -# cache = dict() +cache = dict() supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"} MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout -# def kernel_name_shortener(df, cache, level): -# if level >= 5: -# return df +def kernel_name_shortener(df, cache, level): + if level >= 5: + return df -# columnName = "" -# if "KernelName" in df: -# columnName = "KernelName" -# if "Name" in df: -# columnName = "Name" + columnName = "" + if "KernelName" in df: + columnName = "KernelName" + if "Name" in df: + columnName = "Name" -# if columnName == "KernelName" or columnName == "Name": -# # loop through all indices -# for index in df.index: -# original_name = df.loc[index, columnName] -# if original_name in cache: -# continue + if columnName == "KernelName" or columnName == "Name": + # loop through all indices + for index in df.index: + original_name = df.loc[index, columnName] + if original_name in cache: + continue -# # cache miss, add the shortened name to the dictionary -# new_name = "" -# matches = "" + # cache miss, add the shortened name to the dictionary + new_name = "" + matches = "" -# names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") + names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") -# # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] -# if names_and_args.search(original_name): -# matches = names_and_args.findall(original_name) -# else: -# # Works for first case '__amd_rocclr_fillBuffer.kd' -# # remove .kd and then parse through original regex -# first_case = re.compile(r"([^\s]+)(.kd)") -# Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") -# interim_name = first_case.search(original_name).group(1) -# matches = Mod_name_and_args.findall(interim_name) + # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] + if names_and_args.search(original_name): + matches = names_and_args.findall(original_name) + else: + # Works for first case '__amd_rocclr_fillBuffer.kd' + # remove .kd and then parse through original regex + first_case = re.compile(r"([^\s]+)(.kd)") + Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") + interim_name = first_case.search(original_name).group(1) + matches = Mod_name_and_args.findall(interim_name) -# current_level = 0 -# for name in matches: -# ##can cause errors if a function name or argument is equal to 'clone' -# if name[0] == "clone": -# continue -# if len(name) == 3: -# if name[2] == "::": -# continue + current_level = 0 + for name in matches: + ##can cause errors if a function name or argument is equal to 'clone' + if name[0] == "clone": + continue + if len(name) == 3: + if name[2] == "::": + continue -# if current_level < level: -# new_name += name[0] -# # closing '>' is to be taken account by the while loop -# if name[1].count(">") == 0: -# if current_level < level: -# if not (current_level == level - 1 and name[1].count("<") > 0): -# new_name += name[1] -# current_level += name[1].count("<") + if current_level < level: + new_name += name[0] + # closing '>' is to be taken account by the while loop + if name[1].count(">") == 0: + if current_level < level: + if not (current_level == level - 1 and name[1].count("<") > 0): + new_name += name[1] + current_level += name[1].count("<") -# curr_index = 0 -# # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level -# while name[1].count(">") > 0 and curr_index < len(name[1]): -# if current_level < level: -# new_name += name[1][curr_index:] -# current_level -= name[1][curr_index:].count(">") -# curr_index = len(name[1]) -# elif name[1][curr_index] == (">"): -# current_level -= 1 -# curr_index += 1 + curr_index = 0 + # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level + while name[1].count(">") > 0 and curr_index < len(name[1]): + if current_level < level: + new_name += name[1][curr_index:] + current_level -= name[1][curr_index:].count(">") + curr_index = len(name[1]) + elif name[1][curr_index] == (">"): + current_level -= 1 + curr_index += 1 -# cache[original_name] = new_name -# if new_name == None or new_name == "": -# cache[original_name] = original_name + cache[original_name] = new_name + if new_name == None or new_name == "": + cache[original_name] = original_name -# df[columnName] = df[columnName].map(cache) + df[columnName] = df[columnName].map(cache) -# return df + return df # Verify target directory and setup connection From 59d77f9d8127ac6734ab11736023dbeb169e12c2 Mon Sep 17 00:00:00 2001 From: josantos Date: Thu, 10 Aug 2023 11:13:27 -0500 Subject: [PATCH 06/12] Names shortened/demangled after join_prof - Added kernelVerbose flag in profile_group - Added KernelVerbose flag in analyze_group - Analyze replaces csv with shortened/demangled name - csv_converter uses llvm-cxxfilt Signed-off-by: josantos --- src/omniperf | 67 +++++++++++++++--------- src/omniperf_analyze/omniperf_analyze.py | 10 ++++ src/parser.py | 18 +++++++ src/utils/csv_converter.py | 26 +++++---- 4 files changed, 86 insertions(+), 35 deletions(-) diff --git a/src/omniperf b/src/omniperf index acdfbc3bfe..4195ca64a7 100755 --- a/src/omniperf +++ b/src/omniperf @@ -446,6 +446,21 @@ def characterize_app(args, VER): # Manually join each pmc_perf*.csv output if args.use_rocscope == False: join_prof(workload_dir, args.join_type, log, args.verbose) + #demangle + for filename in os.listdir(workload_dir): + try: + # fileName = file[0 : file.find(".")] + # Only shorten KernelNames if instructed to + if args.kernelVerbose < 5: + t1 = pd.read_csv( + os.path.join(workload_dir, filename), + on_bad_lines="skip", + engine="python", + ) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) + t2.to_csv(fname, index=False) + except pd.errors.EmptyDataError: + print("Skipping empty csv " + filename) # Close log log.close() @@ -660,6 +675,7 @@ def omniperf_profile(args, VER): run_rocscope(args, fname) else: run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose) + # Update timestamps replace_timestamps(workload_dir, log) @@ -667,6 +683,22 @@ def omniperf_profile(args, VER): # Manually join each pmc_perf*.csv output if args.use_rocscope == False: join_prof(workload_dir, args.join_type, log, args.verbose) + #demangle + for filename in os.listdir(workload_dir): + if filename.endswith('.csv'): + try: + # fileName = file[0 : file.find(".")] + # Only shorten KernelNames if instructed to + if args.kernelVerbose < 5: + t1 = pd.read_csv( + os.path.join(workload_dir, filename), + on_bad_lines="skip", + engine="python", + ) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) + t2.to_csv(os.path.join(workload_dir, filename), index=False) + except pd.errors.EmptyDataError: + print("Skipping empty csv " + filename) # Generate sysinfo gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof) @@ -739,7 +771,7 @@ def main(): # PROFILE MODE ############## if args.mode == "profile": - Extractionlvl = 3 #args.extraction_level + Extractionlvl = args.kernelVerbose print("Resolving rocprof") resolve_rocprof() # Cannot access parent directories @@ -777,43 +809,28 @@ def main(): roof_setup(args, my_parser, VER) # Generate roofline roofline_only(args.path, args.device, args.sort, args.mem_level, args.kernel_names, args.verbose) - #demangle - for file in os.listdir(args.path): - if file.endswith(".csv"): - try: - fileName = file[0 : file.find(".")] - # Only shorten KernelNames if instructed to - if Extractionlvl < 5: - t1 = pd.read_csv( - os.listdir(args.path) + "/" + file, - on_bad_lines="skip", - engine="python", - ) - - t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl) - except pd.errors.EmptyDataError: - print("Skipping empty csv " + file) # Profile only else: print("\n-------------\nProfile only\n-------------\n") omniperf_profile(args, VER) + workload_dir = args.path #demangle - for file in os.listdir(args.path): - if file.endswith(".csv"): + for filename in os.listdir(workload_dir): + if filename.endswith('.csv'): try: - fileName = file[0 : file.find(".")] + # fileName = file[0 : file.find(".")] # Only shorten KernelNames if instructed to - if Extractionlvl < 5: + if args.kernelVerbose < 5: t1 = pd.read_csv( - os.listdir(args.path) + "/" + file, + os.path.join(workload_dir, filename), on_bad_lines="skip", engine="python", ) - - t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl) + t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) + t2.to_csv(os.path.join(workload_dir, filename), index=False) except pd.errors.EmptyDataError: - print("Skipping empty csv " + file) + print("Skipping empty csv " + filename) ############## # DATABASE MODE diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index 87fac064db..2a657d3d30 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -45,6 +45,8 @@ import os.path from pathlib import Path from omniperf_analyze.utils import parser, file_io from omniperf_analyze.utils.gui_components.roofline import get_roofline +from utils import csv_converter +import pandas as pd archConfigs = {} @@ -220,7 +222,15 @@ def run_cli(args, runs): # If we assume the panel layout for all archs are similar, it doesn't matter # which archConfig passed into show_all function. # After decide to how to manage kernels display patterns, we can revisit it. + cache =dict() for d in args.path: + #demangle + for filename in os.listdir(d[0]): + if filename.endswith('.csv'): + df = pd.read_csv(os.path.join(d[0],filename)) + new_df = csv_converter.kernel_name_shortener(df, cache, args.kernelVerbose) + new_df.to_csv(os.path.join(d[0],filename), index=False) + file_io.create_df_kernel_top_stats( d[0], runs[d[0]].filter_gpu_ids, diff --git a/src/parser.py b/src/parser.py index 9d6dd8f6f2..8de09542f1 100644 --- a/src/parser.py +++ b/src/parser.py @@ -204,6 +204,15 @@ def parse(my_parser): nargs=argparse.REMAINDER, help="\t\t\tProvide command for profiling after double dash.", ) + profile_group.add_argument( + "-f", + "--kernelVerbose", + required=False, + metavar="", + help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", + default=2, + type=int, + ) ## Roofline Command Line Options roofline_group.add_argument( @@ -514,3 +523,12 @@ def parse(my_parser): action="store_true", help="\t\tRandomly generate a port to launch GUI application.\n\t\tRegistered Ports range inclusive (1024-49151).", ) + analyze_group.add_argument( + "-f", + "--kernelVerbose", + required=False, + metavar="", + help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", + default=2, + type=int, + ) diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index 34f2e82614..35bc5a649f 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -25,6 +25,7 @@ import argparse import collections import os +import subprocess import sys import re import pandas as pd @@ -33,11 +34,11 @@ from pymongo import MongoClient from tqdm import tqdm import shutil + cache = dict() supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"} MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout - def kernel_name_shortener(df, cache, level): if level >= 5: return df @@ -55,6 +56,13 @@ def kernel_name_shortener(df, cache, level): if original_name in cache: continue + cmd = ["llvm-cxxfilt", original_name] + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + demangled_name, e = proc.communicate() + demangled_name = str(demangled_name, "UTF-8").strip() + # cache miss, add the shortened name to the dictionary new_name = "" matches = "" @@ -62,15 +70,14 @@ def kernel_name_shortener(df, cache, level): names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] - if names_and_args.search(original_name): - matches = names_and_args.findall(original_name) + if names_and_args.search(demangled_name): + matches = names_and_args.findall(demangled_name) else: # Works for first case '__amd_rocclr_fillBuffer.kd' - # remove .kd and then parse through original regex - first_case = re.compile(r"([^\s]+)(.kd)") - Mod_name_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]*)") - interim_name = first_case.search(original_name).group(1) - matches = Mod_name_and_args.findall(interim_name) + cache[original_name] = new_name + if new_name == None or new_name == "": + cache[original_name] = demangled_name + continue current_level = 0 for name in matches: @@ -103,13 +110,12 @@ def kernel_name_shortener(df, cache, level): cache[original_name] = new_name if new_name == None or new_name == "": - cache[original_name] = original_name + cache[original_name] = demangled_name df[columnName] = df[columnName].map(cache) return df - # Verify target directory and setup connection def parse(args, profileAndExport): host = args.host From bbb254ef43c0e0f01e4c52d786501ac45ed4111d Mon Sep 17 00:00:00 2001 From: josantos Date: Thu, 10 Aug 2023 11:16:50 -0500 Subject: [PATCH 07/12] reformatting Signed-off-by: josantos --- src/omniperf_analyze/omniperf_analyze.py | 16 +++++++++------- src/utils/csv_converter.py | 22 ++++++++++++---------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index 2a657d3d30..123bdd15a0 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -222,15 +222,17 @@ def run_cli(args, runs): # If we assume the panel layout for all archs are similar, it doesn't matter # which archConfig passed into show_all function. # After decide to how to manage kernels display patterns, we can revisit it. - cache =dict() + cache = dict() for d in args.path: - #demangle + # demangle for filename in os.listdir(d[0]): - if filename.endswith('.csv'): - df = pd.read_csv(os.path.join(d[0],filename)) - new_df = csv_converter.kernel_name_shortener(df, cache, args.kernelVerbose) - new_df.to_csv(os.path.join(d[0],filename), index=False) - + if filename.endswith(".csv"): + df = pd.read_csv(os.path.join(d[0], filename)) + new_df = csv_converter.kernel_name_shortener( + df, cache, args.kernelVerbose + ) + new_df.to_csv(os.path.join(d[0], filename), index=False) + file_io.create_df_kernel_top_stats( d[0], runs[d[0]].filter_gpu_ids, diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index 35bc5a649f..48726bf038 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -39,6 +39,7 @@ cache = dict() supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"} MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout + def kernel_name_shortener(df, cache, level): if level >= 5: return df @@ -116,6 +117,7 @@ def kernel_name_shortener(df, cache, level): return df + # Verify target directory and setup connection def parse(args, profileAndExport): host = args.host @@ -243,16 +245,16 @@ def convert_folder(connectionInfo, Extractionlvl): # os.system(cmd) # else: cmd = ( - "mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline" - ).format( - connectionInfo["username"], - connectionInfo["password"], - connectionInfo["host"], - connectionInfo["port"], - connectionInfo["db"], - connectionInfo["workload"] + "/" + file, - fileName, - ) + "mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline" + ).format( + connectionInfo["username"], + connectionInfo["password"], + connectionInfo["host"], + connectionInfo["port"], + connectionInfo["db"], + connectionInfo["workload"] + "/" + file, + fileName, + ) os.system(cmd) i += 1 except pd.errors.EmptyDataError: From 73933572afd49829c2128a49323e3db28708b2aa Mon Sep 17 00:00:00 2001 From: josantos Date: Thu, 10 Aug 2023 13:15:44 -0500 Subject: [PATCH 08/12] add llvm to github action container Signed-off-by: josantos --- .github/workflows/ubuntu-focal.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index d72aaad70f..f945b7f138 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -30,6 +30,7 @@ jobs: sudo apt-get install -y git sudo apt-get install -y python3-pip sudo apt-get install -y cmake + sudo apt-get install llvm-7 - name: Checkout uses: actions/checkout@v3 - name: Install Python prereqs From 802308cd28fdc30355242804c360d5c1f53a29d2 Mon Sep 17 00:00:00 2001 From: josantos Date: Thu, 10 Aug 2023 13:23:52 -0500 Subject: [PATCH 09/12] use llvm-cxxfilt in /opt/rocm Signed-off-by: josantos --- src/utils/csv_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index 48726bf038..bffe56cc53 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -57,7 +57,7 @@ def kernel_name_shortener(df, cache, level): if original_name in cache: continue - cmd = ["llvm-cxxfilt", original_name] + cmd = ["/opt/rocm/llvm/bin/llvm-cxxfilt", original_name] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 878948e848a9ccb74d24b37cfe43891310cd7915 Mon Sep 17 00:00:00 2001 From: josantos Date: Thu, 10 Aug 2023 13:25:11 -0500 Subject: [PATCH 10/12] removing broken install in gh actions container Signed-off-by: josantos --- .github/workflows/ubuntu-focal.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index f945b7f138..d72aaad70f 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -30,7 +30,6 @@ jobs: sudo apt-get install -y git sudo apt-get install -y python3-pip sudo apt-get install -y cmake - sudo apt-get install llvm-7 - name: Checkout uses: actions/checkout@v3 - name: Install Python prereqs From 4aa33848de02b684ade6e2bb67a83bff59f24993 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 15 Aug 2023 14:00:36 -0500 Subject: [PATCH 11/12] Fixing several bugs on original PR Signed-off-by: coleramos425 --- src/omniperf | 65 ++------ src/omniperf_analyze/omniperf_analyze.py | 13 +- src/parser.py | 17 +- src/utils/csv_converter.py | 191 ++++++++++------------- 4 files changed, 98 insertions(+), 188 deletions(-) diff --git a/src/omniperf b/src/omniperf index 4195ca64a7..4689b02ac5 100755 --- a/src/omniperf +++ b/src/omniperf @@ -53,8 +53,6 @@ from common import ( from common import getVersion -cache = dict() - ################################################ # Helper Functions ################################################ @@ -443,26 +441,12 @@ def characterize_app(args, VER): # Update timestamps replace_timestamps(workload_dir, log) - # Manually join each pmc_perf*.csv output if args.use_rocscope == False: + # Manually join each pmc_perf*.csv output join_prof(workload_dir, args.join_type, log, args.verbose) - #demangle - for filename in os.listdir(workload_dir): - try: - # fileName = file[0 : file.find(".")] - # Only shorten KernelNames if instructed to - if args.kernelVerbose < 5: - t1 = pd.read_csv( - os.path.join(workload_dir, filename), - on_bad_lines="skip", - engine="python", - ) - t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) - t2.to_csv(fname, index=False) - except pd.errors.EmptyDataError: - print("Skipping empty csv " + filename) - - # Close log + # Demangle and overwrite original KernelNames + csv_converter.kernel_name_shortener(workload_dir, args.kernelVerbose) + log.close() @@ -559,6 +543,10 @@ def omniperf_profile(args, VER): print("IP Blocks: All") else: print("IP Blocks: ", args.ipblocks) + if args.kernelVerbose > 5: + print("KernelName verbose level: DISABLED") + else: + print("KernelName verbose level: ", str(args.kernelVerbose)) # Set up directories workload_dir = args.path + "/" + args.name + "/" + args.target @@ -680,25 +668,11 @@ def omniperf_profile(args, VER): # Update timestamps replace_timestamps(workload_dir, log) - # Manually join each pmc_perf*.csv output if args.use_rocscope == False: + # Manually join each pmc_perf*.csv output join_prof(workload_dir, args.join_type, log, args.verbose) - #demangle - for filename in os.listdir(workload_dir): - if filename.endswith('.csv'): - try: - # fileName = file[0 : file.find(".")] - # Only shorten KernelNames if instructed to - if args.kernelVerbose < 5: - t1 = pd.read_csv( - os.path.join(workload_dir, filename), - on_bad_lines="skip", - engine="python", - ) - t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) - t2.to_csv(os.path.join(workload_dir, filename), index=False) - except pd.errors.EmptyDataError: - print("Skipping empty csv " + filename) + # Demangle and overwrite original KernelNames + csv_converter.kernel_name_shortener(workload_dir, args.kernelVerbose) # Generate sysinfo gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof) @@ -814,23 +788,6 @@ def main(): else: print("\n-------------\nProfile only\n-------------\n") omniperf_profile(args, VER) - workload_dir = args.path - #demangle - for filename in os.listdir(workload_dir): - if filename.endswith('.csv'): - try: - # fileName = file[0 : file.find(".")] - # Only shorten KernelNames if instructed to - if args.kernelVerbose < 5: - t1 = pd.read_csv( - os.path.join(workload_dir, filename), - on_bad_lines="skip", - engine="python", - ) - t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose) - t2.to_csv(os.path.join(workload_dir, filename), index=False) - except pd.errors.EmptyDataError: - print("Skipping empty csv " + filename) ############## # DATABASE MODE diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index 123bdd15a0..099618e8af 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -46,11 +46,9 @@ from pathlib import Path from omniperf_analyze.utils import parser, file_io from omniperf_analyze.utils.gui_components.roofline import get_roofline from utils import csv_converter -import pandas as pd archConfigs = {} - ################################################ # Helper Functions ################################################ @@ -222,16 +220,9 @@ def run_cli(args, runs): # If we assume the panel layout for all archs are similar, it doesn't matter # which archConfig passed into show_all function. # After decide to how to manage kernels display patterns, we can revisit it. - cache = dict() for d in args.path: - # demangle - for filename in os.listdir(d[0]): - if filename.endswith(".csv"): - df = pd.read_csv(os.path.join(d[0], filename)) - new_df = csv_converter.kernel_name_shortener( - df, cache, args.kernelVerbose - ) - new_df.to_csv(os.path.join(d[0], filename), index=False) + # Demangle and overwrite original KernelNames + csv_converter.kernel_name_shortener(d[0], args.kernelVerbose) file_io.create_df_kernel_top_stats( d[0], diff --git a/src/parser.py b/src/parser.py index 8de09542f1..e8eb28940b 100644 --- a/src/parser.py +++ b/src/parser.py @@ -205,11 +205,10 @@ def parse(my_parser): help="\t\t\tProvide command for profiling after double dash.", ) profile_group.add_argument( - "-f", "--kernelVerbose", required=False, metavar="", - help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", + help="\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", default=2, type=int, ) @@ -351,15 +350,6 @@ def parse(my_parser): dest="workload", help="\t\t\t\tSpecify name of workload (to remove) or path to workload (to import)", ) - connection_group.add_argument( - "-k", - "--kernelVerbose", - required=False, - metavar="", - help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", - default=2, - type=int, - ) ## Analyze Command Line Options ## ---------------------------- @@ -524,11 +514,10 @@ def parse(my_parser): help="\t\tRandomly generate a port to launch GUI application.\n\t\tRegistered Ports range inclusive (1024-49151).", ) analyze_group.add_argument( - "-f", "--kernelVerbose", required=False, metavar="", - help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)", - default=2, + help="\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 5) (DISABLE: 5)", + default=5, type=int, ) diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index bffe56cc53..428b4de91d 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -32,90 +32,106 @@ import pandas as pd import getpass from pymongo import MongoClient from tqdm import tqdm -import shutil - +import glob cache = dict() + supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"} MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout -def kernel_name_shortener(df, cache, level): - if level >= 5: - return df +def kernel_name_shortener(workload_dir, level): - columnName = "" - if "KernelName" in df: - columnName = "KernelName" - if "Name" in df: - columnName = "Name" + def shorten_file(df, level): + global cache - if columnName == "KernelName" or columnName == "Name": - # loop through all indices - for index in df.index: - original_name = df.loc[index, columnName] - if original_name in cache: - continue + columnName = "" + if "KernelName" in df: + columnName = "KernelName" + if "Name" in df: + columnName = "Name" - cmd = ["/opt/rocm/llvm/bin/llvm-cxxfilt", original_name] + if columnName == "KernelName" or columnName == "Name": + # loop through all indices + for index in df.index: + original_name = df.loc[index, columnName] + if original_name in cache: + continue - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd = ["/opt/rocm/llvm/bin/llvm-cxxfilt", original_name] - demangled_name, e = proc.communicate() - demangled_name = str(demangled_name, "UTF-8").strip() + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # cache miss, add the shortened name to the dictionary - new_name = "" - matches = "" + demangled_name, e = proc.communicate() + demangled_name = str(demangled_name, "UTF-8").strip() - names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") + # cache miss, add the shortened name to the dictionary + new_name = "" + matches = "" + + names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") + + # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] + if names_and_args.search(demangled_name): + matches = names_and_args.findall(demangled_name) + else: + # Works for first case '__amd_rocclr_fillBuffer.kd' + cache[original_name] = new_name + if new_name == None or new_name == "": + cache[original_name] = demangled_name + continue + + current_level = 0 + for name in matches: + ##can cause errors if a function name or argument is equal to 'clone' + if name[0] == "clone": + continue + if len(name) == 3: + if name[2] == "::": + continue + + if current_level < level: + new_name += name[0] + # closing '>' is to be taken account by the while loop + if name[1].count(">") == 0: + if current_level < level: + if not (current_level == level - 1 and name[1].count("<") > 0): + new_name += name[1] + current_level += name[1].count("<") + + curr_index = 0 + # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level + while name[1].count(">") > 0 and curr_index < len(name[1]): + if current_level < level: + new_name += name[1][curr_index:] + current_level -= name[1][curr_index:].count(">") + curr_index = len(name[1]) + elif name[1][curr_index] == (">"): + current_level -= 1 + curr_index += 1 - # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] - if names_and_args.search(demangled_name): - matches = names_and_args.findall(demangled_name) - else: - # Works for first case '__amd_rocclr_fillBuffer.kd' cache[original_name] = new_name if new_name == None or new_name == "": cache[original_name] = demangled_name - continue - current_level = 0 - for name in matches: - ##can cause errors if a function name or argument is equal to 'clone' - if name[0] == "clone": - continue - if len(name) == 3: - if name[2] == "::": - continue + df[columnName] = df[columnName].map(cache) - if current_level < level: - new_name += name[0] - # closing '>' is to be taken account by the while loop - if name[1].count(">") == 0: - if current_level < level: - if not (current_level == level - 1 and name[1].count("<") > 0): - new_name += name[1] - current_level += name[1].count("<") - - curr_index = 0 - # cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level - while name[1].count(">") > 0 and curr_index < len(name[1]): - if current_level < level: - new_name += name[1][curr_index:] - current_level -= name[1][curr_index:].count(">") - curr_index = len(name[1]) - elif name[1][curr_index] == (">"): - current_level -= 1 - curr_index += 1 - - cache[original_name] = new_name - if new_name == None or new_name == "": - cache[original_name] = demangled_name - - df[columnName] = df[columnName].map(cache) - - return df + return df + + # Only shorten if valid shortening level + if level < 5: + for fpath in glob.glob(workload_dir + "/*.csv"): + try: + orig_df = pd.read_csv( + fpath, + on_bad_lines="skip", + engine="python", + ) + modified_df = shorten_file(orig_df, level) + modified_df.to_csv(fpath, index=False) + except pd.errors.EmptyDataError: + print("Skipping empty csv " + str(fpath)) + print("hi") # Verify target directory and setup connection @@ -152,13 +168,6 @@ def parse(args, profileAndExport): db = "omniperf_" + str(args.team) + "_" + str(name) + "_" + soc - # if Extractionlvl >= 5: - # print("KernelName shortening disabled") - # else: - # print("KernelName shortening enabled") - - # print("Kernel name verbose level:", Extractionlvl) - if args.password == "": try: password = getpass.getpass() @@ -203,16 +212,7 @@ def convert_folder(connectionInfo, Extractionlvl): except: print("ERROR: Unable to connect to the server") sys.exit(1) - # Set up directories - # if Extractionlvl < 5: - # newfilepath = connectionInfo["workload"] - # newfilepath_h = newfilepath + "/renamedFiles/" - # if not os.path.exists(newfilepath_h): - # os.mkdir(newfilepath_h) - # newfilepath = newfilepath_h + connectionInfo["db"] + "/" - # if not os.path.exists(newfilepath): - # os.mkdir(newfilepath) - # Upload files + i = 0 file = "blank" for file in tqdm(os.listdir(connectionInfo["workload"])): @@ -220,30 +220,6 @@ def convert_folder(connectionInfo, Extractionlvl): print(connectionInfo["workload"] + "/" + file) try: fileName = file[0 : file.find(".")] - # Only shorten KernelNames if instructed to - # if Extractionlvl < 5: - # t1 = pd.read_csv( - # connectionInfo["workload"] + "/" + file, - # on_bad_lines="skip", - # engine="python", - # ) - - # t2 = kernel_name_shortener(t1, cache, level=Extractionlvl) - # df_saved_file = t2.to_csv(newfilepath + file) - - # cmd = ( - # "mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline" - # ).format( - # connectionInfo["username"], - # connectionInfo["password"], - # connectionInfo["host"], - # connectionInfo["port"], - # connectionInfo["db"], - # newfilepath + file, - # fileName, - # ) - # os.system(cmd) - # else: cmd = ( "mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline" ).format( @@ -265,8 +241,5 @@ def convert_folder(connectionInfo, Extractionlvl): value = {"name": connectionInfo["db"]} newValue = {"name": connectionInfo["db"]} mycol.replace_one(value, newValue, upsert=True) - # Remove tmp directory if we shortened KernelNames - # if Extractionlvl < 5: - # shutil.rmtree(newfilepath_h) print("{} collections added.".format(i)) print("Workload name uploaded") From dc849b264c51181547a2917abc38a24aa032754d Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 15 Aug 2023 14:09:38 -0500 Subject: [PATCH 12/12] Conform to Python formatting Signed-off-by: coleramos425 --- src/omniperf_analyze/omniperf_analyze.py | 1 + src/utils/csv_converter.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index 099618e8af..0b54a696ca 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -49,6 +49,7 @@ from utils import csv_converter archConfigs = {} + ################################################ # Helper Functions ################################################ diff --git a/src/utils/csv_converter.py b/src/utils/csv_converter.py index 428b4de91d..bd199ac3a1 100644 --- a/src/utils/csv_converter.py +++ b/src/utils/csv_converter.py @@ -41,7 +41,6 @@ MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout def kernel_name_shortener(workload_dir, level): - def shorten_file(df, level): global cache @@ -60,7 +59,9 @@ def kernel_name_shortener(workload_dir, level): cmd = ["/opt/rocm/llvm/bin/llvm-cxxfilt", original_name] - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) demangled_name, e = proc.communicate() demangled_name = str(demangled_name, "UTF-8").strip() @@ -69,7 +70,9 @@ def kernel_name_shortener(workload_dir, level): new_name = "" matches = "" - names_and_args = re.compile(r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?") + names_and_args = re.compile( + r"(?P[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?" + ) # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd] if names_and_args.search(demangled_name): @@ -95,7 +98,9 @@ def kernel_name_shortener(workload_dir, level): # closing '>' is to be taken account by the while loop if name[1].count(">") == 0: if current_level < level: - if not (current_level == level - 1 and name[1].count("<") > 0): + if not ( + current_level == level - 1 and name[1].count("<") > 0 + ): new_name += name[1] current_level += name[1].count("<") @@ -117,7 +122,7 @@ def kernel_name_shortener(workload_dir, level): df[columnName] = df[columnName].map(cache) return df - + # Only shorten if valid shortening level if level < 5: for fpath in glob.glob(workload_dir + "/*.csv"):