AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #157 from JoseSantosAMD/kernel_name_mappings

Browse Source 
Kernel name demangle
This commit is contained in:
Cole Ramos
2023-08-15 13:07:22 -07:00
committed by GitHub
parent c809346b1b dc849b264c
commit b79b4d3512
5 changed files with 153 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/omniperf
+13 -5
View File
@@ -260,7 +260,6 @@ def mongo_import(args, profileAndImport):
csv_converter.convert_folder(connectionInfo, Extractionlvl)
print("-- Complete! --")
################################################
# Roofline Helpers
################################################
@@ -442,11 +441,12 @@ def characterize_app(args, VER):
# Update timestamps
replace_timestamps(workload_dir, log)
# Manually join each pmc_perf*.csv output
if args.use_rocscope == False:
# Manually join each pmc_perf*.csv output
join_prof(workload_dir, args.join_type, log, args.verbose)
# Close log
# Demangle and overwrite original KernelNames
csv_converter.kernel_name_shortener(workload_dir, args.kernelVerbose)
log.close()
@@ -543,6 +543,10 @@ def omniperf_profile(args, VER):
print("IP Blocks: All")
else:
print("IP Blocks: ", args.ipblocks)
if args.kernelVerbose > 5:
print("KernelName verbose level: DISABLED")
else:
print("KernelName verbose level: ", str(args.kernelVerbose))
# Set up directories
workload_dir = args.path + "/" + args.name + "/" + args.target
@@ -659,13 +663,16 @@ def omniperf_profile(args, VER):
run_rocscope(args, fname)
else:
run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose)
# Update timestamps
replace_timestamps(workload_dir, log)
# Manually join each pmc_perf*.csv output
if args.use_rocscope == False:
# Manually join each pmc_perf*.csv output
join_prof(workload_dir, args.join_type, log, args.verbose)
# Demangle and overwrite original KernelNames
csv_converter.kernel_name_shortener(workload_dir, args.kernelVerbose)
# Generate sysinfo
gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof)
@@ -738,6 +745,7 @@ def main():
# PROFILE MODE
##############
if args.mode == "profile":
Extractionlvl = args.kernelVerbose
print("Resolving rocprof")
resolve_rocprof()
# Cannot access parent directories
src/omniperf_analyze/assets/layout.css
+9
View File
@@ -213,6 +213,15 @@ ul#nav li {
font-size: 14px;
text-align: left;
}
.dash-dropdown input{
color: black;
}
.dash-dropdown .Select-placeholder{
color: black;
}
.VirtualizedSelectOption {
overflow: hidden;
}
src/omniperf_analyze/omniperf_analyze.py
+4
View File
@@ -45,6 +45,7 @@ import os.path
from pathlib import Path
from omniperf_analyze.utils import parser, file_io
from omniperf_analyze.utils.gui_components.roofline import get_roofline
from utils import csv_converter
archConfigs = {}
@@ -221,6 +222,9 @@ def run_cli(args, runs):
# which archConfig passed into show_all function.
# After decide to how to manage kernels display patterns, we can revisit it.
for d in args.path:
# Demangle and overwrite original KernelNames
csv_converter.kernel_name_shortener(d[0], args.kernelVerbose)
file_io.create_df_kernel_top_stats(
d[0],
runs[d[0]].filter_gpu_ids,
src/parser.py
+16 -9
View File
@@ -204,6 +204,14 @@ def parse(my_parser):
nargs=argparse.REMAINDER,
help="\t\t\tProvide command for profiling after double dash.",
)
profile_group.add_argument(
"--kernelVerbose",
required=False,
metavar="",
help="\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)",
default=2,
type=int,
)
## Roofline Command Line Options
roofline_group.add_argument(
@@ -342,15 +350,6 @@ def parse(my_parser):
dest="workload",
help="\t\t\t\tSpecify name of workload (to remove) or path to workload (to import)",
)
connection_group.add_argument(
"-k",
"--kernelVerbose",
required=False,
metavar="",
help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)",
default=2,
type=int,
)
## Analyze Command Line Options
## ----------------------------
@@ -514,3 +513,11 @@ def parse(my_parser):
action="store_true",
help="\t\tRandomly generate a port to launch GUI application.\n\t\tRegistered Ports range inclusive (1024-49151).",
)
analyze_group.add_argument(
"--kernelVerbose",
required=False,
metavar="",
help="\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 5) (DISABLE: 5)",
default=5,
type=int,
)
src/utils/csv_converter.py
+111 -125
View File
@@ -25,89 +25,118 @@
import argparse
import collections
import os
import subprocess
import sys
import re
import pandas as pd
import getpass
from pymongo import MongoClient
from tqdm import tqdm
import shutil
import glob
cache = dict()
supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"}
MAX_SERVER_SEL_DELAY = 5000 # 5 sec connection timeout
def kernel_name_shortener(df, cache, level):
if level >= 5:
def kernel_name_shortener(workload_dir, level):
def shorten_file(df, level):
global cache
columnName = ""
if "KernelName" in df:
columnName = "KernelName"
if "Name" in df:
columnName = "Name"
if columnName == "KernelName" or columnName == "Name":
# loop through all indices
for index in df.index:
original_name = df.loc[index, columnName]
if original_name in cache:
continue
cmd = ["/opt/rocm/llvm/bin/llvm-cxxfilt", original_name]
proc = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
demangled_name, e = proc.communicate()
demangled_name = str(demangled_name, "UTF-8").strip()
# cache miss, add the shortened name to the dictionary
new_name = ""
matches = ""
names_and_args = re.compile(
r"(?P<name>[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?"
)
# works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd]
if names_and_args.search(demangled_name):
matches = names_and_args.findall(demangled_name)
else:
# Works for first case '__amd_rocclr_fillBuffer.kd'
cache[original_name] = new_name
if new_name == None or new_name == "":
cache[original_name] = demangled_name
continue
current_level = 0
for name in matches:
##can cause errors if a function name or argument is equal to 'clone'
if name[0] == "clone":
continue
if len(name) == 3:
if name[2] == "::":
continue
if current_level < level:
new_name += name[0]
# closing '>' is to be taken account by the while loop
if name[1].count(">") == 0:
if current_level < level:
if not (
current_level == level - 1 and name[1].count("<") > 0
):
new_name += name[1]
current_level += name[1].count("<")
curr_index = 0
# cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level
while name[1].count(">") > 0 and curr_index < len(name[1]):
if current_level < level:
new_name += name[1][curr_index:]
current_level -= name[1][curr_index:].count(">")
curr_index = len(name[1])
elif name[1][curr_index] == (">"):
current_level -= 1
curr_index += 1
cache[original_name] = new_name
if new_name == None or new_name == "":
cache[original_name] = demangled_name
df[columnName] = df[columnName].map(cache)
return df
columnName = ""
if "KernelName" in df:
columnName = "KernelName"
if "Name" in df:
columnName = "Name"
if columnName == "KernelName" or columnName == "Name":
# loop through all indices
for index in df.index:
original_name = df.loc[index, columnName]
if original_name in cache:
continue
# cache miss, add the shortened name to the dictionary
new_name = ""
matches = ""
names_and_args = re.compile(r"(?P<name>[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?")
# works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd]
if names_and_args.search(original_name):
matches = names_and_args.findall(original_name)
else:
# Works for first case '__amd_rocclr_fillBuffer.kd'
# remove .kd and then parse through original regex
first_case = re.compile(r"([^\s]+)(.kd)")
Mod_name_and_args = re.compile(r"(?P<name>[( )A-Za-z0-9_]+)([ ,*<>()]*)")
interim_name = first_case.search(original_name).group(1)
matches = Mod_name_and_args.findall(interim_name)
current_level = 0
for name in matches:
##can cause errors if a function name or argument is equal to 'clone'
if name[0] == "clone":
continue
if len(name) == 3:
if name[2] == "::":
continue
if current_level < level:
new_name += name[0]
# closing '>' is to be taken account by the while loop
if name[1].count(">") == 0:
if current_level < level:
if not (current_level == level - 1 and name[1].count("<") > 0):
new_name += name[1]
current_level += name[1].count("<")
curr_index = 0
# cases include '>' '> >, ' have to go in depth here to not lose account of commas and current level
while name[1].count(">") > 0 and curr_index < len(name[1]):
if current_level < level:
new_name += name[1][curr_index:]
current_level -= name[1][curr_index:].count(">")
curr_index = len(name[1])
elif name[1][curr_index] == (">"):
current_level -= 1
curr_index += 1
cache[original_name] = new_name
if new_name == None or new_name == "":
cache[original_name] = original_name
df[columnName] = df[columnName].map(cache)
return df
# Only shorten if valid shortening level
if level < 5:
for fpath in glob.glob(workload_dir + "/*.csv"):
try:
orig_df = pd.read_csv(
fpath,
on_bad_lines="skip",
engine="python",
)
modified_df = shorten_file(orig_df, level)
modified_df.to_csv(fpath, index=False)
except pd.errors.EmptyDataError:
print("Skipping empty csv " + str(fpath))
print("hi")
# Verify target directory and setup connection
@@ -144,13 +173,6 @@ def parse(args, profileAndExport):
db = "omniperf_" + str(args.team) + "_" + str(name) + "_" + soc
if Extractionlvl >= 5:
print("KernelName shortening disabled")
else:
print("KernelName shortening enabled")
print("Kernel name verbose level:", Extractionlvl)
if args.password == "":
try:
password = getpass.getpass()
@@ -195,16 +217,7 @@ def convert_folder(connectionInfo, Extractionlvl):
except:
print("ERROR: Unable to connect to the server")
sys.exit(1)
# Set up directories
if Extractionlvl < 5:
newfilepath = connectionInfo["workload"]
newfilepath_h = newfilepath + "/renamedFiles/"
if not os.path.exists(newfilepath_h):
os.mkdir(newfilepath_h)
newfilepath = newfilepath_h + connectionInfo["db"] + "/"
if not os.path.exists(newfilepath):
os.mkdir(newfilepath)
# Upload files
i = 0
file = "blank"
for file in tqdm(os.listdir(connectionInfo["workload"])):
@@ -212,42 +225,18 @@ def convert_folder(connectionInfo, Extractionlvl):
print(connectionInfo["workload"] + "/" + file)
try:
fileName = file[0 : file.find(".")]
# Only shorten KernelNames if instructed to
if Extractionlvl < 5:
t1 = pd.read_csv(
connectionInfo["workload"] + "/" + file,
on_bad_lines="skip",
engine="python",
)
t2 = kernel_name_shortener(t1, cache, level=Extractionlvl)
df_saved_file = t2.to_csv(newfilepath + file)
cmd = (
"mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline"
).format(
connectionInfo["username"],
connectionInfo["password"],
connectionInfo["host"],
connectionInfo["port"],
connectionInfo["db"],
newfilepath + file,
fileName,
)
os.system(cmd)
else:
cmd = (
"mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline"
).format(
connectionInfo["username"],
connectionInfo["password"],
connectionInfo["host"],
connectionInfo["port"],
connectionInfo["db"],
connectionInfo["workload"] + "/" + file,
fileName,
)
os.system(cmd)
cmd = (
"mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline"
).format(
connectionInfo["username"],
connectionInfo["password"],
connectionInfo["host"],
connectionInfo["port"],
connectionInfo["db"],
connectionInfo["workload"] + "/" + file,
fileName,
)
os.system(cmd)
i += 1
except pd.errors.EmptyDataError:
print("Skipping empty csv " + file)
@@ -257,8 +246,5 @@ def convert_folder(connectionInfo, Extractionlvl):
value = {"name": connectionInfo["db"]}
newValue = {"name": connectionInfo["db"]}
mycol.replace_one(value, newValue, upsert=True)
# Remove tmp directory if we shortened KernelNames
if Extractionlvl < 5:
shutil.rmtree(newfilepath_h)
print("{} collections added.".format(i))
print("Workload name uploaded")