coleramos425
f7fe3d9efd
Signed-off-by: coleramos425 <colramos@amd.com>
[ROCm/rocprofiler-compute commit: a9d82759ca]
393 行
13 KiB
Python
可执行文件
393 行
13 KiB
Python
可执行文件
##############################################################################bl
|
||
# MIT License
|
||
#
|
||
# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All Rights Reserved.
|
||
#
|
||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||
# of this software and associated documentation files (the "Software"), to deal
|
||
# in the Software without restriction, including without limitation the rights
|
||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||
# copies of the Software, and to permit persons to whom the Software is
|
||
# furnished to do so, subject to the following conditions:
|
||
#
|
||
# The above copyright notice and this permission notice shall be included in all
|
||
# copies or substantial portions of the Software.
|
||
#
|
||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||
# SOFTWARE.
|
||
##############################################################################el
|
||
|
||
import sys, os, pathlib, shutil, subprocess, argparse, glob, re
|
||
import numpy as np
|
||
import math
|
||
import pandas as pd
|
||
|
||
prog = "omniperf"
|
||
|
||
# Per IP block max number of simulutaneous counters
|
||
# GFX IP Blocks
|
||
perfmon_config = {
|
||
"vega10": {
|
||
"SQ": 8,
|
||
"TA": 2,
|
||
"TD": 2,
|
||
"TCP": 4,
|
||
"TCC": 4,
|
||
"CPC": 2,
|
||
"CPF": 2,
|
||
"SPI": 2,
|
||
"GRBM": 2,
|
||
"GDS": 4,
|
||
"TCC_channels": 16,
|
||
},
|
||
"mi50": {
|
||
"SQ": 8,
|
||
"TA": 2,
|
||
"TD": 2,
|
||
"TCP": 4,
|
||
"TCC": 4,
|
||
"CPC": 2,
|
||
"CPF": 2,
|
||
"SPI": 2,
|
||
"GRBM": 2,
|
||
"GDS": 4,
|
||
"TCC_channels": 16,
|
||
},
|
||
"mi100": {
|
||
"SQ": 8,
|
||
"TA": 2,
|
||
"TD": 2,
|
||
"TCP": 4,
|
||
"TCC": 4,
|
||
"CPC": 2,
|
||
"CPF": 2,
|
||
"SPI": 2,
|
||
"GRBM": 2,
|
||
"GDS": 4,
|
||
"TCC_channels": 32,
|
||
},
|
||
"mi200": {
|
||
"SQ": 8,
|
||
"TA": 2,
|
||
"TD": 2,
|
||
"TCP": 4,
|
||
"TCC": 4,
|
||
"CPC": 2,
|
||
"CPF": 2,
|
||
"SPI": 2,
|
||
"GRBM": 2,
|
||
"GDS": 4,
|
||
"TCC_channels": 32,
|
||
},
|
||
}
|
||
|
||
# joins disparate runs less dumbly than rocprof
|
||
def join_prof(workload_dir, out):
|
||
files = glob.glob(workload_dir + "/" + "pmc_perf_*.csv")
|
||
df = None
|
||
|
||
for i, file in enumerate(files):
|
||
#_df = parse_rocprof_kernels(file)
|
||
_df = pd.read_csv(file)
|
||
key = _df.groupby("KernelName").cumcount()
|
||
_df['key'] = _df.KernelName + ' - ' + key.astype(str)
|
||
|
||
if df is None:
|
||
df = _df
|
||
else:
|
||
# join by unique index of kernel
|
||
df = pd.merge(df, _df, how='inner', on='key', suffixes=('', f'_{i}'))
|
||
# now, we can:
|
||
# A) throw away any of the "boring" duplicats
|
||
df = df[[k for k in df.keys() if not any(
|
||
check in k for check in [
|
||
'gpu', 'queue-id', 'queue-index', 'pid', 'tid', 'grd', 'wgr',
|
||
'lds', 'scr', 'vgpr', 'sgpr', 'fbar', 'sig', 'obj'])]]
|
||
# B) any timestamps that are _not_ the duration, which is the one we care
|
||
# about
|
||
df = df[[k for k in df.keys() if not any(
|
||
check in k for check in [
|
||
'stop', 'start', 'DispatchNs', 'CompleteNs'])]]
|
||
# C) sanity check the name and key
|
||
namekeys = [k for k in df.keys() if 'KernelName' in k]
|
||
assert len(namekeys)
|
||
for k in namekeys[1:]:
|
||
assert (df[namekeys[0]] == df[k]).all()
|
||
df = df.drop(columns=namekeys[1:])
|
||
# now take the median of the durations
|
||
dkeys = [k for k in df.keys() if 'duration' in k]
|
||
duration = df[dkeys].median(axis=1)
|
||
# compute min and max, just for sanity
|
||
min_duration = df[dkeys].min(axis=1)
|
||
max_duration = df[dkeys].max(axis=1)
|
||
std_duration = df[dkeys].std(axis=1)
|
||
mean_duration = df[dkeys].mean(axis=1)
|
||
# and replace
|
||
df = df.drop(columns=dkeys)
|
||
df['duration'] = duration
|
||
df['duration[max]'] = max_duration
|
||
df['duration[min]'] = min_duration
|
||
df['duration[std]'] = std_duration
|
||
df['duration[mean]'] = mean_duration
|
||
# finally, join the drop key
|
||
df = df.drop(columns=['key'])
|
||
# and save to file
|
||
df.to_csv(out, index=False)
|
||
# and delete old file(s)
|
||
for file in files:
|
||
os.remove(file)
|
||
|
||
def pmc_perf_split(workload_dir):
|
||
workload_perfmon_dir = workload_dir + "/perfmon"
|
||
lines = open(workload_perfmon_dir + "/pmc_perf.txt", "r").read().splitlines()
|
||
|
||
# Iterate over each line in pmc_perf.txt
|
||
mpattern = r"^pmc:(.*)"
|
||
i = 0
|
||
for line in lines:
|
||
# Verify no comments
|
||
stext = line.split("#")[0].strip()
|
||
if not stext:
|
||
continue
|
||
|
||
# all pmc counters start with "pmc:"
|
||
m = re.match(mpattern, stext)
|
||
if m is None:
|
||
continue
|
||
|
||
# Create separate file for each line
|
||
fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w")
|
||
fd.write(stext + "\n\n")
|
||
fd.write("gpu:\n")
|
||
fd.write("range:\n")
|
||
fd.write("kernel:\n")
|
||
fd.close()
|
||
|
||
i += 1
|
||
|
||
# Remove old pmc_perf.txt input from perfmon dir
|
||
os.remove(workload_perfmon_dir + "/pmc_perf.txt")
|
||
|
||
|
||
|
||
def perfmon_coalesce(pmc_files_list, workload_dir, soc):
|
||
workload_perfmon_dir = workload_dir + "/perfmon"
|
||
|
||
# match pattern for pmc counters
|
||
mpattern = r"^pmc:(.*)"
|
||
pmc_list = dict(
|
||
[
|
||
("SQ", []),
|
||
("GRBM", []),
|
||
("TCP", []),
|
||
("TA", []),
|
||
("TD", []),
|
||
("TCC", []),
|
||
("SPI", []),
|
||
("CPC", []),
|
||
("CPF", []),
|
||
("GDS", []),
|
||
("TCC2", {}), # per-channel TCC perfmon
|
||
]
|
||
)
|
||
for ch in range(perfmon_config[soc]["TCC_channels"]):
|
||
pmc_list["TCC2"][str(ch)] = []
|
||
|
||
# Extract all PMC counters and store in separate buckets
|
||
for fname in pmc_files_list:
|
||
lines = open(fname, "r").read().splitlines()
|
||
|
||
for line in lines:
|
||
# Strip all comements, skip empty lines
|
||
stext = line.split("#")[0].strip()
|
||
if not stext:
|
||
continue
|
||
|
||
# all pmc counters start with "pmc:"
|
||
m = re.match(mpattern, stext)
|
||
if m is None:
|
||
continue
|
||
|
||
# we have found all the counters, store them in buckets
|
||
counters = m.group(1).split()
|
||
if "SQ_ACCUM_PREV_HIRES" in counters:
|
||
# save all level counters separately
|
||
|
||
nindex = counters.index("SQ_ACCUM_PREV_HIRES")
|
||
level_counter = counters[nindex - 1]
|
||
|
||
# Save to level counter file, file name = level counter name
|
||
fd = open(workload_perfmon_dir + "/" + level_counter + ".txt", "w")
|
||
fd.write(stext + "\n\n")
|
||
fd.write("gpu:\n")
|
||
fd.write("range:\n")
|
||
fd.write("kernel:\n")
|
||
fd.close()
|
||
|
||
continue
|
||
|
||
# save normal pmc counters in matching buckets
|
||
for counter in counters:
|
||
IP_block = counter.split(sep="_")[0].upper()
|
||
# SQC and SQ belong to the IP block, coalesce them
|
||
if IP_block == "SQC":
|
||
IP_block = "SQ"
|
||
|
||
if IP_block != "TCC":
|
||
# Insert unique pmc counters into its bucket
|
||
if counter not in pmc_list[IP_block]:
|
||
pmc_list[IP_block].append(counter)
|
||
|
||
else:
|
||
# TCC counters processing
|
||
m = re.match(r"[\s\S]+\[(\d+)\]", counter)
|
||
if m is None:
|
||
# Aggregated TCC counters
|
||
if counter not in pmc_list[IP_block]:
|
||
pmc_list[IP_block].append(counter)
|
||
|
||
else:
|
||
# TCC channel ID
|
||
ch = m.group(1)
|
||
|
||
# fake IP block for per channel TCC
|
||
if str(ch) in pmc_list["TCC2"]:
|
||
# append unique counter into the channel
|
||
if counter not in pmc_list["TCC2"][str(ch)]:
|
||
pmc_list["TCC2"][str(ch)].append(counter)
|
||
else:
|
||
# initial counter in this channel
|
||
pmc_list["TCC2"][str(ch)] = [counter]
|
||
|
||
# sort the per channel counter, so that same counter in all channels can be aligned
|
||
for ch in range(perfmon_config[soc]["TCC_channels"]):
|
||
pmc_list["TCC2"][str(ch)].sort()
|
||
|
||
return pmc_list
|
||
|
||
|
||
def perfmon_emit(pmc_list, workload_dir, soc):
|
||
workload_perfmon_dir = workload_dir + "/perfmon"
|
||
|
||
# Calculate the minimum number of iteration to save the pmc counters
|
||
# non-TCC counters
|
||
pmc_cnt = [
|
||
len(pmc_list[key]) / perfmon_config[soc][key]
|
||
for key in pmc_list
|
||
if key not in ["TCC", "TCC2"]
|
||
]
|
||
|
||
# TCC counters
|
||
tcc_channels = perfmon_config[soc]["TCC_channels"]
|
||
|
||
tcc_cnt = len(pmc_list["TCC"]) / perfmon_config[soc]["TCC"]
|
||
tcc2_cnt = (
|
||
np.array([len(pmc_list["TCC2"][str(ch)]) for ch in range(tcc_channels)])
|
||
/ perfmon_config[soc]["TCC"]
|
||
)
|
||
|
||
# Total number iterations to write pmc: counters line
|
||
niter = max(math.ceil(max(pmc_cnt)), math.ceil(tcc_cnt) + math.ceil(max(tcc2_cnt)))
|
||
|
||
# Emit PMC counters into pmc config file
|
||
fd = open(workload_perfmon_dir + "/pmc_perf.txt", "w")
|
||
|
||
tcc2_index = 0
|
||
for iter in range(niter):
|
||
# Prefix
|
||
line = "pmc: "
|
||
|
||
# Add all non-TCC counters
|
||
for key in pmc_list:
|
||
if key not in ["TCC", "TCC2"]:
|
||
N = perfmon_config[soc][key]
|
||
ip_counters = pmc_list[key][iter * N : iter * N + N]
|
||
if ip_counters:
|
||
line = line + " " + " ".join(ip_counters)
|
||
|
||
# Add TCC counters
|
||
N = perfmon_config[soc]["TCC"]
|
||
tcc_counters = pmc_list["TCC"][iter * N : iter * N + N]
|
||
|
||
if not tcc_counters:
|
||
# TCC per-channel counters
|
||
for ch in range(perfmon_config[soc]["TCC_channels"]):
|
||
tcc_counters += pmc_list["TCC2"][str(ch)][
|
||
tcc2_index * N : tcc2_index * N + N
|
||
]
|
||
|
||
tcc2_index += 1
|
||
|
||
# TCC aggregated counters
|
||
line = line + " " + " ".join(tcc_counters)
|
||
fd.write(line + "\n")
|
||
|
||
fd.write("\ngpu:\n")
|
||
fd.write("range:\n")
|
||
fd.write("kernel:\n")
|
||
fd.close()
|
||
|
||
|
||
def perfmon_filter(workload_dir, perfmon_dir, args):
|
||
workload_perfmon_dir = workload_dir + "/perfmon"
|
||
soc = args.target
|
||
|
||
# Initialize directories
|
||
# TODO: Modify this so that data is appended to previous?
|
||
if not os.path.isdir(workload_dir):
|
||
os.makedirs(workload_dir)
|
||
else:
|
||
shutil.rmtree(workload_dir)
|
||
|
||
os.makedirs(workload_perfmon_dir)
|
||
|
||
ref_pmc_files_list = glob.glob(perfmon_dir + "/" + "pmc_*perf*.txt")
|
||
ref_pmc_files_list += glob.glob(perfmon_dir + "/" + soc + "/pmc_*_perf*.txt")
|
||
|
||
# Perfmon list filtering
|
||
if args.ipblocks != None:
|
||
for i in range(len(args.ipblocks)):
|
||
args.ipblocks[i] = args.ipblocks[i].lower()
|
||
mpattern = "pmc_([a-zA-Z0-9_]+)_perf*"
|
||
|
||
pmc_files_list = []
|
||
for fname in ref_pmc_files_list:
|
||
fbase = os.path.splitext(os.path.basename(fname))[0]
|
||
ip = re.match(mpattern, fbase).group(1)
|
||
if ip in args.ipblocks:
|
||
pmc_files_list.append(fname)
|
||
print("fname: " + fbase + ": Added")
|
||
else:
|
||
print("fname: " + fbase + ": Skipped")
|
||
|
||
else:
|
||
# default: take all perfmons
|
||
pmc_files_list = ref_pmc_files_list
|
||
|
||
# Coalesce and writeback workload specific perfmon
|
||
pmc_list = perfmon_coalesce(pmc_files_list, workload_dir, soc)
|
||
perfmon_emit(pmc_list, workload_dir, soc)
|
||
|
||
|
||
def pmc_filter(workload_dir, perfmon_dir, soc):
|
||
workload_perfmon_dir = workload_dir + "/perfmon"
|
||
|
||
if not os.path.isdir(workload_perfmon_dir):
|
||
os.makedirs(workload_perfmon_dir)
|
||
else:
|
||
shutil.rmtree(workload_perfmon_dir)
|
||
|
||
ref_pmc_files_list = glob.glob(perfmon_dir + "/roofline/" + "pmc_roof_perf.txt")
|
||
# ref_pmc_files_list += glob.glob(perfmon_dir + "/" + soc + "/pmc_*_perf*.txt")
|
||
|
||
pmc_files_list = ref_pmc_files_list
|
||
|
||
# Coalesce and writeback workload specific perfmon
|
||
pmc_list = perfmon_coalesce(pmc_files_list, workload_dir, soc)
|
||
perfmon_emit(pmc_list, workload_dir, soc)
|