Kernel no overwrite (#303)
* Removing kernel name shortener from profile
- Kernel name shortener now in analyze
- db import uses pymongo to import
Signed-off-by: Jose Santos <josantos@amd.com>
* collections now added using pymongo
Signed-off-by: JoseSantosAMD <Jose.Santos@amd.com>
* kernel_name_shortener shortens dataframes and not a workload
- argparser: --kernel-verbose to connection group
- analysis_cli, profiler_rocprof_v2: remove kernel_name_shortener, so that it doesnt overwrite workload
- db_import: demangle kernel_name dfs
Signed-off-by: JoseSantosAMD <Jose.Santos@amd.com>
* fix db name in pymongo command
Signed-off-by: JoseSantosAMD <Jose.Santos@amd.com>
* reformat
Signed-off-by: JoseSantosAMD <Jose.Santos@amd.com>
* fix typo
Signed-off-by: JoseSantosAMD <Jose.Santos@amd.com>
* remove checker for verbose in profiler_base
Signed-off-by: Jose Santos <josantos@amd.com>
* kernel_name_shortener now called in create_df_kerenel_top_stats and
create_df_pmc
Signed-off-by: Jose Santos <josantos@amd.com>
* add kernel_verbose to analysis_webui, remove stubbed values
Signed-off-by: JoseSantosAMD <Jose.Santos@amd.com>
* Update src/utils/kernel_name_shortener.py
keep console_debug
Co-authored-by: Cole Ramos <colramos@amd.com>
Signed-off-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com>
* Update src/utils/kernel_name_shortener.py
Co-authored-by: Cole Ramos <colramos@amd.com>
Signed-off-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com>
---------
Signed-off-by: Jose Santos <josantos@amd.com>
Signed-off-by: JoseSantosAMD <Jose.Santos@amd.com>
Signed-off-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com>
Co-authored-by: Cole Ramos <colramos@amd.com>
[ROCm/rocprofiler-compute commit: 4424f320ad]
此提交包含在：
JoseSantosAMD
GitHub
@@ -228,14 +228,6 @@ Examples:
|
||||
nargs=argparse.REMAINDER,
|
||||
help="\t\t\tProvide command for profiling after double dash.",
|
||||
)
|
||||
profile_group.add_argument(
|
||||
"--kernel-verbose",
|
||||
required=False,
|
||||
metavar="",
|
||||
help="\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)",
|
||||
default=2,
|
||||
type=int,
|
||||
)
|
||||
|
||||
## Roofline Command Line Options
|
||||
roofline_group.add_argument(
|
||||
@@ -383,6 +375,14 @@ Examples:
|
||||
dest="workload",
|
||||
help="\t\t\t\tSpecify name of workload (to remove) or path to workload (to import)",
|
||||
)
|
||||
connection_group.add_argument(
|
||||
"--kernel-verbose",
|
||||
required=False,
|
||||
metavar="",
|
||||
help="\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 5) (DISABLE: 5)",
|
||||
default=5,
|
||||
type=int,
|
||||
)
|
||||
|
||||
## Analyze Command Line Options
|
||||
## ----------------------------
|
||||
|
||||
@@ -39,20 +39,23 @@ class cli_analysis(OmniAnalyze_Base):
|
||||
if self.get_args().random_port:
|
||||
console_error("--gui flag is required to enable --random-port")
|
||||
for d in self.get_args().path:
|
||||
# demangle and overwrite original 'Kernel_Name'
|
||||
kernel_name_shortener(d[0], self.get_args().kernel_verbose)
|
||||
|
||||
file_io.create_df_kernel_top_stats(
|
||||
raw_data_dir=d[0],
|
||||
filter_gpu_ids=self._runs[d[0]].filter_gpu_ids,
|
||||
filter_dispatch_ids=self._runs[d[0]].filter_dispatch_ids,
|
||||
time_unit=self.get_args().time_unit,
|
||||
max_stat_num=self.get_args().max_stat_num,
|
||||
kernel_verbose=self.get_args().kernel_verbose,
|
||||
)
|
||||
# create 'mega dataframe'
|
||||
self._runs[d[0]].raw_pmc = file_io.create_df_pmc(
|
||||
d[0], self.get_args().verbose
|
||||
d[0], self.get_args().kernel_verbose, self.get_args().verbose
|
||||
)
|
||||
# demangle and overwrite original 'Kernel_Name'
|
||||
kernel_name_shortener(
|
||||
self._runs[d[0]].raw_pmc, self.get_args().kernel_verbose
|
||||
)
|
||||
|
||||
# create the loaded table
|
||||
parser.load_table_data(
|
||||
workload=self._runs[d[0]],
|
||||
|
||||
@@ -105,7 +105,7 @@ class webui_analysis(OmniAnalyze_Base):
|
||||
panel_configs = copy.deepcopy(arch_configs.panel_configs)
|
||||
# Generate original raw df
|
||||
base_data[base_run].raw_pmc = file_io.create_df_pmc(
|
||||
self.dest_dir, self.get_args().verbose
|
||||
self.dest_dir, self.get_args().kernel_verbose, self.get_args().verbose
|
||||
)
|
||||
console_debug("analysis", "gui dispatch filter is %s" % disp_filt)
|
||||
console_debug("analysis", "gui kernel filter is %s" % kernel_filter)
|
||||
@@ -123,6 +123,7 @@ class webui_analysis(OmniAnalyze_Base):
|
||||
filter_dispatch_ids=base_data[base_run].filter_dispatch_ids,
|
||||
time_unit=self.get_args().time_unit,
|
||||
max_stat_num=base_data[base_run].filter_top_n,
|
||||
kernel_verbose=self.get_args().kernel_verbose,
|
||||
)
|
||||
# Only display basic metrics if no filters are applied
|
||||
if not (disp_filt or kernel_filter or gcd_filter):
|
||||
@@ -275,10 +276,11 @@ class webui_analysis(OmniAnalyze_Base):
|
||||
filter_dispatch_ids=self._runs[self.dest_dir].filter_dispatch_ids,
|
||||
time_unit=args.time_unit,
|
||||
max_stat_num=args.max_stat_num,
|
||||
kernel_verbose=self.get_args().kernel_verbose,
|
||||
)
|
||||
# create 'mega dataframe'
|
||||
self._runs[self.dest_dir].raw_pmc = file_io.create_df_pmc(
|
||||
self.dest_dir, args.verbose
|
||||
self.dest_dir, self.get_args().kernel_verbose, args.verbose
|
||||
)
|
||||
# create the loaded kernel stats
|
||||
parser.load_kernel_top(self._runs[self.dest_dir], self.dest_dir)
|
||||
|
||||
@@ -330,10 +330,6 @@ class OmniProfiler_Base:
|
||||
console_log("IP Blocks: All")
|
||||
else:
|
||||
console_log("IP Blocks: " + str(self.__args.ipblocks))
|
||||
if self.__args.kernel_verbose > 5:
|
||||
console_log("KernelName verbose: DISABLED")
|
||||
else:
|
||||
console_log("KernelName verbose: " + str(self.__args.kernel_verbose))
|
||||
|
||||
print_status("Collecting Performance Counters")
|
||||
|
||||
|
||||
@@ -86,5 +86,3 @@ class rocprof_v1_profiler(OmniProfiler_Base):
|
||||
self.join_prof()
|
||||
# Replace timestamp data to solve a known rocprof bug
|
||||
replace_timestamps(self.get_args().path)
|
||||
# Demangle and overwrite original KernelNames
|
||||
kernel_name_shortener(self.get_args().path, self.get_args().kernel_verbose)
|
||||
|
||||
@@ -83,5 +83,3 @@ class rocprof_v2_profiler(OmniProfiler_Base):
|
||||
if self.ready_to_profile:
|
||||
# Pass headers to join on
|
||||
self.join_prof()
|
||||
# Demangle and overwrite original KernelNames
|
||||
kernel_name_shortener(self.get_args().path, self.get_args().kernel_verbose)
|
||||
|
||||
@@ -33,6 +33,7 @@ from utils.utils import (
|
||||
)
|
||||
from pymongo import MongoClient
|
||||
from tqdm import tqdm
|
||||
from utils.kernel_name_shortener import kernel_name_shortener
|
||||
|
||||
import os
|
||||
import getpass
|
||||
@@ -61,14 +62,13 @@ class DatabaseConnector:
|
||||
|
||||
@demarcate
|
||||
def prep_import(self):
|
||||
|
||||
# Extract SoC and workload name from sysinfo.csv
|
||||
sys_info = os.path.join(self.connection_info["workload"], "sysinfo.csv")
|
||||
if os.path.isfile(sys_info):
|
||||
sys_info = pd.read_csv(sys_info)
|
||||
try:
|
||||
soc = sys_info["gpu_model"][0]
|
||||
name = sys_info["workload_name"][0]
|
||||
soc = sys_info["gpu_model"][0].strip()
|
||||
name = sys_info["workload_name"][0].strip()
|
||||
except KeyError as e:
|
||||
console_error(
|
||||
f"Outdated workload. Cannot find {e} field. Please reprofile to update."
|
||||
@@ -95,18 +95,25 @@ class DatabaseConnector:
|
||||
)
|
||||
try:
|
||||
fileName = file[0 : file.find(".")]
|
||||
cmd = (
|
||||
"mongoimport --quiet --uri mongodb://{}:{}@{}:{}/{}?authSource=admin --file {} -c {} --drop --type csv --headerline"
|
||||
).format(
|
||||
self.connection_info["username"],
|
||||
self.connection_info["password"],
|
||||
self.connection_info["host"],
|
||||
self.connection_info["port"],
|
||||
self.connection_info["db"],
|
||||
self.connection_info["workload"] + "/" + file,
|
||||
fileName,
|
||||
data = pd.read_csv(self.connection_info["workload"] + "/" + file)
|
||||
|
||||
# Demangle original KernelNames
|
||||
kernel_name_shortener(data, self.args.kernel_verbose)
|
||||
data.reset_index(inplace=True)
|
||||
data_dict = data.to_dict("records")
|
||||
|
||||
client = MongoClient(
|
||||
"mongodb://{}:{}@{}:{}/{}?authSource=admin".format(
|
||||
self.connection_info["username"],
|
||||
self.connection_info["password"],
|
||||
self.connection_info["host"],
|
||||
self.connection_info["port"],
|
||||
self.connection_info["db"],
|
||||
)
|
||||
)
|
||||
os.system(cmd)
|
||||
db = client[self.connection_info["db"]]
|
||||
collection = db[fileName]
|
||||
collection.insert_many(data_dict)
|
||||
i += 1
|
||||
except pd.errors.EmptyDataError:
|
||||
console_warning("database", "Skipping empty file: %s" % file)
|
||||
|
||||
@@ -34,6 +34,7 @@ from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
from utils import schema
|
||||
from utils.utils import console_debug, console_error
|
||||
from utils.kernel_name_shortener import kernel_name_shortener
|
||||
import config
|
||||
|
||||
# TODO: use pandas chunksize or dask to read really large csv file
|
||||
@@ -89,6 +90,7 @@ def create_df_kernel_top_stats(
|
||||
filter_dispatch_ids,
|
||||
time_unit,
|
||||
max_stat_num,
|
||||
kernel_verbose,
|
||||
sortby="sum",
|
||||
):
|
||||
"""
|
||||
@@ -97,6 +99,8 @@ def create_df_kernel_top_stats(
|
||||
# NB:
|
||||
# We even don't have to create pmc_kernel_top.csv explictly
|
||||
df = pd.read_csv(os.path.join(raw_data_dir, schema.pmc_perf_file_prefix + ".csv"))
|
||||
# Demangle original KernelNames
|
||||
kernel_name_shortener(df, kernel_verbose)
|
||||
|
||||
# The logic below for filters are the same as in parser.apply_filters(),
|
||||
# which can be merged together if need it.
|
||||
@@ -154,7 +158,7 @@ def create_df_kernel_top_stats(
|
||||
grouped.to_csv(os.path.join(raw_data_dir, "pmc_kernel_top.csv"), index=False)
|
||||
|
||||
|
||||
def create_df_pmc(raw_data_dir, verbose):
|
||||
def create_df_pmc(raw_data_dir, kernel_verbose, verbose):
|
||||
"""
|
||||
Load all raw pmc counters and join into one df.
|
||||
"""
|
||||
@@ -170,6 +174,8 @@ def create_df_pmc(raw_data_dir, verbose):
|
||||
f == schema.pmc_perf_file_prefix + ".csv"
|
||||
):
|
||||
tmp_df = pd.read_csv(os.path.join(root, f))
|
||||
# Demangle original KernelNames
|
||||
kernel_name_shortener(tmp_df, kernel_verbose)
|
||||
dfs.append(tmp_df)
|
||||
coll_levels.append(f[:-4])
|
||||
final_df = pd.concat(dfs, keys=coll_levels, axis=1, copy=False)
|
||||
|
||||
@@ -23,18 +23,18 @@
|
||||
##############################################################################el
|
||||
|
||||
import os
|
||||
import glob
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
import pandas as pd
|
||||
|
||||
from utils.utils import console_error, console_debug, console_log
|
||||
from utils.utils import console_error, console_log, console_debug
|
||||
|
||||
cache = dict()
|
||||
|
||||
|
||||
# Note: shortener is now dependent on a rocprof install with llvm
|
||||
def kernel_name_shortener(workload_dir, level):
|
||||
def kernel_name_shortener(df, level):
|
||||
def shorten_file(df, level):
|
||||
global cache
|
||||
|
||||
@@ -125,18 +125,9 @@ def kernel_name_shortener(workload_dir, level):
|
||||
"Could not resolve c++filt in expected directory: %s" % cpp_filt
|
||||
)
|
||||
|
||||
for fpath in glob.glob(workload_dir + "/[SQpmc]*.csv"):
|
||||
try:
|
||||
orig_df = pd.read_csv(
|
||||
fpath,
|
||||
on_bad_lines="skip",
|
||||
engine="python",
|
||||
)
|
||||
modified_df = shorten_file(orig_df, level)
|
||||
modified_df.to_csv(fpath, index=False)
|
||||
except pd.errors.EmptyDataError:
|
||||
console_debug(
|
||||
"profiling", "Skipping shortening on empty csv: %s" % str(fpath)
|
||||
)
|
||||
|
||||
console_log("profiling", "Kernel_Name shortening complete.")
|
||||
try:
|
||||
modified_df = shorten_file(df, level)
|
||||
console_log("profiling", "Kernel_Name shortening complete.")
|
||||
return modified_df
|
||||
except pd.errors.EmptyDataError:
|
||||
console_debug("profiling", "Skipping shortening on empty csv")
|
||||
|
||||
新增問題並參考
封鎖使用者