From ab3c584302289b5e4b9f7499fe8fd0b574aaed2b Mon Sep 17 00:00:00 2001 From: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:50:22 -0600 Subject: [PATCH] Pytest (#212) * Initial commit for pytest Signed-off-by: JoseSantosAMD * Check if csvs populated Signed-off-by: JoseSantosAMD * pytest now checks soc for soc specific testing Signed-off-by: JoseSantosAMD * Roof only csvs updated - Removes dir if workload dir already present Signed-off-by: JoseSantosAMD * Tests create difference logs calculated from a baseline Signed-off-by: JoseSantosAMD * pytest scripts updated to revamp test cases Signed-off-by: JoseSantosAMD * Fix dispatch notation Signed-off-by: JoseSantosAMD * Introduce a cmake option to enable python code coverage. It now defaults to being disabled and can be enabled via a -DENABLE_COVERAGE=ON option (#194). Introduce CI on mi100 that leverages code coverage and publishes results along with a testing report. Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * update branch for CI -> 2.x Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * add profile execution test in cmake and enable in CI Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * run all ci tests Signed-off-by: Karl W. Schulz Signed-off-by: JoseSantosAMD * Initial overhaul of Profile mode Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Modify vcopy to enable multi iteration and multi kernel for CI Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Update docs and --help menu to reflect new vcopy usage Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Fix typo in IPBlock logging Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Cache ascii art in source and remove ascii dependency Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Update testing collateral to reflect name change of kernel-verbose option Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Small typo corrections in docs Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Fixing more typos... Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * run profiler tests through queue Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * bump wallclock runlimit for test Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * add mi100 badge Signed-off-by: Karl W. Schulz Signed-off-by: JoseSantosAMD * include step to build hip binary Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * add additional top-level vars for number of kernels expected and dispatch id; tests updated to use these vars so that vcopy binary can be used instead of mixbench Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * add requirements-test.txt file to organize python testing dependencies (#205) * add a requirements-test.txt file to organize additional python packages needed to support testing; update CI to install python packages from dependency files Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * Dynamically load dependency checker from requirements.txt Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Add comments Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Replace any - with _ for checker logic Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Update cmake config with Fix for pyyaml package installation check Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * update mi100 CI config Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * upgrade jsdom version (#206) Signed-off-by: JoseSantosAMD * Adding the log_metric differ function Signed-off-by: JoseSantosAMD * Added missing hasattr checker Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Enable running on mi100 and fixed typos - Also enables RHEL9 roofline Signed-off-by: JoseSantosAMD * update CMakeLists.txt Signed-off-by: JoseSantosAMD * Remove pytest raises SystemExit Add default thresholds Signed-off-by: JoseSantosAMD * Initial overhaul of Analyze mode. Basic CLI is enabled. Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Revert "Remove pytest raises SystemExit" This reverts commit 9762728eb37e740c39ced87ced7dd7a0d423824d. Signed-off-by: JoseSantosAMD * restore formal call to sys.exit() in top-level main; this is needed for companion testing collateral which expects SystemExit exceptions to be raised. Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * Update return code error checks for roofline-only analysis on MI100 Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * disable coverage comment Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * fix comment typo Signed-off-by: Karl W Schulz Signed-off-by: JoseSantosAMD * Move error logging to util.py Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Move perfmon_configs dir into omniperf_soc dir. Rename config dirs for clarity Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Add a supported_archs property to Omniperf base class Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD * Remove pytest raises SystemExit Add default thresholds Signed-off-by: JoseSantosAMD * Revert "Remove pytest raises SystemExit" This reverts commit 9762728eb37e740c39ced87ced7dd7a0d423824d. Signed-off-by: JoseSantosAMD * Add Absolute Difference to report-diff output - Pytest regex updated to read Absolute Difference - Pytest creates Baseline (Assumes vcopy binary present) - Adding Metric logging Signed-off-by: JoseSantosAMD * vcopy binary created before running tests Signed-off-by: JoseSantosAMD * Adding the log_metric differ function Signed-off-by: JoseSantosAMD * Update test_profile_general.py remove extra log_metric function Signed-off-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com> * Update CMakeLists.txt soc_params was merged into a class, it isn't a standalone file anymore Signed-off-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com> --------- Signed-off-by: JoseSantosAMD Signed-off-by: Karl W Schulz Signed-off-by: Karl W. Schulz Signed-off-by: colramos-amd Signed-off-by: JoseSantosAMD <87447437+JoseSantosAMD@users.noreply.github.com> Co-authored-by: Karl W Schulz Co-authored-by: colramos-amd Co-authored-by: Karl W. Schulz [ROCm/rocprofiler-compute commit: 6d7c631cd78a0b03e8ea137a921d2c9685f75081] --- projects/rocprofiler-compute/CMakeLists.txt | 6 - projects/rocprofiler-compute/src/utils/tty.py | 23 +- .../tests/test_profile_general.py | 373 +++++++++++++++--- 3 files changed, 335 insertions(+), 67 deletions(-) diff --git a/projects/rocprofiler-compute/CMakeLists.txt b/projects/rocprofiler-compute/CMakeLists.txt index 10bff8e7b7..64c088a160 100644 --- a/projects/rocprofiler-compute/CMakeLists.txt +++ b/projects/rocprofiler-compute/CMakeLists.txt @@ -245,12 +245,6 @@ install( src/utils/rooflines/roofline-ubuntu20_04-mi200-rocm5 DESTINATION ${CMAKE_INSTALL_BINDIR}/utils/rooflines) -# src/soc_params -install( - DIRECTORY src/soc_params - TYPE BIN - PATTERN "__pycache__" EXCLUDE) - # samples install(DIRECTORY sample TYPE DATA) diff --git a/projects/rocprofiler-compute/src/utils/tty.py b/projects/rocprofiler-compute/src/utils/tty.py index 668567ac08..f63cee0946 100644 --- a/projects/rocprofiler-compute/src/utils/tty.py +++ b/projects/rocprofiler-compute/src/utils/tty.py @@ -119,8 +119,12 @@ def show_all(args, runs, archConfigs, output): ], axis=1, ) - diff = t_df.iloc[:, 1] - t_df.iloc[:, 0] - t_df = diff / t_df.iloc[:, 0].replace(0, 1) + absolute_diff = ( + t_df.iloc[:, 1] - t_df.iloc[:, 0] + ).round(args.decimal) + t_df = absolute_diff / t_df.iloc[:, 0].replace( + 0, 1 + ) if args.verbose >= 2: print("---------", header, t_df) @@ -131,6 +135,7 @@ def show_all(args, runs, archConfigs, output): ) # show value + percentage # TODO: better alignment + t_df = ( cur_df[header] .astype(float) @@ -141,24 +146,28 @@ def show_all(args, runs, archConfigs, output): + "%)" ) df = pd.concat([df, t_df], axis=1) - + df["Abs Diff"] = absolute_diff + # DEBUG: When in a CI setting and flag is set, # then verify metrics meet threshold requirement if args.report_diff: if ( - t_df_pretty.abs() + header in ["Value", "Count", "Avg"] + and t_df_pretty.abs() .gt(args.report_diff) .any() ): - violation_idx = t_df_pretty.index[t_df_pretty.abs() > args.report_diff] + violation_idx = t_df_pretty.index[ + t_df_pretty.abs() > args.report_diff + ] print( "DEBUG ERROR: Dataframe diff exceeds {} threshold requirement\nSee metric {}".format( str(args.report_diff) + "%", - violation_idx.to_numpy() + violation_idx.to_numpy(), ) ) print(df) - sys.exit(1) + else: cur_df_copy = copy.deepcopy(cur_df) cur_df_copy[header] = [ diff --git a/projects/rocprofiler-compute/tests/test_profile_general.py b/projects/rocprofiler-compute/tests/test_profile_general.py index 3950c021d8..8f53bb9d9b 100644 --- a/projects/rocprofiler-compute/tests/test_profile_general.py +++ b/projects/rocprofiler-compute/tests/test_profile_general.py @@ -8,17 +8,15 @@ import subprocess import re import shutil import inspect +import sys omniperf = SourceFileLoader("omniperf", "src/omniperf").load_module() workload_1 = os.path.realpath("workload") kernel_name_1 = "vecCopy(double*, double*, double*, int, int) [clone .kd]" -## kernel_name_1 = "void benchmark_func<__half2, 256, 8u, 11u>(__half2, __half2*) " -# change to directory where app is at -# app_1 = ["./sample/vcopy", "1048576", "256"] -# app_1 = ["./mixbench/build_mi100/mixbench-hip"] -app_1 = ["./tests/vcopy", "-n 1048576", "-b 256"] -num_kernels = 1 +app_1 = ["./sample/vcopy", "-n", "1048576", "-b", "256", "-i", "3"] +num_kernels = 3 + dispatch_id = 0 ALL_CSVS = [ @@ -89,13 +87,10 @@ ROOF_ONLY_FILES = [ "timestamps.csv", ] - -# change to directory where baseline is at -# Baseline_dir = os.path.realpath("Baseline_vcopy_mi100") -Baseline_dir = os.path.realpath("Baseline_mixbench") - # logging function for threshold outliers set to false -LOGGING = False +COUNTER_LOGGING = False +METRIC_LOGGING = True + def metric_compare(test_name, errors_pd, baseline_df, run_df, threshold=5): @@ -190,6 +185,33 @@ def gpu_soc(): soc = gpu_soc() +# change to directory where baseline is at +Baseline_dir = os.path.realpath("Baseline_vcopy_" + soc) +if os.path.exists(Baseline_dir): + shutil.rmtree(Baseline_dir) +subprocess.run( + ["hipcc", "sample/vcopy.cpp", "-o", "sample/vcopy"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, +) +with pytest.raises(SystemExit) as e: + with patch( + "sys.argv", + [ + "omniperf", + "profile", + "-n", + "app_1", + "-VVV", + "--path", + Baseline_dir, + "--", + ] + + app_1, + ): + omniperf.main() + + def logger(file_dict, test_name): for file in file_dict.keys(): @@ -224,6 +246,82 @@ def logger(file_dict, test_name): Baseline_dir + "/" + file.split(".")[0] + "_error_log.csv" ) +def log_metric(test_name, thresholds, args=[]): + t = subprocess.Popen( + [ + sys.executable, + "src/omniperf", + "analyze", + "--path", + Baseline_dir, + ] + + args + + ["--path", workload_1, "--report-diff", "5"], + stdout=subprocess.PIPE, + ) + captured_output = t.communicate(timeout=1300)[0].decode("utf-8") + print("captured_output", captured_output) + if "DEBUG ERROR" in captured_output: + error_df = pd.DataFrame() + output_metric_errors = re.findall(r"(\')([0-9.]*)(\')", captured_output) + naughty_metrics = [x[1] for x in output_metric_errors] + print("naughty metrics:", naughty_metrics) + for metric in naughty_metrics: + metric_info = re.findall( + r"(^" + + metric + + r" *)([()0-9A-Za-z-]+ ?)*(?: *)([0-9.-]*)(?: *)([0-9.-]*)(?: *)\(([-0-9.]*)%\)(?: *)([-0-9.e]*)", + captured_output, + flags=re.MULTILINE, + ) + if len(metric_info): + metric_info = metric_info[0] + if metric_info[-4] != "-100.0": + print("metric info:", metric_info) + print( + re.findall( + r"^" + metric + r".*", captured_output, flags=re.MULTILINE + ) + ) + table_idx = metric_info[1].split(".")[0] + relative_diff = float(metric_info[-2]) + absolute_diff = float(metric_info[-1]) + relative_threshold = thresholds["default"]["relative"] + absolute_threshold = thresholds["default"]["absolute"] + + if table_idx in thresholds: + relative_threshold = thresholds[table_idx]["relative"] + absolute_threshold = thresholds[table_idx]["absolute"] + + if ( + relative_diff > relative_threshold + or absolute_diff > absolute_threshold + ): + new_error = pd.DataFrame.from_dict( + { + "index": [metric_info[0].split()[0]], + "metric": [table_idx], + "Percent Difference": [relative_diff], + "Absolute Difference": [absolute_diff], + "Baseline": [metric_info[-2]], + "Current": [metric_info[-1]], + } + ) + error_df = pd.concat([error_df, new_error]) + + error_df["test_name"] = test_name + if os.path.exists(Baseline_dir + "/metric_error_log.csv"): + error_log = pd.read_csv( + Baseline_dir + "/metric_error_log.csv", + index_col=0, + ) + new_error_log = pd.concat([error_log, error_df]) + new_error_log = new_error_log.reindex(sorted(new_error_log.columns), axis=1) + new_error_log = new_error_log.sort_values(by=["metric", "test_name"]) + new_error_log.to_csv(Baseline_dir + "/metric_error_log.csv") + else: + error_df.to_csv(Baseline_dir + "/metric_error_log.csv") + def test_path(): if os.path.exists(workload_1): @@ -266,8 +364,11 @@ def test_path(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_kernel(): @@ -310,8 +411,11 @@ def test_kernel(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_kernel_summaries(): @@ -354,9 +458,11 @@ def test_kernel_summaries(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) - + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_ipblocks_SQ(): if os.path.exists(workload_1): @@ -440,8 +546,11 @@ def test_ipblocks_SQ(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_ipblocks_SQC(): @@ -493,8 +602,12 @@ def test_ipblocks_SQC(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_TA(): @@ -552,8 +665,11 @@ def test_ipblocks_TA(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_ipblocks_TD(): @@ -615,8 +731,12 @@ def test_ipblocks_TD(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_TCP(): @@ -675,8 +795,13 @@ def test_ipblocks_TCP(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_ipblocks_TCC(): @@ -736,8 +861,12 @@ def test_ipblocks_TCC(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_SPI(): @@ -795,8 +924,12 @@ def test_ipblocks_SPI(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_CPC(): @@ -849,8 +982,11 @@ def test_ipblocks_CPC(): expected_csvs.insert(7, "roofline.csv") assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_ipblocks_CPF(): @@ -901,8 +1037,12 @@ def test_ipblocks_CPF(): expected_csvs.insert(5, "roofline.csv") assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_SQ_CPC(): @@ -989,8 +1129,12 @@ def test_ipblocks_SQ_CPC(): assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_SQ_TA(): @@ -1076,8 +1220,12 @@ def test_ipblocks_SQ_TA(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_SQ_SPI(): @@ -1163,8 +1311,12 @@ def test_ipblocks_SQ_SPI(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_SQ_SQC_TCP_CPC(): @@ -1253,8 +1405,12 @@ def test_ipblocks_SQ_SQC_TCP_CPC(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_ipblocks_SQ_SPI_TA_TCC_CPF(): @@ -1345,8 +1501,11 @@ def test_ipblocks_SQ_SPI_TA_TCC_CPF(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_dispatch_0(): @@ -1389,8 +1548,13 @@ def test_dispatch_0(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_dispatch_0_1(): @@ -1426,15 +1590,21 @@ def test_dispatch_0_1(): if file.endswith(".csv"): file_dict[file] = pd.read_csv(workload_1 + "/" + file) if not "sysinfo" in file and not "roofline" in file: - assert len(file_dict[file].index) == num_kernels + assert len(file_dict[file].index) == 2 + if soc == "mi200": print(sorted(list(file_dict.keys()))) assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_dispatch_2(): @@ -1477,8 +1647,13 @@ def test_dispatch_2(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_kernel_verbose_0(): @@ -1521,8 +1696,13 @@ def test_kernel_verbose_0(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_kernel_verbose_1(): @@ -1565,8 +1745,13 @@ def test_kernel_verbose_1(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_kernel_verbose_2(): @@ -1609,8 +1794,11 @@ def test_kernel_verbose_2(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_kernel_verbose_3(): @@ -1653,8 +1841,11 @@ def test_kernel_verbose_3(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) def test_kernel_verbose_4(): @@ -1697,8 +1888,13 @@ def test_kernel_verbose_4(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_kernel_verbose_5(): @@ -1741,8 +1937,13 @@ def test_kernel_verbose_5(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_join_type_grid(): @@ -1785,8 +1986,12 @@ def test_join_type_grid(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_join_type_kernel(): @@ -1829,8 +2034,13 @@ def test_join_type_kernel(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_device_0(): @@ -1876,8 +2086,13 @@ def test_device_0(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_no_roof(): @@ -1945,8 +2160,13 @@ def test_no_roof(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_sort_dispatches(): @@ -1998,8 +2218,13 @@ def test_sort_dispatches(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_sort_kernels(): @@ -2050,8 +2275,13 @@ def test_sort_kernels(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_mem_levels_HBM(): @@ -2103,8 +2333,13 @@ def test_mem_levels_HBM(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_mem_levels_L2(): @@ -2156,8 +2391,13 @@ def test_mem_levels_L2(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_mem_levels_vL1D(): @@ -2208,8 +2448,13 @@ def test_mem_levels_vL1D(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + + def test_mem_levels_LDS(): @@ -2260,8 +2505,12 @@ def test_mem_levels_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_mem_levels_HBM_LDS(): @@ -2313,8 +2562,12 @@ def test_mem_levels_HBM_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_mem_levels_vL1D_LDS(): @@ -2366,8 +2619,12 @@ def test_mem_levels_vL1D_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_mem_levels_L2_vL1D_LDS(): @@ -2419,8 +2676,12 @@ def test_mem_levels_L2_vL1D_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + def test_kernel_names(): @@ -2481,5 +2742,9 @@ def test_kernel_names(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if LOGGING: + if COUNTER_LOGGING: logger(file_dict, inspect.stack()[0][3]) + + if METRIC_LOGGING: + log_metric(inspect.stack()[0][3], {"default": {"absolute": 1, "relative": 5}}) +