From bec3f337c1b52da35b66b4208ce78adcd15ec0e0 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Wed, 17 Jan 2024 13:33:03 -0600 Subject: [PATCH] Update list of metrics with very low variance Signed-off-by: JoseSantosAMD [ROCm/rocprofiler-compute commit: f461d0052b42800600cb2ab4820e0b3e785c1189] --- .../tests/test_profile_general.py | 1275 +++++++++++------ 1 file changed, 839 insertions(+), 436 deletions(-) diff --git a/projects/rocprofiler-compute/tests/test_profile_general.py b/projects/rocprofiler-compute/tests/test_profile_general.py index 1e2d09ba23..e12f1caeb5 100644 --- a/projects/rocprofiler-compute/tests/test_profile_general.py +++ b/projects/rocprofiler-compute/tests/test_profile_general.py @@ -21,6 +21,7 @@ config["kernel_name_1"] = "vecCopy(double*, double*, double*, int, int) [clone . config["app_1"] = ["./tests/vcopy", "-n", "1048576", "-b", "256", "-i", "3"] config["cleanup"] = True config["COUNTER_LOGGING"] = False +config["METRIC_COMPARE"] = True config["METRIC_LOGGING"] = False baseline_opts = ["omniperf", "profile", "-n", "app_1", "-VVV"] @@ -31,9 +32,9 @@ baseline_opts = ["omniperf", "profile", "-n", "app_1", "-VVV"] num_kernels = 3 dispatch_id = 0 -DEFAULT_ABS_DIFF = 2.2 -DEFAULT_REL_DIFF = 8 -MAX_METRIC_VIOLATIONS = 2 +DEFAULT_ABS_DIFF = 5 +DEFAULT_REL_DIFF = 5 +MAX_REOCCURING_COUNT = 3 ALL_CSVS = [ "SQ_IFETCH_LEVEL.csv", @@ -102,63 +103,214 @@ ROOF_ONLY_FILES = [ "timestamps.csv", ] -# logging function for threshold outliers set to false -COUNTER_LOGGING = False -METRIC_LOGGING = False +# Must not change relative difference is zero +FIXED_METRICS = [ + "2.1.0", + "2.1.1", + "2.1.2", + "2.1.3", + "2.1.4", + "2.1.5", + "2.1.6", + "2.1.10", + "2.1.12", + "2.1.13", + "2.1.14", + "2.1.15", + # "2.1.17", + "2.1.19", + # "2.1.21", + "6.1.5", + "6.2.1", + "6.2.2", + "6.2.3", + "6.2.4", + "6.2.5", + "6.2.6", + "6.2.7", + "6.2.8", + "6.2.9", + "6.2.10", + "6.2.11", + "6.2.12", + "6.2.13", + "7.1.0", + "7.1.1", + "7.1.2", + "7.1.3", + "7.1.4", + "7.1.5", + "7.1.6", + "7.1.7", + "7.1.8", + "7.1.9", + "7.2.6", + "10.1.0", + "10.1.1", + "10.1.2", + "10.1.3", + "10.1.4", + "10.1.5", + "10.1.6", + "10.1.7", + "10.2.0", + "10.2.1", + "10.2.2", + "10.2.3", + "10.2.4", + "10.2.5", + "10.2.6", + "10.2.7", + "10.2.8", + "10.2.9", + "10.2.10", + "10.2.11", + "10.2.12", + "10.2.13", + "10.2.14", + "10.3.0", + "10.3.1", + "10.3.2", + "10.3.3", + "10.3.4", + "10.3.5", + "10.3.6", + "10.3.7", + "10.4.0", + "10.4.1", + "10.4.2", + "10.4.3", + "10.4.4", + "11.2.1", + "11.2.4", + "11.2.5", + "11.2.6", + "11.3.0", + "11.3.1", + "11.3.2", + "11.3.3", + "11.3.4", + "11.3.5", + "12.1.0", + "12.1.1", + "12.1.2", + "12.1.3", + "12.2.0", + "12.2.1", + "12.2.3", + "12.2.4", + "12.2.5", + "12.2.6", + "12.2.7", + "12.2.8", + "12.2.9", + "13.2.0", + # "13.2.1", + "13.2.2", + # "13.2.3", + # "14.1.1", + "14.2.0", + # "14.2.1", + "14.2.2", + # "14.2.3", + # "14.2.4", + "14.2.5", + "14.2.6", + "14.2.7", + "14.2.8", + "14.2.9", + "14.2.10", + "14.2.11", + "14.3.0", + "14.3.1", + "14.3.2", + "14.3.3", + "15.1.3", + "15.1.4", + "15.1.5", + "15.1.6", + "15.1.7", + "15.1.8", + "15.1.9", + "15.1.10", + "15.1.11", + "15.1.12", + "15.1.13", + "15.1.14", + "15.1.15", + "15.2.2", + "15.2.3", + "15.2.4", + "15.2.5", + "15.2.6", + "16.1.0", + "16.1.3", + "16.2.2", + "16.2.3", + "16.2.4", + "16.3.0", + "16.3.1", + "16.3.2", + "16.3.3", + "16.3.5", + "16.3.6", + "16.3.7", + # "16.3.8", # rel diff is -102 + "16.3.9", + "16.3.10", + "16.3.11", + "16.3.12", + "16.4.0", + "16.4.1", + "16.4.2", + "16.4.3", + "16.4.4", + "16.4.5", + "16.4.6", + "16.4.7", + "16.4.8", + "16.4.9", + "16.4.10", + "16.4.11", + "16.5.0", + "16.5.4", + # "17.1.1", + "17.2.2", + "17.2.6", + "17.2.7", + "17.2.12", + # "17.3.0", # rel diff is 0.06 + "17.3.1", + "17.3.3", + "17.3.4", + "17.3.5", + "17.3.6", + # "17.3.7", # rel diff is -0.06 + # "17.3.8", + "17.3.10", + "17.3.12", + "17.3.13", + "17.3.17", + "18.1.0", + "18.1.1", + "18.1.2", + "18.1.3", + "18.1.4", + "5.1.2", + "5.1.6", + # "5.2.2", + "6.1.4", + "18.1.5", + "18.1.6", + "18.1.7", + "18.1.10", + "18.1.11", + "18.1.12", + "18.1.14", + "18.1.15", + "18.1.17", +] -# Absolute Difference < 2 -CONSISTENT_ABS_METRIC_INDICES = [ - "2.1.8", - # "2.1.28", - "2.1.9", - "5.1.3", - "13.1.0", - "14.1.0", - "5.2.4", - "11.2.0", - "11.2.2", - "11.2.3", - "13.1.0", - "13.2.3", - "16.5.3", - "17.2.3", - "17.2.8", - "17.2.9", - "17.3.8", - "17.3.9", - "17.3.11", - "17.3.14", - "17.3.15", - "17.3.16", - "18.1.13", - "18.1.16", -] -# Percent Difference < 8 -CONSISTENT_REL_METRIC_INDICES = [ - "2.1.26", - # "5.1.0", - # "5.1.1", - # "5.2.0", - # "5.2.1", - "5.2.3", - # "5.2.4", - "5.2.6", - "5.2.8", - # "6.1.0", - # "6.1.1", - "6.1.3" "6.1.6", - # "6.1.7", - # "6.2.0", - # "7.2.1", - # "7.2.3", - "7.2.4", - "7.2.7", - # "14.1.0", - "16.2.0", - # "16.3.14", - # "16.3.15", - "17.1.0", -] # check for parallel resource allocation test_utils.check_resource_allocation() @@ -256,27 +408,7 @@ def gpu_soc(): soc = gpu_soc() -if config["METRIC_LOGGING"]: - # change to directory where baseline is at - Baseline_dir = os.path.realpath("Baseline_vcopy_" + soc) - if os.path.exists(Baseline_dir): - shutil.rmtree(Baseline_dir) - with pytest.raises(SystemExit) as e: - with patch( - "sys.argv", - [ - "omniperf", - "profile", - "-n", - "app_1", - "-VVV", - "--path", - Baseline_dir, - "--", - ] - + app_1, - ): - omniperf.main() +Baseline_dir = os.path.realpath("tests/workloads/Baseline_vcopy_" + soc) def log_counter(file_dict, test_name): @@ -312,7 +444,7 @@ def log_counter(file_dict, test_name): ) -def log_metric(test_name, thresholds, args=[]): +def baseline_compare_metric(test_name, workload_dir, thresholds, args=[]): t = subprocess.Popen( [ sys.executable, @@ -322,7 +454,7 @@ def log_metric(test_name, thresholds, args=[]): Baseline_dir, ] + args - + ["--path", workload_dir, "--report-diff", str(DEFAULT_REL_DIFF)], + + ["--path", workload_dir, "--report-diff", "-1"], stdout=subprocess.PIPE, ) captured_output = t.communicate(timeout=1300)[0].decode("utf-8") @@ -348,45 +480,100 @@ def log_metric(test_name, thresholds, args=[]): if len(metric_info): metric_info = metric_info[0] metric_idx = metric_info[0] - table_idx = metric_info[0].split(".")[0] + metric_name = metric_info[1].strip() + baseline_val = metric_info[-3] + current_val = metric_info[-4] relative_diff = float(metric_info[-2]) absolute_diff = float(metric_info[-1]) if relative_diff > -99 or relative_diff < -101: - relative_threshold = thresholds["default"]["relative"] - absolute_threshold = thresholds["default"]["absolute"] - - if table_idx in thresholds: - relative_threshold = thresholds[table_idx]["relative"] - absolute_threshold = thresholds[table_idx]["absolute"] - if ( - abs(relative_diff) > relative_threshold - and (metric_idx in CONSISTENT_REL_METRIC_INDICES) - ) or ( - abs(absolute_diff) > absolute_threshold - and (metric_idx in CONSISTENT_ABS_METRIC_INDICES) + relative_threshold = abs(thresholds["default"]["relative"]) + absolute_threshold = abs(thresholds["default"]["absolute"]) + if metric_idx in thresholds.keys(): + # print(metric_idx+" is in thresholds.keys") + relative_threshold = thresholds[metric_idx]["relative"] + absolute_threshold = thresholds[metric_idx]["absolute"] + # pick higher threshold, one of the thresholds is zero + isValid = ( + (abs(absolute_diff) <= absolute_threshold) + if absolute_threshold > relative_threshold + else (abs(relative_diff) <= relative_threshold) + ) + if not isValid: + print( + "index " + metric_name + " is not good enough.\nabsolute: ", + absolute_diff, + "relative: ", + relative_diff, + thresholds[metric_idx], + ) + assert 0 + else : + # print( + # "index " + metric_name + " is good enough.\nabsolute: ", + # absolute_diff, + # "relative: ", + # relative_diff, + # thresholds[metric_idx], + # ) + continue + if metric_idx in FIXED_METRICS: + # print(metric_idx+" is in FIXED_METRICS") + isValid = (abs(absolute_diff) == 0) and (abs(relative_diff) == 0) or (current_val == 0) + if not isValid: + print( + "index " + + metric_idx + + " difference is supposed to be 0, absolute:", + absolute_diff, + "relative: ", + relative_diff, + ) + # assert 0 + + #Used for debugging metric lists + if config["METRIC_LOGGING"] and ( + (abs(relative_diff) <= abs(relative_threshold) + or ( + abs(absolute_diff) + <= abs(absolute_threshold) + )) ): + print("logging...") + print(metric_info) + new_error = pd.DataFrame.from_dict( { - "Index": [metric_info[0]], - "Metric": [metric_info[1].strip()], + "Index": [metric_idx], + "Metric": [metric_name], "Percent Difference": [relative_diff], "Absolute Difference": [absolute_diff], - "Baseline": [metric_info[-3]], - "Current": [metric_info[-4]], + "Baseline": [baseline_val], + "Current": [current_val], "Test Name": [test_name], } ) error_df = pd.concat([error_df, new_error]) - counts = error_df.groupby("Index").cumcount() - failed_metrics = error_df.loc[counts > MAX_METRIC_VIOLATIONS] - if failed_metrics.any(axis=None): + counts = error_df.groupby(["Index", "Test Name"]).cumcount() + reoccurring_metrics = error_df.loc[counts > MAX_REOCCURING_COUNT] + reoccurring_metrics["counts"] = counts[counts > MAX_REOCCURING_COUNT] + if reoccurring_metrics.any(axis=None): print( - "Warning, these metrics are varying too much", - failed_metrics, + "These metrics appear alot\n", + reoccurring_metrics, ) + print(list(reoccurring_metrics["Index"])) - if not error_df.empty: - error_df.to_csv(Baseline_dir + "/metric_error_log.csv") + # log into csv + if not error_df.empty: + error_df.to_csv(Baseline_dir + "/metric_error_log.csv") + + +def logging(test_name, workload_dir, file_dict, thresholds, args=[]): + if config["COUNTER_LOGGING"]: + log_counter(file_dict, test_name) + + if config["METRIC_COMPARE"]: + baseline_compare_metric(test_name, workload_dir, thresholds, args) # -- @@ -406,14 +593,34 @@ def test_path(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.22": {"absolute": 0, "relative": 5}, + "2.1.27": {"absolute": 0, "relative": 5}, + "5.1.5": {"absolute": 5, "relative": 0}, + "5.2.8": {"absolute": 1, "relative": 0}, + "5.2.11": {"absolute": 2, "relative": 0}, + "6.1.1": {"absolute": 0, "relative": 6}, + "6.1.2": {"absolute": 0, "relative": 5}, + "13.1.0": {"absolute": 1, "relative": 0}, + "14.1.0": {"absolute": 0, "relative": 5}, + "16.1.2": {"absolute": 1, "relative": 0}, + "16.2.0": {"absolute": 0, "relative": 7}, + "16.3.4": {"absolute": 0, "relative": 5}, + "17.1.2": {"absolute": 0, "relative": 3}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.2.14": {"absolute": 5, "relative": 0}, + "17.3.14": {"absolute": 1, "relative": 0}, + "17.3.15": {"absolute": 1, "relative": 0}, + "17.3.16": {"absolute": 2, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -456,14 +663,12 @@ def test_no_roof(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -499,14 +704,12 @@ def test_kernel_names(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -529,14 +732,12 @@ def test_device_filter(): # TODO - verify expected device id in results - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -553,14 +754,12 @@ def test_kernel(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -577,14 +776,12 @@ def test_kernel_summaries(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -643,14 +840,30 @@ def test_ipblocks_SQ(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.9": {"absolute": 2, "relative": 0}, + "2.1.21": {"absolute": 1, "relative": 0}, + "11.2.2": {"absolute": 1, "relative": 0}, + "11.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 1, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 1, "relative": 0}, + "17.3.11": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -675,14 +888,26 @@ def test_ipblocks_SQC(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.21": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 1, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 1, "relative": 0}, + "17.3.11": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -712,14 +937,33 @@ def test_ipblocks_TA(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.21": {"absolute": 1, "relative": 0}, + "5.1.0": {"absolute": 0, "relative": 5}, + "5.2.0": {"absolute": 0, "relative": 5}, + "6.1.0": {"absolute": 0, "relative": 5}, + "7.2.1": {"absolute": 0, "relative": 5}, + "11.2.0": {"absolute": 1, "relative": 0}, + "13.2.3": {"absolute": 1, "relative": 0}, + "14.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 1, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 1, "relative": 0}, + "17.3.11": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -753,14 +997,27 @@ def test_ipblocks_TD(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.21": {"absolute": 1, "relative": 0}, + "11.2.0": {"absolute": 1, "relative": 0}, + "13.2.3": {"absolute": 1, "relative": 0}, + "14.2.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 1, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 1, "relative": 0}, + "17.3.11": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -792,14 +1049,27 @@ def test_ipblocks_TCP(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.21": {"absolute": 1, "relative": 0}, + "11.2.0": {"absolute": 1, "relative": 0}, + "13.2.3": {"absolute": 1, "relative": 0}, + "14.2.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 1, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 1, "relative": 0}, + "17.3.11": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -832,14 +1102,28 @@ def test_ipblocks_TCC(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "6.1.3": {"absolute": 1, "relative": 5}, + "11.2.0": {"absolute": 1, "relative": 0}, + "13.2.3": {"absolute": 1, "relative": 0}, + "14.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.2.14": {"absolute": 6, "relative": 0}, + "17.3.14": {"absolute": 1, "relative": 0}, + "17.3.15": {"absolute": 1, "relative": 0}, + "17.3.16": {"absolute": 2, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -870,14 +1154,35 @@ def test_ipblocks_SPI(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.21": {"absolute": 1, "relative": 0}, + "5.1.0": {"absolute": 0, "relative": 7}, + "5.2.0": {"absolute": 0, "relative": 7}, + "6.1.0": {"absolute": 0, "relative": 7}, + "6.1.1": {"absolute": 0, "relative": 7}, + "6.1.3": {"absolute": 0, "relative": 5}, + "7.2.1": {"absolute": 0, "relative": 7}, + "11.2.0": {"absolute": 1, "relative": 0}, + "13.2.3": {"absolute": 5, "relative": 0}, + "14.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 6, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 1, "relative": 0}, + "17.3.11": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -904,14 +1209,34 @@ def test_ipblocks_CPC(): expected_csvs.insert(7, "roofline.csv") assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.21": {"absolute": 1, "relative": 0}, + "5.1.0": {"absolute": 0, "relative": 7}, + "5.2.1": {"absolute": 0, "relative": 7}, + "5.2.8": {"absolute": 1, "relative": 0}, + "6.1.0": {"absolute": 0, "relative": 5}, + "7.2.1": {"absolute": 0, "relative": 5}, + "11.2.0": {"absolute": 1, "relative": 0}, + "13.2.3": {"absolute": 5, "relative": 0}, + "14.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 6, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 2, "relative": 0}, + "17.3.11": {"absolute": 2, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -936,14 +1261,30 @@ def test_ipblocks_CPF(): expected_csvs.insert(5, "roofline.csv") assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.21": {"absolute": 1, "relative": 0}, + "5.1.5": {"absolute": 2, "relative": 0}, + "11.2.0": {"absolute": 1, "relative": 0}, + "13.2.3": {"absolute": 5, "relative": 0}, + "14.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 6, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 2, "relative": 0}, + "17.3.11": {"absolute": 2, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1002,14 +1343,32 @@ def test_ipblocks_SQ_CPC(): assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.9": {"absolute": 1, "relative": 0}, + "2.1.21": {"absolute": 1, "relative": 0}, + "5.2.6": {"absolute": 0, "relative": 6}, + "5.2.8": {"absolute": 2, "relative": 0}, + "6.1.3": {"absolute": 0, "relative": 6}, + "11.2.2": {"absolute": 1, "relative": 0}, + "11.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 6, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 2, "relative": 0}, + "17.3.11": {"absolute": 2, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1067,14 +1426,34 @@ def test_ipblocks_SQ_TA(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.9": {"absolute": 1, "relative": 0}, + "2.1.21": {"absolute": 1, "relative": 0}, + "5.1.0": {"absolute": 0, "relative": 5}, + "5.2.0": {"absolute": 0, "relative": 5}, + "6.1.0": {"absolute": 0, "relative": 5}, + "6.1.3": {"absolute": 0, "relative": 6}, + "7.2.1": {"absolute": 0, "relative": 5}, + "11.2.2": {"absolute": 1, "relative": 0}, + "11.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 6, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 2, "relative": 0}, + "17.3.11": {"absolute": 2, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1132,15 +1511,33 @@ def test_ipblocks_SQ_SPI(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) - + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.9": {"absolute": 1, "relative": 0}, + "2.1.21": {"absolute": 1, "relative": 0}, + "6.1.3": {"absolute": 0, "relative": 5}, + "7.2.7": {"absolute": 0, "relative": 5}, + "11.2.2": {"absolute": 1, "relative": 0}, + "11.2.3": {"absolute": 1, "relative": 0}, + "13.1.0": {"absolute": 1, "relative": 0}, + "14.1.0": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.1": {"absolute": 6, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 2, "relative": 0}, + "17.3.11": {"absolute": 2, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1198,14 +1595,36 @@ def test_ipblocks_SQ_SQC_TCP_CPC(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.16": {"absolute": 0, "relative": 5}, + "2.1.18": {"absolute": 0, "relative": 5}, + "2.1.20": {"absolute": 0, "relative": 5}, + "2.1.21": {"absolute": 1, "relative": 0}, + "2.1.27": {"absolute": 0, "relative": 5}, + "5.2.6": {"absolute": 0, "relative": 10}, + "5.2.8": {"absolute": 1, "relative": 0}, + "7.2.0": {"absolute": 0, "relative": 5}, + "13.1.0": {"absolute": 0, "relative": 5}, + "14.1.0": {"absolute": 0, "relative": 5}, + "16.1.2": {"absolute": 0, "relative": 5}, + "16.3.4": {"absolute": 0, "relative": 5}, + "16.5.3": {"absolute": 5, "relative": 0}, + "17.1.1": {"absolute": 1, "relative": 0}, + "17.2.3": {"absolute": 1, "relative": 0}, + "17.3.6": {"absolute": 1, "relative": 0}, + "17.3.8": {"absolute": 1, "relative": 0}, + "17.3.1": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1265,14 +1684,29 @@ def test_ipblocks_SQ_SPI_TA_TCC_CPF(): ] assert sorted(list(file_dict.keys())) == expected_csvs - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "2.1.9": {"absolute": 1, "relative": 0}, + "5.1.1": {"absolute": 0, "relative": 10}, + "5.1.5": {"absolute": 3, "relative": 0}, + "11.2.2": {"absolute": 1, "relative": 0}, + "11.2.3": {"absolute": 1, "relative": 0}, + "16.3.8": {"absolute": 1, "relative": 0}, + "16.5.3": {"absolute": 1, "relative": 0}, + "17.1.0": {"absolute": 0, "relative": 5}, + "17.2.13": {"absolute": 5, "relative": 0}, + "17.3.14": {"absolute": 1, "relative": 0}, + "17.3.15": {"absolute": 1, "relative": 0}, + "18.1.0": {"absolute": 1, "relative": 0}, + "18.1.6": {"absolute": 1, "relative": 0}, + "18.1.13": {"absolute": 1, "relative": 0}, + "18.1.16": {"absolute": 1, "relative": 0}, + }, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1289,18 +1723,16 @@ def test_dispatch_0(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - [ - "--dispatch", - "0", - ], - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + [ + "--dispatch", + "0", + ], + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1317,15 +1749,13 @@ def test_dispatch_0_1(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ["--dispatch", "0", "1"], - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ["--dispatch", "0", "1"], + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1342,18 +1772,16 @@ def test_dispatch_2(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - [ - "--dispatch", - str(dispatch_id), - ], - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + [ + "--dispatch", + str(dispatch_id), + ], + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1370,16 +1798,21 @@ def test_kernel_verbose_0(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + { + "default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}, + "15.1.2": {"absolute": 5, "relative": 0}, + "17.2.3": {"absolute": 5, "relative": 0}, + "17.2.14": {"absolute": 5, "relative": 0}, + "17.3.9": {"absolute": 5, "relative": 0}, + "18.1.16": {"absolute": 5, "relative": 0}, + }, + ) - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) - - test_utils.clean_output_dir(config["cleanup"], workload_dir) + # test_utils.clean_output_dir(config["cleanup"], workload_dir) @pytest.mark.verbosity @@ -1394,14 +1827,12 @@ def test_kernel_verbose_1(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1418,14 +1849,12 @@ def test_kernel_verbose_2(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1442,14 +1871,12 @@ def test_kernel_verbose_3(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1466,14 +1893,12 @@ def test_kernel_verbose_4(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1490,14 +1915,12 @@ def test_kernel_verbose_5(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1514,14 +1937,12 @@ def test_join_type_grid(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1539,14 +1960,12 @@ def test_join_type_kernel(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1566,19 +1985,19 @@ def test_sort_dispatches(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) + if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1604,14 +2023,12 @@ def test_sort_kernels(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1637,14 +2054,12 @@ def test_mem_levels_HBM(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1670,14 +2085,12 @@ def test_mem_levels_L2(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1703,14 +2116,12 @@ def test_mem_levels_vL1D(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1736,14 +2147,12 @@ def test_mem_levels_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1769,14 +2178,12 @@ def test_mem_levels_HBM_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1802,14 +2209,12 @@ def test_mem_levels_vL1D_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir) @@ -1834,13 +2239,11 @@ def test_mem_levels_L2_vL1D_LDS(): else: assert sorted(list(file_dict.keys())) == ALL_CSVS - if config["COUNTER_LOGGING"]: - log_counter(file_dict, inspect.stack()[0][3]) - - if config["METRIC_LOGGING"]: - log_metric( - inspect.stack()[0][3], - {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, - ) + logging( + inspect.stack()[0][3], + workload_dir, + file_dict, + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) test_utils.clean_output_dir(config["cleanup"], workload_dir)