diff --git a/tests/test_profile_general.py b/tests/test_profile_general.py index 8f53bb9d9b..b763c72d90 100644 --- a/tests/test_profile_general.py +++ b/tests/test_profile_general.py @@ -14,11 +14,15 @@ omniperf = SourceFileLoader("omniperf", "src/omniperf").load_module() workload_1 = os.path.realpath("workload") kernel_name_1 = "vecCopy(double*, double*, double*, int, int) [clone .kd]" -app_1 = ["./sample/vcopy", "-n", "1048576", "-b", "256", "-i", "3"] +app_1 = ["./tests/vcopy", "-n", "1048576", "-b", "256", "-i", "3"] num_kernels = 3 dispatch_id = 0 +DEFAULT_ABS_DIFF = 2.2 +DEFAULT_REL_DIFF = 8 +MAX_METRIC_VIOLATIONS = 2 + ALL_CSVS = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -74,7 +78,6 @@ ALL_CSVS_MI200 = [ "sysinfo.csv", "timestamps.csv", ] - ROOF_ONLY_FILES = [ "empirRoof_gpu-ALL_fp32.pdf", "empirRoof_gpu-ALL_int8_fp16.pdf", @@ -91,6 +94,59 @@ ROOF_ONLY_FILES = [ COUNTER_LOGGING = False METRIC_LOGGING = True +# Absolute Difference < 2 +CONSISTENT_ABS_METRIC_INDICES = [ + "2.1.8", + # "2.1.28", + "2.1.9", + "5.1.3", + "13.1.0", + "14.1.0", + "5.2.4", + "11.2.0", + "11.2.2", + "11.2.3", + "13.1.0", + "13.2.3", + "16.5.3", + "17.2.3", + "17.2.8", + "17.2.9", + "17.3.8", + "17.3.9", + "17.3.11", + "17.3.14", + "17.3.15", + "17.3.16", + "18.1.13", + "18.1.16", +] +# Percent Difference < 8 +CONSISTENT_REL_METRIC_INDICES = [ + "2.1.26", + # "5.1.0", + # "5.1.1", + # "5.2.0", + # "5.2.1", + "5.2.3", + # "5.2.4", + "5.2.6", + "5.2.8", + # "6.1.0", + # "6.1.1", + "6.1.3" "6.1.6", + # "6.1.7", + # "6.2.0", + # "7.2.1", + # "7.2.3", + "7.2.4", + "7.2.7", + # "14.1.0", + "16.2.0", + # "16.3.14", + # "16.3.15", + "17.1.0", +] def metric_compare(test_name, errors_pd, baseline_df, run_df, threshold=5): @@ -169,7 +225,6 @@ def gpu_soc(): soc_regex = re.compile(r"^\s*Name\s*:\s+ ([a-zA-Z0-9]+)\s*$", re.MULTILINE) gpu_id = list(filter(soc_regex.match, rocminfo))[0].split()[1] - print("gpu_id", gpu_id) if gpu_id == "gfx906": return "mi50" elif gpu_id == "gfx908": @@ -189,11 +244,6 @@ soc = gpu_soc() Baseline_dir = os.path.realpath("Baseline_vcopy_" + soc) if os.path.exists(Baseline_dir): shutil.rmtree(Baseline_dir) -subprocess.run( - ["hipcc", "sample/vcopy.cpp", "-o", "sample/vcopy"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) with pytest.raises(SystemExit) as e: with patch( "sys.argv", @@ -212,11 +262,9 @@ with pytest.raises(SystemExit) as e: omniperf.main() - -def logger(file_dict, test_name): +def log_counter(file_dict, test_name): for file in file_dict.keys(): if file == "pmc_perf.csv" or "SQ" in file: - print(file) # read file in Baseline df_1 = pd.read_csv(Baseline_dir + "/" + file, index_col=0) # get corresponding file from current test run @@ -246,6 +294,7 @@ def logger(file_dict, test_name): Baseline_dir + "/" + file.split(".")[0] + "_error_log.csv" ) + def log_metric(test_name, thresholds, args=[]): t = subprocess.Popen( [ @@ -256,72 +305,72 @@ def log_metric(test_name, thresholds, args=[]): Baseline_dir, ] + args - + ["--path", workload_1, "--report-diff", "5"], + + ["--path", workload_1, "--report-diff", str(DEFAULT_REL_DIFF)], stdout=subprocess.PIPE, ) captured_output = t.communicate(timeout=1300)[0].decode("utf-8") - print("captured_output", captured_output) + assert t.returncode == 0 + if "DEBUG ERROR" in captured_output: error_df = pd.DataFrame() + if os.path.exists(Baseline_dir + "/metric_error_log.csv"): + error_df = pd.read_csv( + Baseline_dir + "/metric_error_log.csv", + index_col=0, + ) output_metric_errors = re.findall(r"(\')([0-9.]*)(\')", captured_output) - naughty_metrics = [x[1] for x in output_metric_errors] - print("naughty metrics:", naughty_metrics) - for metric in naughty_metrics: + high_diff_metrics = [x[1] for x in output_metric_errors] + for metric in high_diff_metrics: metric_info = re.findall( r"(^" + metric - + r" *)([()0-9A-Za-z-]+ ?)*(?: *)([0-9.-]*)(?: *)([0-9.-]*)(?: *)\(([-0-9.]*)%\)(?: *)([-0-9.e]*)", + + r")(?: *)([()0-9A-Za-z- ]+ )(?: *)([0-9.-]*)(?: *)([0-9.-]*)(?: *)\(([-0-9.]*)%\)(?: *)([-0-9.e]*)", captured_output, flags=re.MULTILINE, ) if len(metric_info): metric_info = metric_info[0] - if metric_info[-4] != "-100.0": - print("metric info:", metric_info) - print( - re.findall( - r"^" + metric + r".*", captured_output, flags=re.MULTILINE - ) - ) - table_idx = metric_info[1].split(".")[0] - relative_diff = float(metric_info[-2]) - absolute_diff = float(metric_info[-1]) + metric_idx = metric_info[0] + table_idx = metric_info[0].split(".")[0] + relative_diff = float(metric_info[-2]) + absolute_diff = float(metric_info[-1]) + if relative_diff > -99 or relative_diff < -101: relative_threshold = thresholds["default"]["relative"] absolute_threshold = thresholds["default"]["absolute"] if table_idx in thresholds: relative_threshold = thresholds[table_idx]["relative"] absolute_threshold = thresholds[table_idx]["absolute"] - if ( - relative_diff > relative_threshold - or absolute_diff > absolute_threshold + abs(relative_diff) > relative_threshold + and (metric_idx in CONSISTENT_REL_METRIC_INDICES) + ) or ( + abs(absolute_diff) > absolute_threshold + and (metric_idx in CONSISTENT_ABS_METRIC_INDICES) ): new_error = pd.DataFrame.from_dict( { - "index": [metric_info[0].split()[0]], - "metric": [table_idx], + "Index": [metric_info[0]], + "Metric": [metric_info[1].strip()], "Percent Difference": [relative_diff], "Absolute Difference": [absolute_diff], - "Baseline": [metric_info[-2]], - "Current": [metric_info[-1]], + "Baseline": [metric_info[-3]], + "Current": [metric_info[-4]], + "Test Name": [test_name], } ) error_df = pd.concat([error_df, new_error]) + counts = error_df.groupby("Index").cumcount() + failed_metrics = error_df.loc[counts > MAX_METRIC_VIOLATIONS] + if failed_metrics.any(axis=None): + print( + "Warning, these metrics are varying too much", + failed_metrics, + ) - error_df["test_name"] = test_name - if os.path.exists(Baseline_dir + "/metric_error_log.csv"): - error_log = pd.read_csv( - Baseline_dir + "/metric_error_log.csv", - index_col=0, - ) - new_error_log = pd.concat([error_log, error_df]) - new_error_log = new_error_log.reindex(sorted(new_error_log.columns), axis=1) - new_error_log = new_error_log.sort_values(by=["metric", "test_name"]) - new_error_log.to_csv(Baseline_dir + "/metric_error_log.csv") - else: + if not error_df.empty: error_df.to_csv(Baseline_dir + "/metric_error_log.csv") - + def test_path(): if os.path.exists(workload_1): @@ -352,23 +401,24 @@ def test_path(): for file in files_in_workload: if file.endswith(".csv"): file_dict[file] = pd.read_csv(workload_1 + "/" + file, index_col=0) - print("length is: ", len(file_dict[file].index)) - print(file_dict[file]) # TODO: verify contents: we know function evaluated if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel(): @@ -406,16 +456,19 @@ def test_kernel(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel_summaries(): @@ -453,16 +506,20 @@ def test_kernel_summaries(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) + def test_ipblocks_SQ(): if os.path.exists(workload_1): @@ -498,7 +555,7 @@ def test_ipblocks_SQ(): file_dict[file] = pd.read_csv(workload_1 + "/" + file) if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -547,10 +604,13 @@ def test_ipblocks_SQ(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_SQC(): @@ -587,7 +647,7 @@ def test_ipblocks_SQC(): file_dict[file] = pd.read_csv(workload_1 + "/" + file) if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", "pmc_perf_0.csv", @@ -603,11 +663,13 @@ def test_ipblocks_SQC(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_TA(): @@ -645,7 +707,7 @@ def test_ipblocks_TA(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", @@ -666,10 +728,13 @@ def test_ipblocks_TA(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_TD(): @@ -707,7 +772,7 @@ def test_ipblocks_TD(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", @@ -732,11 +797,13 @@ def test_ipblocks_TD(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_TCP(): @@ -773,7 +840,7 @@ def test_ipblocks_TCP(): file_dict[file] = pd.read_csv(workload_1 + "/" + file) if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", @@ -796,12 +863,13 @@ def test_ipblocks_TCP(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_TCC(): @@ -839,7 +907,7 @@ def test_ipblocks_TCC(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", "pmc_perf_0.csv", @@ -862,11 +930,13 @@ def test_ipblocks_TCC(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_SPI(): @@ -903,7 +973,7 @@ def test_ipblocks_SPI(): file_dict[file] = pd.read_csv(workload_1 + "/" + file) if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", @@ -925,11 +995,13 @@ def test_ipblocks_SPI(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_CPC(): @@ -966,7 +1038,7 @@ def test_ipblocks_CPC(): file_dict[file] = pd.read_csv(workload_1 + "/" + file) if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", "pmc_perf_0.csv", @@ -983,10 +1055,13 @@ def test_ipblocks_CPC(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_CPF(): @@ -1023,7 +1098,7 @@ def test_ipblocks_CPF(): file_dict[file] = pd.read_csv(workload_1 + "/" + file) if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "pmc_perf.csv", "pmc_perf_0.csv", @@ -1038,11 +1113,13 @@ def test_ipblocks_CPF(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_SQ_CPC(): @@ -1081,7 +1158,7 @@ def test_ipblocks_SQ_CPC(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -1130,11 +1207,13 @@ def test_ipblocks_SQ_CPC(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_SQ_TA(): @@ -1173,7 +1252,7 @@ def test_ipblocks_SQ_TA(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -1221,11 +1300,13 @@ def test_ipblocks_SQ_TA(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_SQ_SPI(): @@ -1264,7 +1345,7 @@ def test_ipblocks_SQ_SPI(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -1312,11 +1393,13 @@ def test_ipblocks_SQ_SPI(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_SQ_SQC_TCP_CPC(): @@ -1357,7 +1440,7 @@ def test_ipblocks_SQ_SQC_TCP_CPC(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -1406,11 +1489,13 @@ def test_ipblocks_SQ_SQC_TCP_CPC(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_ipblocks_SQ_SPI_TA_TCC_CPF(): @@ -1452,7 +1537,7 @@ def test_ipblocks_SQ_SPI_TA_TCC_CPF(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels - print(sorted(list(file_dict.keys()))) + expected_csvs = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -1502,10 +1587,13 @@ def test_ipblocks_SQ_SPI_TA_TCC_CPF(): assert sorted(list(file_dict.keys())) == expected_csvs if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_dispatch_0(): @@ -1543,18 +1631,23 @@ def test_dispatch_0(): if not "sysinfo" in file and not "roofline" in file: assert len(file_dict[file].index) == 1 if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + [ + "--dispatch", + "0", + ], + ) def test_dispatch_0_1(): @@ -1593,18 +1686,20 @@ def test_dispatch_0_1(): assert len(file_dict[file].index) == 2 if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ["--dispatch", "0", "1"], + ) def test_dispatch_2(): @@ -1642,18 +1737,23 @@ def test_dispatch_2(): if not "sysinfo" in file and not "roofline" in file: assert len(file_dict[file].index) == 1 if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + [ + "--dispatch", + str(dispatch_id), + ], + ) def test_kernel_verbose_0(): @@ -1691,18 +1791,19 @@ def test_kernel_verbose_0(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel_verbose_1(): @@ -1740,18 +1841,19 @@ def test_kernel_verbose_1(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel_verbose_2(): @@ -1789,16 +1891,19 @@ def test_kernel_verbose_2(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel_verbose_3(): @@ -1836,16 +1941,19 @@ def test_kernel_verbose_3(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel_verbose_4(): @@ -1883,18 +1991,19 @@ def test_kernel_verbose_4(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel_verbose_5(): @@ -1932,18 +2041,19 @@ def test_kernel_verbose_5(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_join_type_grid(): @@ -1981,17 +2091,19 @@ def test_join_type_grid(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_join_type_kernel(): @@ -2029,18 +2141,19 @@ def test_join_type_kernel(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_device_0(): @@ -2081,18 +2194,19 @@ def test_device_0(): else: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_no_roof(): @@ -2129,7 +2243,7 @@ def test_no_roof(): if not "sysinfo" in file: assert len(file_dict[file].index) >= num_kernels if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -2161,12 +2275,13 @@ def test_no_roof(): assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_sort_dispatches(): @@ -2213,18 +2328,19 @@ def test_sort_dispatches(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_sort_kernels(): @@ -2270,18 +2386,19 @@ def test_sort_kernels(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_mem_levels_HBM(): @@ -2328,18 +2445,19 @@ def test_mem_levels_HBM(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_mem_levels_L2(): @@ -2386,18 +2504,19 @@ def test_mem_levels_L2(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_mem_levels_vL1D(): @@ -2443,18 +2562,19 @@ def test_mem_levels_vL1D(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) - - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_mem_levels_LDS(): @@ -2500,17 +2620,19 @@ def test_mem_levels_LDS(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_mem_levels_HBM_LDS(): @@ -2557,17 +2679,19 @@ def test_mem_levels_HBM_LDS(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_mem_levels_vL1D_LDS(): @@ -2614,17 +2738,19 @@ def test_mem_levels_vL1D_LDS(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_mem_levels_L2_vL1D_LDS(): @@ -2671,17 +2797,19 @@ def test_mem_levels_L2_vL1D_LDS(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES else: assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) - - if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1 , "relative": 5}}) + log_counter(file_dict, inspect.stack()[0][3]) + if METRIC_LOGGING: + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + ) def test_kernel_names(): @@ -2726,7 +2854,7 @@ def test_kernel_names(): elif file.endswith(".pdf"): file_dict[file] = "pdf" if soc == "mi200": - print(sorted(list(file_dict.keys()))) + assert sorted(list(file_dict.keys())) == [ "empirRoof_gpu-ALL_fp32.pdf", "empirRoof_gpu-ALL_int8_fp16.pdf", @@ -2743,8 +2871,10 @@ def test_kernel_names(): assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: - logger(file_dict, inspect.stack()[0][3]) + log_counter(file_dict, inspect.stack()[0][3]) if METRIC_LOGGING: - log_metric(inspect.stack()[0][3], {"default": {"absolute": 1, "relative": 5}}) - + log_metric( + inspect.stack()[0][3], + {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, + )