diff --git a/projects/rocprofiler-compute/tests/test_profile_general.py b/projects/rocprofiler-compute/tests/test_profile_general.py index 6613adddce..5eef82e052 100644 --- a/projects/rocprofiler-compute/tests/test_profile_general.py +++ b/projects/rocprofiler-compute/tests/test_profile_general.py @@ -17,6 +17,7 @@ kernel_name_1 = "vecCopy(double*, double*, double*, int, int) [clone .kd]" app_1 = ["./tests/vcopy", "-n", "1048576", "-b", "256", "-i", "3"] #app_1 = ["./sample/vcopy", "-n", "1048576", "-b", "256", "-i", "3"] baseline_opts = ["omniperf", "profile", "-n", "app_1", "-VVV"] +cleanup = True # whether to clean up profile data after tests or not num_kernels = 3 dispatch_id = 0 @@ -427,6 +428,8 @@ def test_path(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.misc def test_no_roof(): @@ -440,16 +443,8 @@ def test_no_roof(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": print(sorted(list(file_dict.keys()))) assert sorted(list(file_dict.keys())) == [ @@ -491,6 +486,8 @@ def test_no_roof(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.misc def test_kernel_names(): @@ -511,18 +508,8 @@ def test_kernel_names(): return # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": print(sorted(list(file_dict.keys()))) assert sorted(list(file_dict.keys())) == [ @@ -549,12 +536,15 @@ def test_kernel_names(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + + @pytest.mark.misc def test_device_filter(): workload_dir = test_utils.get_output_dir() - device_id="0" + device_id = "0" if "HIP_VISIBLE_DEVICES" in os.environ: - device_id=os.environ["HIP_VISIBLE_DEVICES"] + device_id = os.environ["HIP_VISIBLE_DEVICES"] with pytest.raises(SystemExit) as e: with patch( "sys.argv", @@ -564,26 +554,15 @@ def test_device_filter(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - if "roofline" in file: - assert len(file_dict[file].index) - else: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": print(sorted(list(file_dict.keys()))) assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: assert sorted(list(file_dict.keys())) == ALL_CSVS - #TODO - verify expected device id in results + # TODO - verify expected device id in results if COUNTER_LOGGING: log_counter(file_dict, inspect.stack()[0][3]) @@ -594,6 +573,9 @@ def test_device_filter(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + + @pytest.mark.kernel_execution def test_kernel(): workload_dir = test_utils.get_output_dir() @@ -608,16 +590,8 @@ def test_kernel(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -633,6 +607,8 @@ def test_kernel(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.kernel_execution def test_kernel_summaries(): @@ -648,16 +624,8 @@ def test_kernel_summaries(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -673,6 +641,8 @@ def test_kernel_summaries(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_SQ(): @@ -686,17 +656,8 @@ def test_ipblocks_SQ(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - expected_csvs = [ "SQ_IFETCH_LEVEL.csv", "SQ_INST_LEVEL_LDS.csv", @@ -752,6 +713,7 @@ def test_ipblocks_SQ(): inspect.stack()[0][3], {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) @pytest.mark.ipblocks @@ -766,17 +728,8 @@ def test_ipblocks_SQC(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - expected_csvs = [ "pmc_perf.csv", "pmc_perf_0.csv", @@ -800,6 +753,8 @@ def test_ipblocks_SQC(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_TA(): @@ -813,16 +768,7 @@ def test_ipblocks_TA(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) @@ -853,6 +799,8 @@ def test_ipblocks_TA(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_TD(): @@ -866,16 +814,7 @@ def test_ipblocks_TD(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) @@ -910,6 +849,8 @@ def test_ipblocks_TD(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_TCP(): @@ -923,17 +864,7 @@ def test_ipblocks_TCP(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ "pmc_perf.csv", @@ -964,6 +895,8 @@ def test_ipblocks_TCP(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_TCC(): @@ -977,16 +910,7 @@ def test_ipblocks_TCC(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ @@ -1019,6 +943,8 @@ def test_ipblocks_TCC(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_SPI(): @@ -1032,17 +958,7 @@ def test_ipblocks_SPI(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ "pmc_perf.csv", @@ -1072,6 +988,8 @@ def test_ipblocks_SPI(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_CPC(): @@ -1085,17 +1003,8 @@ def test_ipblocks_CPC(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - expected_csvs = [ "pmc_perf.csv", "pmc_perf_0.csv", @@ -1120,6 +1029,8 @@ def test_ipblocks_CPC(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_CPF(): @@ -1133,17 +1044,9 @@ def test_ipblocks_CPF(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - + print(sorted(list(file_dict.keys()))) expected_csvs = [ "pmc_perf.csv", "pmc_perf_0.csv", @@ -1166,6 +1069,8 @@ def test_ipblocks_CPF(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_SQ_CPC(): @@ -1181,16 +1086,7 @@ def test_ipblocks_SQ_CPC(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ @@ -1249,6 +1145,8 @@ def test_ipblocks_SQ_CPC(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_SQ_TA(): @@ -1264,16 +1162,7 @@ def test_ipblocks_SQ_TA(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ @@ -1331,6 +1220,8 @@ def test_ipblocks_SQ_TA(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_SQ_SPI(): @@ -1346,16 +1237,7 @@ def test_ipblocks_SQ_SPI(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ @@ -1413,6 +1295,8 @@ def test_ipblocks_SQ_SPI(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_SQ_SQC_TCP_CPC(): @@ -1428,16 +1312,7 @@ def test_ipblocks_SQ_SQC_TCP_CPC(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ @@ -1496,6 +1371,8 @@ def test_ipblocks_SQ_SQC_TCP_CPC(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.ipblocks def test_ipblocks_SQ_SPI_TA_TCC_CPF(): @@ -1521,16 +1398,7 @@ def test_ipblocks_SQ_SPI_TA_TCC_CPF(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) expected_csvs = [ @@ -1590,6 +1458,8 @@ def test_ipblocks_SQ_SPI_TA_TCC_CPF(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.dispatch def test_dispatch_0(): @@ -1603,16 +1473,8 @@ def test_dispatch_0(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, 1) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file and not "roofline" in file: - assert len(file_dict[file].index) == 1 if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1632,6 +1494,8 @@ def test_dispatch_0(): ], ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.dispatch def test_dispatch_0_1(): @@ -1645,21 +1509,13 @@ def test_dispatch_0_1(): # assert successful run assert e.value.code == 0 - - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file and not "roofline" in file: - assert len(file_dict[file].index) == 2 + file_dict = test_utils.check_csv_files(workload_dir, 2) if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 else: + print(sorted(list(file_dict.keys()))) assert sorted(list(file_dict.keys())) == ALL_CSVS if COUNTER_LOGGING: @@ -1672,6 +1528,8 @@ def test_dispatch_0_1(): ["--dispatch", "0", "1"], ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.dispatch def test_dispatch_2(): @@ -1687,16 +1545,8 @@ def test_dispatch_2(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, 1) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file and not "roofline" in file: - assert len(file_dict[file].index) == 1 if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1716,6 +1566,8 @@ def test_dispatch_2(): ], ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.verbosity def test_kernel_verbose_0(): @@ -1731,16 +1583,8 @@ def test_kernel_verbose_0(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1756,6 +1600,8 @@ def test_kernel_verbose_0(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.verbosity def test_kernel_verbose_1(): @@ -1771,16 +1617,8 @@ def test_kernel_verbose_1(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1796,6 +1634,8 @@ def test_kernel_verbose_1(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.verbosity def test_kernel_verbose_2(): @@ -1811,16 +1651,8 @@ def test_kernel_verbose_2(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1836,6 +1668,8 @@ def test_kernel_verbose_2(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.verbosity def test_kernel_verbose_3(): @@ -1851,16 +1685,8 @@ def test_kernel_verbose_3(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1876,6 +1702,8 @@ def test_kernel_verbose_3(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.verbosity def test_kernel_verbose_4(): @@ -1891,16 +1719,8 @@ def test_kernel_verbose_4(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1916,6 +1736,8 @@ def test_kernel_verbose_4(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.verbosity def test_kernel_verbose_5(): @@ -1931,16 +1753,8 @@ def test_kernel_verbose_5(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1956,6 +1770,8 @@ def test_kernel_verbose_5(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.join def test_join_type_grid(): @@ -1969,16 +1785,8 @@ def test_join_type_grid(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -1994,6 +1802,8 @@ def test_join_type_grid(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.join def test_join_type_kernel(): @@ -2009,16 +1819,8 @@ def test_join_type_kernel(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels if soc == "mi200": assert sorted(list(file_dict.keys())) == ALL_CSVS_MI200 @@ -2034,6 +1836,8 @@ def test_join_type_kernel(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.sort def test_sort_dispatches(): @@ -2056,17 +1860,6 @@ def test_sort_dispatches(): # assert successful run assert e.value.code == 0 - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2082,6 +1875,8 @@ def test_sort_dispatches(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.sort def test_sort_kernels(): @@ -2102,18 +1897,8 @@ def test_sort_kernels(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - file_dict = {} - # Check if csvs have data - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2129,6 +1914,8 @@ def test_sort_kernels(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.mem def test_mem_levels_HBM(): @@ -2150,18 +1937,8 @@ def test_mem_levels_HBM(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2177,6 +1954,8 @@ def test_mem_levels_HBM(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.mem def test_mem_levels_L2(): @@ -2198,18 +1977,8 @@ def test_mem_levels_L2(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2225,6 +1994,8 @@ def test_mem_levels_L2(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.mem def test_mem_levels_vL1D(): @@ -2245,18 +2016,8 @@ def test_mem_levels_vL1D(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2272,6 +2033,8 @@ def test_mem_levels_vL1D(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.mem def test_mem_levels_LDS(): @@ -2292,18 +2055,8 @@ def test_mem_levels_LDS(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2319,6 +2072,8 @@ def test_mem_levels_LDS(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.mem def test_mem_levels_HBM_LDS(): @@ -2339,18 +2094,8 @@ def test_mem_levels_HBM_LDS(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2366,6 +2111,8 @@ def test_mem_levels_HBM_LDS(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.mem def test_mem_levels_vL1D_LDS(): @@ -2386,18 +2133,8 @@ def test_mem_levels_vL1D_LDS(): # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2413,6 +2150,8 @@ def test_mem_levels_vL1D_LDS(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) + test_utils.clean_output_dir(cleanup, workload_dir) + @pytest.mark.mem def test_mem_levels_L2_vL1D_LDS(): @@ -2441,18 +2180,8 @@ def test_mem_levels_L2_vL1D_LDS(): return # assert successful run assert e.value.code == 0 + file_dict = test_utils.check_csv_files(workload_dir, num_kernels) - files_in_workload = os.listdir(workload_dir) - - # Check if csvs have data - file_dict = {} - for file in files_in_workload: - if file.endswith(".csv"): - file_dict[file] = pd.read_csv(workload_dir + "/" + file) - if not "sysinfo" in file: - assert len(file_dict[file].index) >= num_kernels - elif file.endswith(".pdf"): - file_dict[file] = "pdf" if soc == "mi200": assert sorted(list(file_dict.keys())) == ROOF_ONLY_FILES @@ -2468,70 +2197,4 @@ def test_mem_levels_L2_vL1D_LDS(): {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, ) - -# def test_kernel_names(): -# if os.path.exists(workload_1): -# shutil.rmtree(workload_1) -# with pytest.raises(SystemExit) as e: -# with patch( -# "sys.argv", -# [ -# "omniperf", -# "profile", -# "-n", -# "app_1", -# "-VVV", -# "--path", -# workload_1, -# "--roof-only", -# "--kernel-names", -# "--", -# ] -# + app_1, -# ): -# omniperf.main() - -# if soc == "mi100": -# # assert that it did not run -# assert e.value.code >= 1 -# # Do not continue testing -# return -# # assert successful run -# assert e.value.code == 0 - -# files_in_workload = os.listdir(workload_1) - -# # Check if csvs have data -# file_dict = {} -# for file in files_in_workload: -# if file.endswith(".csv"): -# file_dict[file] = pd.read_csv(workload_1 + "/" + file) -# if not "sysinfo" in file: -# assert len(file_dict[file].index) >= num_kernels -# elif file.endswith(".pdf"): -# file_dict[file] = "pdf" -# if soc == "mi200": - -# assert sorted(list(file_dict.keys())) == [ -# "empirRoof_gpu-ALL_fp32.pdf", -# "empirRoof_gpu-ALL_int8_fp16.pdf", -# "kernelName_legend.pdf", -# "pmc_perf.csv", -# "pmc_perf_0.csv", -# "pmc_perf_1.csv", -# "pmc_perf_2.csv", -# "roofline.csv", -# "sysinfo.csv", -# "timestamps.csv", -# ] -# else: -# assert sorted(list(file_dict.keys())) == ALL_CSVS - -# if COUNTER_LOGGING: -# log_counter(file_dict, inspect.stack()[0][3]) - -# if METRIC_LOGGING: -# log_metric( -# inspect.stack()[0][3], -# {"default": {"absolute": DEFAULT_ABS_DIFF, "relative": DEFAULT_REL_DIFF}}, -# ) + test_utils.clean_output_dir(cleanup, workload_dir) diff --git a/projects/rocprofiler-compute/tests/test_utils.py b/projects/rocprofiler-compute/tests/test_utils.py index 95561cf18c..14e56f7540 100644 --- a/projects/rocprofiler-compute/tests/test_utils.py +++ b/projects/rocprofiler-compute/tests/test_utils.py @@ -23,10 +23,10 @@ ##############################################################################el # Common helper routines for testing collateral - import inspect import os import shutil +import pandas as pd def check_resource_allocation(): @@ -62,3 +62,39 @@ def get_output_dir(suffix="_output", clean_existing=True): if os.path.exists(output_dir): shutil.rmtree(output_dir) return output_dir + + +def clean_output_dir(cleanup, output_dir): + """Remove output directory generated from omniperf execution + + Args: + cleanup (boolean): flag to enable/disable directory cleanup + output_dir (string): name of directory to remove + """ + if cleanup: + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + return + + +def check_csv_files(output_dir, num_kernels): + """Check profiling output csv files for expected number of entries (based on kernel invocations) + + Args: + output_dir (string): output directory containing csv files + num_kernels (int): number of kernels expected to have been profiled + + Returns: + dict: dictionary housing file contents as pandas dataframe + """ + + file_dict = {} + files_in_workload = os.listdir(output_dir) + for file in files_in_workload: + if file.endswith(".csv"): + file_dict[file] = pd.read_csv(output_dir + "/" + file) + if not "sysinfo" in file: + assert len(file_dict[file].index) >= num_kernels + elif file.endswith(".pdf"): + file_dict[file] = "pdf" + return file_dict