Split roofline tests, and fix none outputs (#1913)

* Split roofline tests

* Use N/A for missing values

* Test eval_expression for no valid data

* Fixed tests

* Updated Changelog for N/A

* Fixed platform specific test failure

projects/rocprofiler-compute/CHANGELOG.md

@@ -49,6 +49,9 @@ Full documentation for ROCm Compute Profiler is available at [https://rocm.docs.
 
 * `amdsmi` python API is used instead of `amd-smi` CLI to query GPU specifications.
 
+* Empty cells replaced with `N/A` for unavailable metrics in analysis.
+
+
 ### Removed
 
 * Removed `database` mode from ROCm Compute Profiler in favor of other visualization methods, rather than Grafana and MongoDB integration, such as the upcoming Analysis DB-based Visualizer.

projects/rocprofiler-compute/CMakeLists.txt

@@ -287,10 +287,19 @@ add_test(
 )
 
 add_test(
-    NAME test_profile_roofline
+    NAME test_profile_roofline_1
     COMMAND
-        ${Python3_EXECUTABLE} -m pytest -m roofline
+        ${Python3_EXECUTABLE} -m pytest -m roofline_1
-        --junitxml=tests/test_profile_roofline.xml ${COV_OPTION}
+        --junitxml=tests/test_profile_roofline_1.xml ${COV_OPTION}
+        ${PROJECT_SOURCE_DIR}/tests/test_profile_general.py
+    WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+)
+
+add_test(
+    NAME test_profile_roofline_2
+    COMMAND
+        ${Python3_EXECUTABLE} -m pytest -m roofline_2
+        --junitxml=tests/test_profile_roofline_2.xml ${COV_OPTION}
         ${PROJECT_SOURCE_DIR}/tests/test_profile_general.py
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
 )
@@ -339,7 +348,8 @@ set_tests_properties(
     test_profile_sort
     test_profile_misc
     test_profile_path
-    test_profile_roofline
+    test_profile_roofline_1
+    test_profile_roofline_2
     test_profile_section
     test_profile_pc_sampling
     test_profile_sets_func
@@ -445,7 +455,8 @@ if(${ENABLE_COVERAGE})
         test_profile_sort
         test_profile_misc
         test_profile_path
-        test_profile_roofline
+        test_profile_roofline_1
+        test_profile_roofline_2
         test_profile_section
         test_profile_sets_func
         test_analyze_commands

projects/rocprofiler-compute/pyproject.toml

@@ -103,7 +103,8 @@ markers = [
     "sets_perf",
     "pc_sampling",
     "live_attach_detach",
-    "roofline",
+    "roofline_1",
+    "roofline_2",
     "path",
     "sci_notion",
 ]

projects/rocprofiler-compute/src/utils/parser.py

@@ -166,7 +166,7 @@ def to_avg(
         else:
             return float(a)
     elif isinstance(a, str):
-        if not a:
+        if not a or a == "N/A":
             return np.nan
         return float(a)
     else:
@@ -347,29 +347,27 @@ class MetricEvaluator:
             )
 
             if eval_result is None or np.isnan(eval_result).any():
-                return ""
+                return "N/A"
             else:
                 return eval_result
 
         except (TypeError, NameError, KeyError) as exception:
             if "empirical_peak" in str(exception):
-                console_warning(
+                console_warning(f"Missing empirical peak data: {exception}.")
-                    f"Missing empirical peak data: {exception}. Using empty value."
+                return "N/A"
-                )
-                return ""
             else:
                 console_warning(f"Failed to evaluate expression '{expr}': {exception}.")
-                return ""
+                return "N/A"
 
         except AttributeError as attribute_error:
             if str(attribute_error) == "'NoneType' object has no attribute 'get'":
                 console_warning(
                     f"Failed to evaluate expression '{expr}': {attribute_error}."
                 )
-                return ""
+                return "N/A"
             else:
                 console_error("analysis", str(attribute_error))
-                return ""
+                return "N/A"
 
 
 def build_eval_string(equation: str, coll_level: str, config: dict) -> str:

projects/rocprofiler-compute/src/utils/tty.py

@@ -351,7 +351,7 @@ def process_table_data(
                         # Base run - just add the rounded values
                         cur_df_copy = copy.deepcopy(cur_df)
                         cur_df_copy[header] = [
-                            (round(float(x), args.decimal) if x != "" else x)
+                            (round(float(x), args.decimal) if x != "N/A" else x)
                             for x in base_df[header]
                         ]
                         result_df = pd.concat([result_df, cur_df_copy[header]], axis=1)

projects/rocprofiler-compute/tests/test_analyze_commands.py

@@ -26,7 +26,7 @@
 import os
 import shutil
 from pathlib import Path
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
 
 import pandas as pd
 import pytest
@@ -1379,6 +1379,42 @@ def test_update_functions_coverage():
     assert result[0].isupper()
 
 
+def test_metric_evaluation_no_valid_data():
+    """Test emetric evaluation with no valid data"""
+    import numpy as np
+
+    from utils.parser import MetricEvaluator
+
+    metric_evaluator = MetricEvaluator({}, {}, {})
+    with patch("builtins.eval") as mock_eval, patch("builtins.compile"):
+        # Test when eval returns None
+        mock_eval.return_value = None
+        assert metric_evaluator.eval_expression("Mock Metric") == "N/A"
+
+        # Test when eval returns NaN
+        mock_eval.return_value = np.nan
+        assert metric_evaluator.eval_expression("Mock Metric") == "N/A"
+
+        # Test when eval raises an exception
+        mock_eval.side_effect = TypeError("Mock exception")
+        assert metric_evaluator.eval_expression("Mock Metric") == "N/A"
+
+        mock_eval.side_effect = NameError("empirical_peak")
+        assert metric_evaluator.eval_expression("Mock Metric") == "N/A"
+
+        mock_eval.side_effect = KeyError("Some KeyError")
+        assert metric_evaluator.eval_expression("Mock Metric") == "N/A"
+
+        with patch("sys.exit"):
+            mock_eval.side_effect = AttributeError("Some AttributeError")
+            assert metric_evaluator.eval_expression("Mock Metric") == "N/A"
+
+        mock_eval.side_effect = AttributeError(
+            "'NoneType' object has no attribute 'get'"
+        )
+        assert metric_evaluator.eval_expression("Mock Metric") == "N/A"
+
+
 @pytest.fixture
 def sample_time_data():
     return pd.DataFrame({

projects/rocprofiler-compute/tests/test_profile_general.py

@@ -635,7 +635,7 @@ def test_path_csv(
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roof_basic_validation(binary_handler_profile_rocprof_compute):
     """
     Test basic roofline PDF generation with full validation pipeline.
@@ -671,7 +671,7 @@ def test_roof_basic_validation(binary_handler_profile_rocprof_compute):
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roof_multiple_data_types(binary_handler_profile_rocprof_compute):
     """Test roofline with multiple data types"""
     if soc in ("MI100"):
@@ -708,7 +708,7 @@ def test_roof_multiple_data_types(binary_handler_profile_rocprof_compute):
             test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roof_invalid_data_type(binary_handler_profile_rocprof_compute):
     """Test roofline with invalid data type"""
     if soc in ("MI100"):
@@ -737,7 +737,7 @@ def test_roof_invalid_data_type(binary_handler_profile_rocprof_compute):
         test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roof_file_validation(binary_handler_profile_rocprof_compute):
     """Test file validation paths in roofline"""
     if soc in ("MI100"):
@@ -766,7 +766,7 @@ def test_roof_file_validation(binary_handler_profile_rocprof_compute):
         test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roof_rocpd(binary_handler_profile_rocprof_compute):
     if soc == "MI100":
         pytest.skip("Roofline not supported on MI100")
@@ -847,7 +847,7 @@ def test_analyze_rocpd(
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roofline_workload_dir_not_set_error():
     """
     Test roof_setup() error: "Workload directory is not set. Cannot perform setup."
@@ -904,7 +904,7 @@ def test_roofline_workload_dir_not_set_error():
         pytest.skip("Could not import roofline module for direct testing")
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roof_workload_dir_validation(binary_handler_profile_rocprof_compute):
     if soc in ("MI100"):
         assert True
@@ -928,7 +928,7 @@ def test_roof_workload_dir_validation(binary_handler_profile_rocprof_compute):
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roofline_empty_kernel_names_handling(binary_handler_profile_rocprof_compute):
     """
     Test roofline behavior when kernel filter doesn't match any
@@ -968,7 +968,7 @@ def test_roofline_empty_kernel_names_handling(binary_handler_profile_rocprof_com
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roofline_kernel_filter(binary_handler_profile_rocprof_compute):
     """
     Test roofline multi-attempt profiling with `--kernel`
@@ -1006,7 +1006,7 @@ def test_roofline_kernel_filter(binary_handler_profile_rocprof_compute):
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_1
 def test_roofline_unsupported_datatype_error(binary_handler_profile_rocprof_compute):
     """
     Test datatype validation error in empirical_roofline()
@@ -1032,7 +1032,7 @@ def test_roofline_unsupported_datatype_error(binary_handler_profile_rocprof_comp
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roof_plot_modes(binary_handler_profile_rocprof_compute):
     if soc in ("MI100"):
         assert True
@@ -1078,7 +1078,7 @@ def test_roof_plot_modes(binary_handler_profile_rocprof_compute):
         test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roof_cli_plot_generation(binary_handler_profile_rocprof_compute):
     if soc in ("MI100"):
         assert True
@@ -1104,7 +1104,7 @@ def test_roof_cli_plot_generation(binary_handler_profile_rocprof_compute):
         pytest.skip("plotext not available for CLI testing")
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roof_error_handling(binary_handler_profile_rocprof_compute):
     if soc in ("MI100"):
         assert True
@@ -1124,7 +1124,7 @@ def test_roof_error_handling(binary_handler_profile_rocprof_compute):
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roofline_missing_file_handling(binary_handler_profile_rocprof_compute):
     """
     Test handling of missing roofline.csv file
@@ -1176,7 +1176,7 @@ def test_roofline_missing_file_handling(binary_handler_profile_rocprof_compute):
         pytest.skip("Could not import roofline module for direct testing")
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roofline_invalid_datatype_cli(binary_handler_profile_rocprof_compute):
     """
     Test CLI plot generation with invalid datatype
@@ -1226,7 +1226,7 @@ def test_roofline_invalid_datatype_cli(binary_handler_profile_rocprof_compute):
         pytest.skip("Could not import roofline module for direct testing")
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roofline_ceiling_data_validation(binary_handler_profile_rocprof_compute):
     """
     Test ceiling data validation in generate_plot()
@@ -1246,7 +1246,7 @@ def test_roofline_ceiling_data_validation(binary_handler_profile_rocprof_compute
     test_utils.clean_output_dir(config["cleanup"], workload_dir)
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roofline_plot_points_data_generation():
     """
     Test that plot points data structure is correctly generated with:
@@ -1346,7 +1346,7 @@ def test_roofline_plot_points_data_generation():
         pytest.skip("Could not import roofline module for direct testing")
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roofline_bound_status_calculation():
     """
     Test _determine_kernel_bound_status() correctly classifies kernels as
@@ -1418,7 +1418,7 @@ def test_roofline_bound_status_calculation():
         pytest.skip("Could not import roofline module for direct testing")
 
 
-@pytest.mark.roofline
+@pytest.mark.roofline_2
 def test_roofline_many_kernels_dynamic_height(binary_handler_profile_rocprof_compute):
     """
     Test roofline PDF generation with many kernels (10+) to verify: