From 76ea35787d6348ad0cf3544cf19962af0dfcb142 Mon Sep 17 00:00:00 2001 From: abchoudh-amd Date: Wed, 19 Nov 2025 15:36:08 +0530 Subject: [PATCH] Split roofline tests, and fix none outputs (#1913) * Split roofline tests * Use N/A for missing values * Test eval_expression for no valid data * Fixed tests * Updated Changelog for N/A * Fixed platform specific test failure --- projects/rocprofiler-compute/CHANGELOG.md | 3 ++ projects/rocprofiler-compute/CMakeLists.txt | 21 +++++++--- projects/rocprofiler-compute/pyproject.toml | 3 +- .../rocprofiler-compute/src/utils/parser.py | 16 ++++---- projects/rocprofiler-compute/src/utils/tty.py | 2 +- .../tests/test_analyze_commands.py | 38 ++++++++++++++++++- .../tests/test_profile_general.py | 38 +++++++++---------- 7 files changed, 85 insertions(+), 36 deletions(-) diff --git a/projects/rocprofiler-compute/CHANGELOG.md b/projects/rocprofiler-compute/CHANGELOG.md index ec46b6848d..9b50097899 100644 --- a/projects/rocprofiler-compute/CHANGELOG.md +++ b/projects/rocprofiler-compute/CHANGELOG.md @@ -49,6 +49,9 @@ Full documentation for ROCm Compute Profiler is available at [https://rocm.docs. * `amdsmi` python API is used instead of `amd-smi` CLI to query GPU specifications. +* Empty cells replaced with `N/A` for unavailable metrics in analysis. + + ### Removed * Removed `database` mode from ROCm Compute Profiler in favor of other visualization methods, rather than Grafana and MongoDB integration, such as the upcoming Analysis DB-based Visualizer. diff --git a/projects/rocprofiler-compute/CMakeLists.txt b/projects/rocprofiler-compute/CMakeLists.txt index 6b81527e8b..25fdf218d2 100644 --- a/projects/rocprofiler-compute/CMakeLists.txt +++ b/projects/rocprofiler-compute/CMakeLists.txt @@ -287,10 +287,19 @@ add_test( ) add_test( - NAME test_profile_roofline + NAME test_profile_roofline_1 COMMAND - ${Python3_EXECUTABLE} -m pytest -m roofline - --junitxml=tests/test_profile_roofline.xml ${COV_OPTION} + ${Python3_EXECUTABLE} -m pytest -m roofline_1 + --junitxml=tests/test_profile_roofline_1.xml ${COV_OPTION} + ${PROJECT_SOURCE_DIR}/tests/test_profile_general.py + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} +) + +add_test( + NAME test_profile_roofline_2 + COMMAND + ${Python3_EXECUTABLE} -m pytest -m roofline_2 + --junitxml=tests/test_profile_roofline_2.xml ${COV_OPTION} ${PROJECT_SOURCE_DIR}/tests/test_profile_general.py WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} ) @@ -339,7 +348,8 @@ set_tests_properties( test_profile_sort test_profile_misc test_profile_path - test_profile_roofline + test_profile_roofline_1 + test_profile_roofline_2 test_profile_section test_profile_pc_sampling test_profile_sets_func @@ -445,7 +455,8 @@ if(${ENABLE_COVERAGE}) test_profile_sort test_profile_misc test_profile_path - test_profile_roofline + test_profile_roofline_1 + test_profile_roofline_2 test_profile_section test_profile_sets_func test_analyze_commands diff --git a/projects/rocprofiler-compute/pyproject.toml b/projects/rocprofiler-compute/pyproject.toml index 2f76c71478..713c67928f 100644 --- a/projects/rocprofiler-compute/pyproject.toml +++ b/projects/rocprofiler-compute/pyproject.toml @@ -103,7 +103,8 @@ markers = [ "sets_perf", "pc_sampling", "live_attach_detach", - "roofline", + "roofline_1", + "roofline_2", "path", "sci_notion", ] diff --git a/projects/rocprofiler-compute/src/utils/parser.py b/projects/rocprofiler-compute/src/utils/parser.py index 6e9499acd3..a2e67e906b 100755 --- a/projects/rocprofiler-compute/src/utils/parser.py +++ b/projects/rocprofiler-compute/src/utils/parser.py @@ -166,7 +166,7 @@ def to_avg( else: return float(a) elif isinstance(a, str): - if not a: + if not a or a == "N/A": return np.nan return float(a) else: @@ -347,29 +347,27 @@ class MetricEvaluator: ) if eval_result is None or np.isnan(eval_result).any(): - return "" + return "N/A" else: return eval_result except (TypeError, NameError, KeyError) as exception: if "empirical_peak" in str(exception): - console_warning( - f"Missing empirical peak data: {exception}. Using empty value." - ) - return "" + console_warning(f"Missing empirical peak data: {exception}.") + return "N/A" else: console_warning(f"Failed to evaluate expression '{expr}': {exception}.") - return "" + return "N/A" except AttributeError as attribute_error: if str(attribute_error) == "'NoneType' object has no attribute 'get'": console_warning( f"Failed to evaluate expression '{expr}': {attribute_error}." ) - return "" + return "N/A" else: console_error("analysis", str(attribute_error)) - return "" + return "N/A" def build_eval_string(equation: str, coll_level: str, config: dict) -> str: diff --git a/projects/rocprofiler-compute/src/utils/tty.py b/projects/rocprofiler-compute/src/utils/tty.py index 76587b2ef0..396abc13b7 100644 --- a/projects/rocprofiler-compute/src/utils/tty.py +++ b/projects/rocprofiler-compute/src/utils/tty.py @@ -351,7 +351,7 @@ def process_table_data( # Base run - just add the rounded values cur_df_copy = copy.deepcopy(cur_df) cur_df_copy[header] = [ - (round(float(x), args.decimal) if x != "" else x) + (round(float(x), args.decimal) if x != "N/A" else x) for x in base_df[header] ] result_df = pd.concat([result_df, cur_df_copy[header]], axis=1) diff --git a/projects/rocprofiler-compute/tests/test_analyze_commands.py b/projects/rocprofiler-compute/tests/test_analyze_commands.py index f9e134872e..e726848bab 100644 --- a/projects/rocprofiler-compute/tests/test_analyze_commands.py +++ b/projects/rocprofiler-compute/tests/test_analyze_commands.py @@ -26,7 +26,7 @@ import os import shutil from pathlib import Path -from unittest.mock import Mock +from unittest.mock import Mock, patch import pandas as pd import pytest @@ -1379,6 +1379,42 @@ def test_update_functions_coverage(): assert result[0].isupper() +def test_metric_evaluation_no_valid_data(): + """Test emetric evaluation with no valid data""" + import numpy as np + + from utils.parser import MetricEvaluator + + metric_evaluator = MetricEvaluator({}, {}, {}) + with patch("builtins.eval") as mock_eval, patch("builtins.compile"): + # Test when eval returns None + mock_eval.return_value = None + assert metric_evaluator.eval_expression("Mock Metric") == "N/A" + + # Test when eval returns NaN + mock_eval.return_value = np.nan + assert metric_evaluator.eval_expression("Mock Metric") == "N/A" + + # Test when eval raises an exception + mock_eval.side_effect = TypeError("Mock exception") + assert metric_evaluator.eval_expression("Mock Metric") == "N/A" + + mock_eval.side_effect = NameError("empirical_peak") + assert metric_evaluator.eval_expression("Mock Metric") == "N/A" + + mock_eval.side_effect = KeyError("Some KeyError") + assert metric_evaluator.eval_expression("Mock Metric") == "N/A" + + with patch("sys.exit"): + mock_eval.side_effect = AttributeError("Some AttributeError") + assert metric_evaluator.eval_expression("Mock Metric") == "N/A" + + mock_eval.side_effect = AttributeError( + "'NoneType' object has no attribute 'get'" + ) + assert metric_evaluator.eval_expression("Mock Metric") == "N/A" + + @pytest.fixture def sample_time_data(): return pd.DataFrame({ diff --git a/projects/rocprofiler-compute/tests/test_profile_general.py b/projects/rocprofiler-compute/tests/test_profile_general.py index 70fb8ef904..d798359066 100644 --- a/projects/rocprofiler-compute/tests/test_profile_general.py +++ b/projects/rocprofiler-compute/tests/test_profile_general.py @@ -635,7 +635,7 @@ def test_path_csv( test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roof_basic_validation(binary_handler_profile_rocprof_compute): """ Test basic roofline PDF generation with full validation pipeline. @@ -671,7 +671,7 @@ def test_roof_basic_validation(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roof_multiple_data_types(binary_handler_profile_rocprof_compute): """Test roofline with multiple data types""" if soc in ("MI100"): @@ -708,7 +708,7 @@ def test_roof_multiple_data_types(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roof_invalid_data_type(binary_handler_profile_rocprof_compute): """Test roofline with invalid data type""" if soc in ("MI100"): @@ -737,7 +737,7 @@ def test_roof_invalid_data_type(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roof_file_validation(binary_handler_profile_rocprof_compute): """Test file validation paths in roofline""" if soc in ("MI100"): @@ -766,7 +766,7 @@ def test_roof_file_validation(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roof_rocpd(binary_handler_profile_rocprof_compute): if soc == "MI100": pytest.skip("Roofline not supported on MI100") @@ -847,7 +847,7 @@ def test_analyze_rocpd( test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roofline_workload_dir_not_set_error(): """ Test roof_setup() error: "Workload directory is not set. Cannot perform setup." @@ -904,7 +904,7 @@ def test_roofline_workload_dir_not_set_error(): pytest.skip("Could not import roofline module for direct testing") -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roof_workload_dir_validation(binary_handler_profile_rocprof_compute): if soc in ("MI100"): assert True @@ -928,7 +928,7 @@ def test_roof_workload_dir_validation(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roofline_empty_kernel_names_handling(binary_handler_profile_rocprof_compute): """ Test roofline behavior when kernel filter doesn't match any @@ -968,7 +968,7 @@ def test_roofline_empty_kernel_names_handling(binary_handler_profile_rocprof_com test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roofline_kernel_filter(binary_handler_profile_rocprof_compute): """ Test roofline multi-attempt profiling with `--kernel` @@ -1006,7 +1006,7 @@ def test_roofline_kernel_filter(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_1 def test_roofline_unsupported_datatype_error(binary_handler_profile_rocprof_compute): """ Test datatype validation error in empirical_roofline() @@ -1032,7 +1032,7 @@ def test_roofline_unsupported_datatype_error(binary_handler_profile_rocprof_comp test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roof_plot_modes(binary_handler_profile_rocprof_compute): if soc in ("MI100"): assert True @@ -1078,7 +1078,7 @@ def test_roof_plot_modes(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roof_cli_plot_generation(binary_handler_profile_rocprof_compute): if soc in ("MI100"): assert True @@ -1104,7 +1104,7 @@ def test_roof_cli_plot_generation(binary_handler_profile_rocprof_compute): pytest.skip("plotext not available for CLI testing") -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roof_error_handling(binary_handler_profile_rocprof_compute): if soc in ("MI100"): assert True @@ -1124,7 +1124,7 @@ def test_roof_error_handling(binary_handler_profile_rocprof_compute): test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roofline_missing_file_handling(binary_handler_profile_rocprof_compute): """ Test handling of missing roofline.csv file @@ -1176,7 +1176,7 @@ def test_roofline_missing_file_handling(binary_handler_profile_rocprof_compute): pytest.skip("Could not import roofline module for direct testing") -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roofline_invalid_datatype_cli(binary_handler_profile_rocprof_compute): """ Test CLI plot generation with invalid datatype @@ -1226,7 +1226,7 @@ def test_roofline_invalid_datatype_cli(binary_handler_profile_rocprof_compute): pytest.skip("Could not import roofline module for direct testing") -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roofline_ceiling_data_validation(binary_handler_profile_rocprof_compute): """ Test ceiling data validation in generate_plot() @@ -1246,7 +1246,7 @@ def test_roofline_ceiling_data_validation(binary_handler_profile_rocprof_compute test_utils.clean_output_dir(config["cleanup"], workload_dir) -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roofline_plot_points_data_generation(): """ Test that plot points data structure is correctly generated with: @@ -1346,7 +1346,7 @@ def test_roofline_plot_points_data_generation(): pytest.skip("Could not import roofline module for direct testing") -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roofline_bound_status_calculation(): """ Test _determine_kernel_bound_status() correctly classifies kernels as @@ -1418,7 +1418,7 @@ def test_roofline_bound_status_calculation(): pytest.skip("Could not import roofline module for direct testing") -@pytest.mark.roofline +@pytest.mark.roofline_2 def test_roofline_many_kernels_dynamic_height(binary_handler_profile_rocprof_compute): """ Test roofline PDF generation with many kernels (10+) to verify: