diff --git a/src/rocprof_compute_soc/soc_gfx908.py b/src/rocprof_compute_soc/soc_gfx908.py index b4cdfe42f4..bcaac75bd3 100644 --- a/src/rocprof_compute_soc/soc_gfx908.py +++ b/src/rocprof_compute_soc/soc_gfx908.py @@ -34,9 +34,7 @@ class gfx908_soc(OmniSoC_Base): def __init__(self, args, mspec): super().__init__(args, mspec) self.set_arch("gfx908") - self.set_compatible_profilers( - ["rocprofv1"] - ) + self.set_compatible_profilers(["rocprofv1"]) # Per IP block max number of simultaneous counters. GFX IP Blocks self.set_perfmon_config(mi_gpu_specs.get_perfmon_config("gfx908")) diff --git a/src/roofline.py b/src/roofline.py index 3b260f9e39..beb3760ccc 100644 --- a/src/roofline.py +++ b/src/roofline.py @@ -53,15 +53,19 @@ from utils.utils import mibench SYMBOLS = [0, 1, 2, 3, 4, 5, 13, 17, 18, 20] + def wrap_text(text, width=92): """ Wraps text using textwrap and joins lines with
for Plotly. """ if not isinstance(text, str): text = str(text) - wrapped_lines = textwrap.wrap(text, width=width, break_long_words=True, replace_whitespace=False) + wrapped_lines = textwrap.wrap( + text, width=width, break_long_words=True, replace_whitespace=False + ) return "
".join(wrapped_lines) + class Roofline: def __init__(self, args, mspec, run_parameters=None): self.__args = args @@ -76,7 +80,7 @@ class Roofline: "mem_level": "ALL", "include_kernel_names": False, "is_standalone": False, - "roofline_data_type": ["FP32"] # default to FP32 + "roofline_data_type": ["FP32"], # default to FP32 } ) self.__ai_data = None @@ -105,7 +109,11 @@ class Roofline: def roof_setup(self): # Setup the workload directory for roofline profiling. workload_dir_val = self.__run_parameters.get("workload_dir") - if workload_dir_val and Path(workload_dir_val).name == "workloads" and Path(workload_dir_val).parent == Path(os.getcwd()): + if ( + workload_dir_val + and Path(workload_dir_val).name == "workloads" + and Path(workload_dir_val).parent == Path(os.getcwd()) + ): app_name = getattr(self.__args, "name", "default_app_name") gpu_model_name = getattr(self.__mspec, "gpu_model", "default_gpu_model") self.__run_parameters["workload_dir"] = str( @@ -119,7 +127,9 @@ class Roofline: if current_workload_dir: Path(current_workload_dir).mkdir(parents=True, exist_ok=True) else: - console_error("Workload directory is not set. Cannot perform setup.", exit=False) + console_error( + "Workload directory is not set. Cannot perform setup.", exit=False + ) @demarcate def empirical_roofline( @@ -134,7 +144,9 @@ class Roofline: self.roof_setup() console_debug("roofline", "Path: %s" % self.__run_parameters.get("workload_dir")) - self.__ai_data = calc_ai(self.__mspec, self.__run_parameters.get("sort_type"), ret_df) + self.__ai_data = calc_ai( + self.__mspec, self.__run_parameters.get("sort_type"), ret_df + ) msg = "AI at each mem level:" for i in self.__ai_data: @@ -146,7 +158,11 @@ class Roofline: for dt in self.__run_parameters.get("roofline_data_type", []): gpu_arch = getattr(self.__mspec, "gpu_arch", "unknown_arch") - if 'SUPPORTED_DATATYPES' not in globals() or gpu_arch not in SUPPORTED_DATATYPES or str(dt) not in SUPPORTED_DATATYPES[gpu_arch]: + if ( + "SUPPORTED_DATATYPES" not in globals() + or gpu_arch not in SUPPORTED_DATATYPES + or str(dt) not in SUPPORTED_DATATYPES[gpu_arch] + ): console_error( "{} is not a supported datatype for roofline profiling on {} (arch: {})".format( str(dt), getattr(self.__mspec, "gpu_model", "N/A"), gpu_arch @@ -180,7 +196,10 @@ class Roofline: if self.__run_parameters.get("include_kernel_names", False): if self.__ai_data is None: - console_error("Roofline Error: self.__ai_data is not populated. Cannot generate kernel names info.", exit=False) + console_error( + "Roofline Error: self.__ai_data is not populated. Cannot generate kernel names info.", + exit=False, + ) original_kernel_names = [] else: original_kernel_names = self.__ai_data.get("kernelNames", []) @@ -191,14 +210,27 @@ class Roofline: self.__figure.layout = {} if num_kernels == 0: - console_log("roofline", "No kernel names found to generate 'Kernel Names and Markers' info.") - self.__figure.add_annotation(text="No kernel names to display.", - showarrow=False, xref="paper", yref="paper", x=0.5, y=0.5) + console_log( + "roofline", + "No kernel names found to generate 'Kernel Names and Markers' info.", + ) + self.__figure.add_annotation( + text="No kernel names to display.", + showarrow=False, + xref="paper", + yref="paper", + x=0.5, + y=0.5, + ) self.__figure.update_layout( - title_text="Kernel Names and Markers", title_x=0.5, - xaxis=dict(visible=False), yaxis=dict(visible=False), - plot_bgcolor='white', paper_bgcolor='white', - height=200, width=400 + title_text="Kernel Names and Markers", + title_x=0.5, + xaxis=dict(visible=False), + yaxis=dict(visible=False), + plot_bgcolor="white", + paper_bgcolor="white", + height=200, + width=400, ) else: symbols_list = [] @@ -210,19 +242,21 @@ class Roofline: self.__figure = go.Figure() - self.__figure.add_trace(go.Scatter( - x=[0.1] * num_kernels, - y=list(range(num_kernels, 0, -1)), - mode='markers', - marker=dict( - symbol=symbols_list, - size=15, - color='blue', - line=dict(width=1, color='black') - ), - showlegend=False, - hoverinfo='skip' - )) + self.__figure.add_trace( + go.Scatter( + x=[0.1] * num_kernels, + y=list(range(num_kernels, 0, -1)), + mode="markers", + marker=dict( + symbol=symbols_list, + size=15, + color="blue", + line=dict(width=1, color="black"), + ), + showlegend=False, + hoverinfo="skip", + ) + ) for i, kernel_name in enumerate(kernel_names_list): self.__figure.add_annotation( @@ -230,64 +264,64 @@ class Roofline: y=num_kernels - i, text=wrap_text(kernel_name), showarrow=False, - xanchor='left', - yanchor='middle', - align='left', - font=dict(size=11, color='black') + xanchor="left", + yanchor="middle", + align="left", + font=dict(size=11, color="black"), ) self.__figure.add_annotation( - x=0.1, y=num_kernels + 1, + x=0.1, + y=num_kernels + 1, text="Symbol", showarrow=False, - xanchor='center', - yanchor='middle', - font=dict(size=12, color='black') + xanchor="center", + yanchor="middle", + font=dict(size=12, color="black"), ) self.__figure.add_annotation( - x=0.25, y=num_kernels + 1, + x=0.25, + y=num_kernels + 1, text="Kernel Name", showarrow=False, - xanchor='left', - yanchor='middle', - font=dict(size=12, color='black') + xanchor="left", + yanchor="middle", + font=dict(size=12, color="black"), ) for i in range(num_kernels + 1): self.__figure.add_shape( type="line", - x0=0, x1=1, - y0=i + 0.5, y1=i + 0.5, - line=dict(color="lightgray", width=1) + x0=0, + x1=1, + y0=i + 0.5, + y1=i + 0.5, + line=dict(color="lightgray", width=1), ) self.__figure.add_shape( type="line", - x0=0.2, x1=0.2, - y0=0.5, y1=num_kernels + 1.5, - line=dict(color="lightgray", width=1) + x0=0.2, + x1=0.2, + y0=0.5, + y1=num_kernels + 1.5, + line=dict(color="lightgray", width=1), ) self.__figure.update_layout( title="Kernel Names and Corresponding Markers", title_x=0.5, - xaxis=dict( - visible=False, - range=[0, 1] - ), + xaxis=dict(visible=False, range=[0, 1]), yaxis=dict( - visible=False, - range=[0, num_kernels + 2], - autorange=False + visible=False, range=[0, num_kernels + 2], autorange=False ), height=max(400, num_kernels * 40 + 150), width=1000, margin=dict(l=50, r=50, t=70, b=30), - plot_bgcolor='white', - paper_bgcolor='white' + plot_bgcolor="white", + paper_bgcolor="white", ) - # Output will be different depending on interaction type: # Save PDFs if we're in "standalone roofline" mode, otherwise return HTML to be used in GUI output if self.__run_parameters["is_standalone"]: @@ -440,15 +474,25 @@ class Roofline: if mem_level_config == "ALL": cache_hierarchy = ["HBM", "L2", "L1", "LDS"] else: - cache_hierarchy = mem_level_config if isinstance(mem_level_config, list) else [mem_level_config] + cache_hierarchy = ( + mem_level_config + if isinstance(mem_level_config, list) + else [mem_level_config] + ) # Plot peak BW ceiling(s) for cache_level in cache_hierarchy: - if (not self.__ceiling_data or cache_level.lower() not in self.__ceiling_data or - not isinstance(self.__ceiling_data[cache_level.lower()], (list, tuple)) or - len(self.__ceiling_data[cache_level.lower()]) < 3): - console_error(f"Ceiling data for {cache_level} is missing or malformed for dtype {dtype}.", exit=False) + if ( + not self.__ceiling_data + or cache_level.lower() not in self.__ceiling_data + or not isinstance(self.__ceiling_data[cache_level.lower()], (list, tuple)) + or len(self.__ceiling_data[cache_level.lower()]) < 3 + ): + console_error( + f"Ceiling data for {cache_level} is missing or malformed for dtype {dtype}.", + exit=False, + ) continue fig.add_trace( diff --git a/tests/test_utils.py b/tests/test_utils.py index 4248025393..46cdbe7ec4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -23,31 +23,34 @@ ##############################################################################el # Common helper routines for testing collateral import logging + logging.trace = lambda *args, **kwargs: None -import inspect -import os -import re -import shutil -from pathlib import Path - -import pandas as pd -import tempfile import builtins -import pytest -from unittest import mock -import subprocess -import selectors +import inspect import io import json -import utils.utils as utils -import logging import locale +import logging +import os +import re +import selectors +import shutil +import subprocess +import tempfile +from pathlib import Path +from unittest import mock + +import pandas as pd +import pytest + +import utils.utils as utils # ============================================================================= # HELPER FUNCTIONS FOR TESTING # ============================================================================= + def check_resource_allocation(): """Check if CTEST resource allocation is enabled for parallel testing and set HIP_VISIBLE_DEVICES variable accordingly with assigned gpu index. @@ -150,10 +153,12 @@ def check_csv_files(output_dir, num_devices, num_kernels): file_dict[file] = "pdf" return file_dict + # ============================================================================= # VERSION UTILITIES TESTS # ============================================================================= + def test_get_version_finds_version_in_home(tmp_path, monkeypatch): """Test that get_version correctly reads version and SHA from a VERSION file in the given directory. @@ -167,13 +172,20 @@ def test_get_version_finds_version_in_home(tmp_path, monkeypatch): version_content = "1.2.3" version_file = tmp_path / "VERSION" version_file.write_text(version_content) - monkeypatch.setattr(utils, "capture_subprocess_output", lambda *a, **k: (True, "abc123")) - monkeypatch.setattr(utils, "console_error", lambda *a, **k: pytest.fail("console_error should not be called")) + monkeypatch.setattr( + utils, "capture_subprocess_output", lambda *a, **k: (True, "abc123") + ) + monkeypatch.setattr( + utils, + "console_error", + lambda *a, **k: pytest.fail("console_error should not be called"), + ) result = utils.get_version(tmp_path) assert result["version"] == version_content assert result["sha"] == "abc123" assert result["mode"] == "dev" + def test_get_version_finds_version_in_parent(tmp_path, monkeypatch): """Test that get_version finds VERSION file in a parent directory when not present in the given directory. @@ -189,8 +201,14 @@ def test_get_version_finds_version_in_parent(tmp_path, monkeypatch): version_content = "2.0.0" version_file = parent / "VERSION" version_file.write_text(version_content) - monkeypatch.setattr(utils, "capture_subprocess_output", lambda *a, **k: (True, "def456")) - monkeypatch.setattr(utils, "console_error", lambda *a, **k: pytest.fail("console_error should not be called")) + monkeypatch.setattr( + utils, "capture_subprocess_output", lambda *a, **k: (True, "def456") + ) + monkeypatch.setattr( + utils, + "console_error", + lambda *a, **k: pytest.fail("console_error should not be called"), + ) child = parent / "child" child.mkdir() result = utils.get_version(child) @@ -198,6 +216,7 @@ def test_get_version_finds_version_in_parent(tmp_path, monkeypatch): assert result["sha"] == "def456" assert result["mode"] == "dev" + def test_get_version_console_error_when_no_version(monkeypatch): """Test that get_version calls console_error when no VERSION file is found in any directory. @@ -210,15 +229,18 @@ def test_get_version_console_error_when_no_version(monkeypatch): fake_path = Path("/nonexistent/path") monkeypatch.setattr(builtins, "open", mock.Mock(side_effect=FileNotFoundError)) called = {} + def fake_console_error(msg, *args, **kwargs): called["msg"] = msg raise RuntimeError("console_error called") + monkeypatch.setattr(utils, "console_error", fake_console_error) monkeypatch.setattr(utils, "capture_subprocess_output", lambda *a, **k: (False, "")) with pytest.raises(RuntimeError, match="console_error called"): utils.get_version(fake_path) assert "Cannot find VERSION file" in called["msg"] + def test_get_version_git_success(tmp_path, monkeypatch): """ Test get_version returns correct version info when git command succeeds. @@ -233,14 +255,21 @@ def test_get_version_git_success(tmp_path, monkeypatch): version_content = "1.0.0" version_file = tmp_path / "VERSION" version_file.write_text(version_content) - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "abc123")) - monkeypatch.setattr("utils.utils.console_error", lambda *a, **k: pytest.fail("console_error should not be called")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "abc123") + ) + monkeypatch.setattr( + "utils.utils.console_error", + lambda *a, **k: pytest.fail("console_error should not be called"), + ) import utils.utils as utils_mod + result = utils_mod.get_version(tmp_path) assert result["version"] == version_content assert result["sha"] == "abc123" assert result["mode"] == "dev" + def test_get_version_git_fails_sha_file(tmp_path, monkeypatch): """ Test get_version returns correct version info when git fails but VERSION.sha exists. @@ -258,15 +287,23 @@ def test_get_version_git_fails_sha_file(tmp_path, monkeypatch): sha_file = tmp_path / "VERSION.sha" version_file.write_text(version_content) sha_file.write_text(sha_content) - def fail_git(*a, **k): return (False, "git error") + + def fail_git(*a, **k): + return (False, "git error") + monkeypatch.setattr("utils.utils.capture_subprocess_output", fail_git) - monkeypatch.setattr("utils.utils.console_error", lambda *a, **k: pytest.fail("console_error should not be called")) + monkeypatch.setattr( + "utils.utils.console_error", + lambda *a, **k: pytest.fail("console_error should not be called"), + ) import utils.utils as utils_mod + result = utils_mod.get_version(tmp_path) assert result["version"] == version_content assert result["sha"] == sha_content assert result["mode"] == "release" + def test_get_version_git_and_sha_fail(tmp_path, monkeypatch): """ Test get_version returns unknown sha and mode when both git and VERSION.sha fail. @@ -281,26 +318,37 @@ def test_get_version_git_and_sha_fail(tmp_path, monkeypatch): version_content = "3.0.0" version_file = tmp_path / "VERSION" version_file.write_text(version_content) - def fail_git(*a, **k): return (False, "git error") + + def fail_git(*a, **k): + return (False, "git error") + monkeypatch.setattr("utils.utils.capture_subprocess_output", fail_git) - monkeypatch.setattr("utils.utils.console_error", lambda *a, **k: pytest.fail("console_error should not be called")) + monkeypatch.setattr( + "utils.utils.console_error", + lambda *a, **k: pytest.fail("console_error should not be called"), + ) import utils.utils as utils_mod + result = utils_mod.get_version(tmp_path) assert result["version"] == version_content assert result["sha"] == "unknown" assert result["mode"] == "unknown" - + + # ============================================================================= # ROCPROF DETECTION TESTS # ============================================================================= - + + def test_detect_rocprof_env_rocprof_not_found(monkeypatch): """ Test detect_rocprof when ROCPROF is set to 'rocprof' but the binary cannot be found. Should revert to default 'rocprof' and call console_warning, then fail with console_error. """ + class DummyArgs: rocprofiler_sdk_library_path = "/fake/path" + # Set ROCPROF to 'rocprof' monkeypatch.setenv("ROCPROF", "rocprof") # shutil.which returns None for 'rocprof' @@ -308,68 +356,107 @@ def test_detect_rocprof_env_rocprof_not_found(monkeypatch): # Track calls to console_warning and console_error warnings = [] errors = [] - monkeypatch.setattr("utils.utils.console_warning", lambda msg, *a, **k: warnings.append(msg)) + monkeypatch.setattr( + "utils.utils.console_warning", lambda msg, *a, **k: warnings.append(msg) + ) + def fake_console_error(msg, *a, **k): errors.append(msg) raise RuntimeError("console_error called") + monkeypatch.setattr("utils.utils.console_error", fake_console_error) import utils.utils as utils_mod + with pytest.raises(RuntimeError, match="console_error called"): utils_mod.detect_rocprof(DummyArgs()) assert any("Unable to resolve path to rocprofv3 binary" in w for w in warnings) - assert any("Please verify installation or set ROCPROF environment variable" in e for e in errors) + assert any( + "Please verify installation or set ROCPROF environment variable" in e + for e in errors + ) + def test_detect_rocprof_env_rocprof_found(monkeypatch): """ Test detect_rocprof when ROCPROF is set to 'rocprof' and the binary is found. Should resolve the path and return 'rocprof'. """ + class DummyArgs: rocprofiler_sdk_library_path = "/fake/path" + monkeypatch.setenv("ROCPROF", "rocprof") # shutil.which returns a fake path for 'rocprof' - monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/rocprof" if cmd == "rocprof" else None) + monkeypatch.setattr( + "shutil.which", lambda cmd: "/usr/bin/rocprof" if cmd == "rocprof" else None + ) # Path.resolve returns the same path for simplicity monkeypatch.setattr("pathlib.Path.resolve", lambda self: self) # Track debug logs logs = [] - monkeypatch.setattr("utils.utils.console_debug", lambda msg, *a, **k: logs.append(str(msg))) + monkeypatch.setattr( + "utils.utils.console_debug", lambda msg, *a, **k: logs.append(str(msg)) + ) import utils.utils as utils_mod + result = utils_mod.detect_rocprof(DummyArgs()) assert result == "rocprof" - assert any("ROC Profiler: /usr/bin/rocprof" in l or "rocprof_cmd is rocprof" in l for l in logs) + assert any( + "ROC Profiler: /usr/bin/rocprof" in l or "rocprof_cmd is rocprof" in l + for l in logs + ) + + def test_detect_rocprof_env_not_set(monkeypatch): """ Test detect_rocprof when ROCPROF is not set in the environment. Should default to 'rocprofv3' and resolve its path. """ + class DummyArgs: rocprofiler_sdk_library_path = "/fake/path" + monkeypatch.delenv("ROCPROF", raising=False) - monkeypatch.setattr("shutil.which", lambda cmd: "/usr/bin/rocprofv3" if cmd == "rocprofv3" else None) + monkeypatch.setattr( + "shutil.which", lambda cmd: "/usr/bin/rocprofv3" if cmd == "rocprofv3" else None + ) monkeypatch.setattr("pathlib.Path.resolve", lambda self: self) logs = [] - monkeypatch.setattr("utils.utils.console_debug", lambda msg, *a, **k: logs.append(str(msg))) + monkeypatch.setattr( + "utils.utils.console_debug", lambda msg, *a, **k: logs.append(str(msg)) + ) import utils.utils as utils_mod + result = utils_mod.detect_rocprof(DummyArgs()) assert result == "rocprofv3" - assert any("ROC Profiler: /usr/bin/rocprofv3" in l or "rocprof_cmd is rocprofv3" in l for l in logs) + assert any( + "ROC Profiler: /usr/bin/rocprofv3" in l or "rocprof_cmd is rocprofv3" in l + for l in logs + ) + + def test_detect_rocprof_sdk(monkeypatch): """ Test detect_rocprof when ROCPROF is set to 'rocprofiler-sdk' and the library path exists. Should return 'rocprofiler-sdk'. """ + class DummyArgs: rocprofiler_sdk_library_path = "/some/sdk/path" + monkeypatch.setenv("ROCPROF", "rocprofiler-sdk") monkeypatch.setattr("pathlib.Path.exists", lambda self: True) logs = [] - monkeypatch.setattr("utils.utils.console_debug", lambda msg, *a, **k: logs.append(str(msg))) + monkeypatch.setattr( + "utils.utils.console_debug", lambda msg, *a, **k: logs.append(str(msg)) + ) import utils.utils as utils_mod + result = utils_mod.detect_rocprof(DummyArgs()) assert result == "rocprofiler-sdk" assert any("rocprof_cmd is rocprofiler-sdk" in l for l in logs) + # ============================================================================= # SUBPROCESS UTILITIES TESTS # ============================================================================= @@ -380,7 +467,7 @@ def test_detect_rocprof_sdk(monkeypatch): # Ensures all output lines are properly captured through the selector mechanism. # """ # lines = ["line1\n", "line2\n"] - + # class DummyStdout: # def __init__(self, lines): # self._lines = lines @@ -393,7 +480,7 @@ def test_detect_rocprof_sdk(monkeypatch): # return "" # def fileno(self): # return 1 # stdout file descriptor - + # class DummyProcess: # def __init__(self): # self.stdout = DummyStdout(lines) @@ -406,12 +493,12 @@ def test_detect_rocprof_sdk(monkeypatch): # return 0 # def wait(self): # return 0 - + # dummy_process = DummyProcess() # def dummy_popen(*args, **kwargs): # return dummy_process # monkeypatch.setattr("subprocess.Popen", dummy_popen) - + # class DummySelector: # def __init__(self): # self._registered = [] @@ -429,87 +516,113 @@ def test_detect_rocprof_sdk(monkeypatch): # return [] # def close(self): # pass - + # monkeypatch.setattr("selectors.DefaultSelector", DummySelector) # monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) # monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + # import utils.utils as utils_mod # success, output = utils_mod.capture_subprocess_output(["echo", "test"]) - + # assert success is True # assert "line1" in output and "line2" in output + def test_capture_subprocess_output_with_new_env(monkeypatch): """ Test capture_subprocess_output with custom environment variables. Verifies that new_env parameter is properly passed to subprocess. """ + class DummyProcess: def __init__(self): - self.stdout = type('MockStdout', (), {'readline': lambda: '', 'fileno': lambda: 1})() + self.stdout = type( + "MockStdout", (), {"readline": lambda: "", "fileno": lambda: 1} + )() self._poll_count = 0 + def poll(self): if self._poll_count == 0: self._poll_count += 1 return None return 0 + def wait(self): return 0 - + dummy_process = DummyProcess() popen_calls = [] - + def dummy_popen(*args, **kwargs): popen_calls.append(kwargs) return dummy_process - + monkeypatch.setattr("subprocess.Popen", dummy_popen) - + class DummySelector: - def register(self, fileobj, event, callback): pass - def select(self, timeout=1): return [] - def close(self): pass - + def register(self, fileobj, event, callback): + pass + + def select(self, timeout=1): + return [] + + def close(self): + pass + monkeypatch.setattr("selectors.DefaultSelector", DummySelector) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + import utils.utils as utils_mod + custom_env = {"CUSTOM_VAR": "test_value"} utils_mod.capture_subprocess_output(["echo", "test"], new_env=custom_env) - + # Verify that custom environment was passed assert len(popen_calls) == 1 assert popen_calls[0]["env"] == custom_env + def test_capture_subprocess_output_profile_mode(monkeypatch): """ Test capture_subprocess_output with profileMode flag enabled. Verifies different behavior when profiling mode is active. """ + class DummyProcess: def __init__(self): - self.stdout = type('MockStdout', (), {'readline': lambda: '', 'fileno': lambda: 1})() - def poll(self): return 0 - def wait(self): return 0 - + self.stdout = type( + "MockStdout", (), {"readline": lambda: "", "fileno": lambda: 1} + )() + + def poll(self): + return 0 + + def wait(self): + return 0 + monkeypatch.setattr("subprocess.Popen", lambda *a, **k: DummyProcess()) - + class DummySelector: - def register(self, fileobj, event, callback): pass - def select(self, timeout=1): return [] - def close(self): pass - + def register(self, fileobj, event, callback): + pass + + def select(self, timeout=1): + return [] + + def close(self): + pass + monkeypatch.setattr("selectors.DefaultSelector", DummySelector) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + import utils.utils as utils_mod + success, output = utils_mod.capture_subprocess_output( ["echo", "test"], profileMode=True, enable_logging=False ) - + assert success is True assert isinstance(output, str) @@ -519,110 +632,149 @@ def test_capture_subprocess_output_failure(monkeypatch): Test capture_subprocess_output returns (False, output) when subprocess exits with nonzero code. """ lines = ["fail\n"] + class DummyStdout: def __init__(self, lines): self._lines = lines self._idx = 0 + def readline(self): if self._idx < len(self._lines): val = self._lines[self._idx] self._idx += 1 return val return "" + class DummyProcess: def __init__(self): self.stdout = DummyStdout(lines) self._poll_count = 0 + def poll(self): if self._poll_count == 0: self._poll_count += 1 return None return 1 + def wait(self): return 1 + dummy_process = DummyProcess() + def dummy_popen(*args, **kwargs): return dummy_process + monkeypatch.setattr("subprocess.Popen", dummy_popen) + class DummySelector: def __init__(self): self._registered = [] + def register(self, fileobj, event, callback): self._registered.append((fileobj, event, callback)) + def select(self): if hasattr(self, "_called"): return [] self._called = True - key_obj = type("Key", (), { - "data": staticmethod(self._registered[0][2]), - "fileobj": self._registered[0][0] - })() + key_obj = type( + "Key", + (), + { + "data": staticmethod(self._registered[0][2]), + "fileobj": self._registered[0][0], + }, + )() return [(key_obj, 1)] + def close(self): - pass + pass + monkeypatch.setattr("selectors.DefaultSelector", DummySelector) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) import utils.utils as utils_mod + success, output = utils_mod.capture_subprocess_output(["fail", "test"]) assert success is False assert "fail" in output + def test_capture_subprocess_output_unicode_decode(monkeypatch): """ Test capture_subprocess_output handles UnicodeDecodeError in handle_output gracefully. """ + class DummyStdout: def __init__(self): self._called = False + def readline(self): if not self._called: self._called = True raise UnicodeDecodeError("utf-8", b"", 0, 1, "reason") return "" + class DummyProcess: def __init__(self): self.stdout = DummyStdout() self._poll_count = 0 + def poll(self): if self._poll_count == 0: self._poll_count += 1 return None return 0 + def wait(self): return 0 + dummy_process = DummyProcess() + def dummy_popen(*args, **kwargs): return dummy_process + monkeypatch.setattr("subprocess.Popen", dummy_popen) + class DummySelector: def __init__(self): self._registered = [] + def register(self, fileobj, event, callback): self._registered.append((fileobj, event, callback)) + def select(self): if hasattr(self, "_called"): return [] self._called = True - key_obj = type("Key", (), { - "data": staticmethod(self._registered[0][2]), - "fileobj": self._registered[0][0] - })() + key_obj = type( + "Key", + (), + { + "data": staticmethod(self._registered[0][2]), + "fileobj": self._registered[0][0], + }, + )() return [(key_obj, 1)] + def close(self): - pass + pass + monkeypatch.setattr("selectors.DefaultSelector", DummySelector) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) import utils.utils as utils_mod + success, output = utils_mod.capture_subprocess_output(["echo", "test"]) assert success is True assert output == "" - + + # ============================================================================= # JSON DATA PARSING TESTS # ============================================================================= + def test_get_agent_dict_basic(): """ Test get_agent_dict correctly maps agent IDs to agent objects. @@ -632,50 +784,56 @@ def test_get_agent_dict_basic(): { "agents": [ {"id": {"handle": 1}, "type": 2, "node_id": 100}, - {"id": {"handle": 2}, "type": 2, "node_id": 200} + {"id": {"handle": 2}, "type": 2, "node_id": 200}, ] } ] } - + result = utils.get_agent_dict(data) - + # Verify correct mapping assert len(result) == 2 assert result[1]["node_id"] == 100 assert result[2]["node_id"] == 200 assert result[1]["type"] == 2 assert result[2]["type"] == 2 + + def test_get_agent_dict_empty_agents(): """ Test get_agent_dict with an empty agents list. """ data = {"rocprofiler-sdk-tool": [{"agents": []}]} - + result = utils.get_agent_dict(data) - + assert result == {} + + def test_get_agent_dict_missing_keys(monkeypatch): """ Test get_agent_dict behavior when expected keys are missing. """ # Case 1: Missing 'agents' key data1 = {"rocprofiler-sdk-tool": [{}]} - + with pytest.raises(KeyError): utils.get_agent_dict(data1) - + # Case 2: Missing 'rocprofiler-sdk-tool' key data2 = {} - + with pytest.raises(KeyError): utils.get_agent_dict(data2) - + # Case 3: Empty 'rocprofiler-sdk-tool' list data3 = {"rocprofiler-sdk-tool": []} - + with pytest.raises(IndexError): utils.get_agent_dict(data3) + + def test_get_agent_dict_duplicate_agent_ids(): """ Test get_agent_dict behavior with duplicate agent IDs. @@ -686,17 +844,19 @@ def test_get_agent_dict_duplicate_agent_ids(): { "agents": [ {"id": {"handle": 1}, "type": 2, "node_id": 100, "name": "first"}, - {"id": {"handle": 1}, "type": 2, "node_id": 200, "name": "second"} + {"id": {"handle": 1}, "type": 2, "node_id": 200, "name": "second"}, ] } ] } - + result = utils.get_agent_dict(data) - + assert len(result) == 1 assert result[1]["node_id"] == 200 assert result[1]["name"] == "second" + + def test_get_agent_dict_non_integer_handles(): """ Test get_agent_dict with non-integer handle values. @@ -706,18 +866,19 @@ def test_get_agent_dict_non_integer_handles(): { "agents": [ {"id": {"handle": "agent_1"}, "type": 2, "node_id": 100}, - {"id": {"handle": "agent_2"}, "type": 2, "node_id": 200} + {"id": {"handle": "agent_2"}, "type": 2, "node_id": 200}, ] } ] } - + result = utils.get_agent_dict(data) - + assert len(result) == 2 assert result["agent_1"]["node_id"] == 100 assert result["agent_2"]["node_id"] == 200 - + + # Tests for get_gpuid_dict function ========================================================= def test_get_gpuid_dict_basic(): """Test that get_gpuid_dict correctly maps agent IDs to GPU IDs for a basic case. @@ -737,11 +898,13 @@ def test_get_gpuid_dict_basic(): } ] } - + expected = {101: 0, 100: 1, 102: 2} - + result = utils.get_gpuid_dict(data) assert result == expected + + def test_get_gpuid_dict_no_gpu_agents(): """Test that get_gpuid_dict returns an empty dictionary when no GPU agents are present. Args: @@ -760,9 +923,11 @@ def test_get_gpuid_dict_no_gpu_agents(): } ] } - + result = utils.get_gpuid_dict(data) assert result == {} + + def test_get_gpuid_dict_mixed_agents(): """Test that get_gpuid_dict correctly ignores non-GPU agents and only maps GPU agents. Args: @@ -782,12 +947,14 @@ def test_get_gpuid_dict_mixed_agents(): } ] } - + # Expected mapping after sorting by node_id and filtering by type 2: 100->0, 102->1 expected = {100: 0, 102: 1} - + result = utils.get_gpuid_dict(data) assert result == expected + + def test_get_gpuid_dict_sorting(): """Test that get_gpuid_dict correctly sorts GPU agents by node_id to determine GPU ID ordering. Args: @@ -800,18 +967,20 @@ def test_get_gpuid_dict_sorting(): { "agents": [ {"id": {"handle": 100}, "node_id": 10, "type": 2}, # GPU agent - {"id": {"handle": 101}, "node_id": 5, "type": 2}, # GPU agent - {"id": {"handle": 102}, "node_id": 8, "type": 2}, # GPU agent - {"id": {"handle": 103}, "node_id": 1, "type": 2}, # GPU agent + {"id": {"handle": 101}, "node_id": 5, "type": 2}, # GPU agent + {"id": {"handle": 102}, "node_id": 8, "type": 2}, # GPU agent + {"id": {"handle": 103}, "node_id": 1, "type": 2}, # GPU agent ] } ] } - + expected = {103: 0, 101: 1, 102: 2, 100: 3} - + result = utils.get_gpuid_dict(data) assert result == expected + + def test_get_gpuid_dict_empty_agents(): """Test that get_gpuid_dict handles an empty agents list correctly. Args: @@ -820,21 +989,16 @@ def test_get_gpuid_dict_empty_agents(): None: Asserts that an empty dictionary is returned when the agents list is empty. """ # Sample data with empty agents list - data = { - "rocprofiler-sdk-tool": [ - { - "agents": [] - } - ] - } - + data = {"rocprofiler-sdk-tool": [{"agents": []}]} + result = utils.get_gpuid_dict(data) assert result == {} + # Tests for v3_json_get_counters function ========================================================= def test_v3_json_get_counters_normal_case(): """Test v3_json_get_counters with a valid data structure containing multiple counters. - + This test verifies that the function correctly extracts counters from the JSON data and creates a mapping using (agent_id, counter_id) tuples as keys. """ @@ -842,41 +1006,51 @@ def test_v3_json_get_counters_normal_case(): "rocprofiler-sdk-tool": [ { "counters": [ - {"id": {"handle": 1}, "agent_id": {"handle": 100}, "name": "counter1"}, - {"id": {"handle": 2}, "agent_id": {"handle": 100}, "name": "counter2"}, - {"id": {"handle": 1}, "agent_id": {"handle": 200}, "name": "counter3"} + { + "id": {"handle": 1}, + "agent_id": {"handle": 100}, + "name": "counter1", + }, + { + "id": {"handle": 2}, + "agent_id": {"handle": 100}, + "name": "counter2", + }, + { + "id": {"handle": 1}, + "agent_id": {"handle": 200}, + "name": "counter3", + }, ] } ] } - + counter_map = utils.v3_json_get_counters(data) - + assert len(counter_map) == 3 assert counter_map[(100, 1)]["name"] == "counter1" assert counter_map[(100, 2)]["name"] == "counter2" assert counter_map[(200, 1)]["name"] == "counter3" + + def test_v3_json_get_counters_empty_counters(): """Test v3_json_get_counters with an empty counters array. - + This test ensures the function handles the case where no counters are present and returns an empty dictionary. """ - data = { - "rocprofiler-sdk-tool": [ - { - "counters": [] - } - ] - } - + data = {"rocprofiler-sdk-tool": [{"counters": []}]} + counter_map = utils.v3_json_get_counters(data) - + assert len(counter_map) == 0 assert counter_map == {} + + def test_v3_json_get_counters_duplicate_keys(): """Test v3_json_get_counters with duplicate (agent_id, counter_id) tuples. - + This test verifies that when multiple counters have the same (agent_id, counter_id) tuple, the last counter overwrites previous ones in the returned dictionary. """ @@ -884,20 +1058,30 @@ def test_v3_json_get_counters_duplicate_keys(): "rocprofiler-sdk-tool": [ { "counters": [ - {"id": {"handle": 1}, "agent_id": {"handle": 100}, "name": "counter1"}, - {"id": {"handle": 1}, "agent_id": {"handle": 100}, "name": "counter2"} + { + "id": {"handle": 1}, + "agent_id": {"handle": 100}, + "name": "counter1", + }, + { + "id": {"handle": 1}, + "agent_id": {"handle": 100}, + "name": "counter2", + }, ] } ] } - + counter_map = utils.v3_json_get_counters(data) - + assert len(counter_map) == 1 assert counter_map[(100, 1)]["name"] == "counter2" + + def test_v3_json_get_counters_various_value_types(): """Test v3_json_get_counters with different types of values for handles. - + This test ensures the function correctly handles different data types (integers and strings) for the handle values. """ @@ -905,81 +1089,85 @@ def test_v3_json_get_counters_various_value_types(): "rocprofiler-sdk-tool": [ { "counters": [ - {"id": {"handle": 1}, "agent_id": {"handle": 100}, "name": "counter1"}, - {"id": {"handle": "2"}, "agent_id": {"handle": 100}, "name": "counter2"}, - {"id": {"handle": 3}, "agent_id": {"handle": "200"}, "name": "counter3"} + { + "id": {"handle": 1}, + "agent_id": {"handle": 100}, + "name": "counter1", + }, + { + "id": {"handle": "2"}, + "agent_id": {"handle": 100}, + "name": "counter2", + }, + { + "id": {"handle": 3}, + "agent_id": {"handle": "200"}, + "name": "counter3", + }, ] } ] } - + counter_map = utils.v3_json_get_counters(data) - + assert len(counter_map) == 3 assert counter_map[(100, 1)]["name"] == "counter1" assert counter_map[(100, "2")]["name"] == "counter2" assert counter_map[("200", 3)]["name"] == "counter3" + + def test_v3_json_get_counters_missing_key(): """Test v3_json_get_counters raises KeyError when required keys are missing. - + This test verifies that the function raises a KeyError when the agent_id key is missing. """ data = { "rocprofiler-sdk-tool": [ - { - "counters": [ - {"id": {"handle": 1}, "name": "counter1"} # Missing agent_id - ] - } + {"counters": [{"id": {"handle": 1}, "name": "counter1"}]} # Missing agent_id ] } - + with pytest.raises(KeyError): utils.v3_json_get_counters(data) + + def test_v3_json_get_counters_missing_nested_key(): """Test v3_json_get_counters raises KeyError when nested required keys are missing. - - This test verifies that the function raises a KeyError when the handle key + + This test verifies that the function raises a KeyError when the handle key is missing from the id dictionary. """ data = { "rocprofiler-sdk-tool": [ - { - "counters": [ - {"id": {}, "agent_id": {"handle": 100}, "name": "counter1"} - ] - } + {"counters": [{"id": {}, "agent_id": {"handle": 100}, "name": "counter1"}]} ] } - + with pytest.raises(KeyError): utils.v3_json_get_counters(data) + + def test_v3_json_get_counters_data_structure(): """Test that v3_json_get_counters preserves the entire counter object in the mapping. - + This test ensures that the function stores the entire counter object in the mapping, not just selected fields. """ counter_object = { - "id": {"handle": 1}, - "agent_id": {"handle": 100}, + "id": {"handle": 1}, + "agent_id": {"handle": 100}, "name": "counter1", "description": "Test counter", "block": "SQ", "event_id": 123, - "enabled": True + "enabled": True, } - - data = { - "rocprofiler-sdk-tool": [ - { - "counters": [counter_object] - } - ] - } - + + data = {"rocprofiler-sdk-tool": [{"counters": [counter_object]}]} + counter_map = utils.v3_json_get_counters(data) - + assert len(counter_map) == 1 assert counter_map[(100, 1)] == counter_object assert counter_map[(100, 1)]["description"] == "Test counter" @@ -987,13 +1175,14 @@ def test_v3_json_get_counters_data_structure(): assert counter_map[(100, 1)]["event_id"] == 123 assert counter_map[(100, 1)]["enabled"] is True + def test_v3_json_get_dispatches_normal_case(): """ Test v3_json_get_dispatches with valid data containing multiple dispatch records. - + Args: None - + Returns: None: Asserts the function correctly maps all dispatch records by their correlation IDs. """ @@ -1002,87 +1191,85 @@ def test_v3_json_get_dispatches_normal_case(): { "buffer_records": { "kernel_dispatch": [ - {"correlation_id": {"internal": "id1"}, "start_timestamp": 100, "end_timestamp": 200}, - {"correlation_id": {"internal": "id2"}, "start_timestamp": 300, "end_timestamp": 400}, - {"correlation_id": {"internal": "id3"}, "start_timestamp": 500, "end_timestamp": 600}, + { + "correlation_id": {"internal": "id1"}, + "start_timestamp": 100, + "end_timestamp": 200, + }, + { + "correlation_id": {"internal": "id2"}, + "start_timestamp": 300, + "end_timestamp": 400, + }, + { + "correlation_id": {"internal": "id3"}, + "start_timestamp": 500, + "end_timestamp": 600, + }, ] } } ] } - + result = utils.v3_json_get_dispatches(data) - + assert len(result) == 3 assert result["id1"]["start_timestamp"] == 100 assert result["id2"]["end_timestamp"] == 400 assert result["id3"]["correlation_id"]["internal"] == "id3" + + def test_v3_json_get_dispatches_empty_case(): """ Test v3_json_get_dispatches with data containing no dispatch records. - + Args: None - + Returns: None: Asserts the function returns an empty dictionary when no dispatch records are present. """ - data = { - "rocprofiler-sdk-tool": [ - { - "buffer_records": { - "kernel_dispatch": [] - } - } - ] - } - + data = {"rocprofiler-sdk-tool": [{"buffer_records": {"kernel_dispatch": []}}]} + result = utils.v3_json_get_dispatches(data) - + assert len(result) == 0 assert isinstance(result, dict) + + def test_v3_json_get_dispatches_missing_fields(): """ Test v3_json_get_dispatches handling of data with missing required fields. - + Args: None - + Returns: None: Asserts the function raises a KeyError when required fields are missing. """ - data = { - "rocprofiler-sdk-tool": [ - { - "buffer_records": {} - } - ] - } - + data = {"rocprofiler-sdk-tool": [{"buffer_records": {}}]} + with pytest.raises(KeyError): utils.v3_json_get_dispatches(data) - + data = { "rocprofiler-sdk-tool": [ - { - "buffer_records": { - "kernel_dispatch": [ - {"start_timestamp": 100} - ] - } - } + {"buffer_records": {"kernel_dispatch": [{"start_timestamp": 100}]}} ] } - + with pytest.raises(KeyError): utils.v3_json_get_dispatches(data) + + def test_v3_json_get_dispatches_duplicate_ids(): """ Test v3_json_get_dispatches handling of duplicate correlation IDs. - + Args: None - + Returns: None: Asserts that when duplicate correlation IDs exist, the function keeps the latest record. """ @@ -1091,567 +1278,813 @@ def test_v3_json_get_dispatches_duplicate_ids(): { "buffer_records": { "kernel_dispatch": [ - {"correlation_id": {"internal": "id1"}, "start_timestamp": 100, "end_timestamp": 200}, - {"correlation_id": {"internal": "id1"}, "start_timestamp": 300, "end_timestamp": 400}, # Duplicate ID - {"correlation_id": {"internal": "id3"}, "start_timestamp": 500, "end_timestamp": 600}, + { + "correlation_id": {"internal": "id1"}, + "start_timestamp": 100, + "end_timestamp": 200, + }, + { + "correlation_id": {"internal": "id1"}, + "start_timestamp": 300, + "end_timestamp": 400, + }, # Duplicate ID + { + "correlation_id": {"internal": "id3"}, + "start_timestamp": 500, + "end_timestamp": 600, + }, ] } } ] } - + result = utils.v3_json_get_dispatches(data) - + assert len(result) == 2 - assert result["id1"]["start_timestamp"] == 300 + assert result["id1"]["start_timestamp"] == 300 assert result["id1"]["end_timestamp"] == 400 assert "id3" in result - + + # ============================================================================= # JSON TO CSV CONVERSION TESTS # ============================================================================= + def test_v3_json_to_csv_basic_functionality(tmp_path, monkeypatch): - """ - Test basic functionality of v3_json_to_csv with a minimal valid JSON input. - - Args: - tmp_path (pathlib.Path): Temporary directory for test files - monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior - """ - - valid_json = { - "rocprofiler-sdk-tool": [{ + """ + Test basic functionality of v3_json_to_csv with a minimal valid JSON input. + + Args: + tmp_path (pathlib.Path): Temporary directory for test files + monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior + """ + + valid_json = { + "rocprofiler-sdk-tool": [ + { "metadata": {"pid": 12345}, - "agents": [{"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}], - "counters": [{"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER1"}], - "kernel_symbols": {"kernel1": {"formatted_kernel_name": "TestKernel", "private_segment_size": 0}}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64} + ], + "counters": [ + {"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER1"} + ], + "kernel_symbols": { + "kernel1": { + "formatted_kernel_name": "TestKernel", + "private_segment_size": 0, + } + }, "buffer_records": { - "kernel_dispatch": [{"correlation_id": {"internal": "corr1"}, "start_timestamp": 100, "end_timestamp": 200}] + "kernel_dispatch": [ + { + "correlation_id": {"internal": "corr1"}, + "start_timestamp": 100, + "end_timestamp": 200, + } + ] }, "callback_records": { - "counter_collection": [{ - "thread_id": 67890, - "lds_block_size_v": 0, - "arch_vgpr_count": 32, - "sgpr_count": 16, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 1, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 2}, - "kernel_id": "kernel1", - "grid_size": {"x": 1, "y": 1, "z": 1}, - "workgroup_size": {"x": 64, "y": 1, "z": 1} + "counter_collection": [ + { + "thread_id": 67890, + "lds_block_size_v": 0, + "arch_vgpr_count": 32, + "sgpr_count": 16, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 1, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 2}, + "kernel_id": "kernel1", + "grid_size": {"x": 1, "y": 1, "z": 1}, + "workgroup_size": {"x": 64, "y": 1, "z": 1}, + }, + "correlation_id": { + "internal": "corr1", + "external": "ext1", + }, }, - "correlation_id": {"internal": "corr1", "external": "ext1"} - }, - "records": [{"counter_id": {"handle": 101}, "value": 42}] - }] - } - }] - } - - json_path = tmp_path / "test.json" - with open(json_path, "w") as f: - json.dump(valid_json, f) - - csv_path = tmp_path / "output.csv" - - monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: {"corr1": valid_json["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][0]}) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: {1: valid_json["rocprofiler-sdk-tool"][0]["agents"][0]}) - monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0}) - monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: {(1, 101): {"name": "COUNTER1"}}) - - utils.v3_json_to_csv(json_path, csv_path) - - assert csv_path.exists() - df = pd.read_csv(csv_path) - - assert "Dispatch_ID" in df.columns - assert "GPU_ID" in df.columns - assert "Kernel_Name" in df.columns - assert "COUNTER1" in df.columns - assert len(df) == 1 - assert df["Dispatch_ID"][0] == 1 - assert df["Kernel_Name"][0] == "TestKernel" - assert df["COUNTER1"][0] == 42 - assert df["Start_Timestamp"][0] == 100 - assert df["End_Timestamp"][0] == 200 + "records": [{"counter_id": {"handle": 101}, "value": 42}], + } + ] + }, + } + ] + } + + json_path = tmp_path / "test.json" + with open(json_path, "w") as f: + json.dump(valid_json, f) + + csv_path = tmp_path / "output.csv" + + monkeypatch.setattr( + utils, + "v3_json_get_dispatches", + lambda data: { + "corr1": valid_json["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][0] + }, + ) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: {1: valid_json["rocprofiler-sdk-tool"][0]["agents"][0]}, + ) + monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0}) + monkeypatch.setattr( + utils, "v3_json_get_counters", lambda data: {(1, 101): {"name": "COUNTER1"}} + ) + + utils.v3_json_to_csv(json_path, csv_path) + + assert csv_path.exists() + df = pd.read_csv(csv_path) + + assert "Dispatch_ID" in df.columns + assert "GPU_ID" in df.columns + assert "Kernel_Name" in df.columns + assert "COUNTER1" in df.columns + assert len(df) == 1 + assert df["Dispatch_ID"][0] == 1 + assert df["Kernel_Name"][0] == "TestKernel" + assert df["COUNTER1"][0] == 42 + assert df["Start_Timestamp"][0] == 100 + assert df["End_Timestamp"][0] == 200 + def test_v3_json_to_csv_no_dispatches(tmp_path, monkeypatch): """ Test v3_json_to_csv with a JSON file that has no dispatches. Should create an empty CSV with headers. - + Args: tmp_path (pathlib.Path): Temporary directory for test files monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ - + empty_json = { - "rocprofiler-sdk-tool": [{ - "metadata": {"pid": 12345}, - "agents": [{"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}], - "counters": [], - "kernel_symbols": {}, - "buffer_records": {"kernel_dispatch": []}, - "callback_records": {"counter_collection": []} - }] + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 12345}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64} + ], + "counters": [], + "kernel_symbols": {}, + "buffer_records": {"kernel_dispatch": []}, + "callback_records": {"counter_collection": []}, + } + ] } - + json_path = tmp_path / "empty.json" with open(json_path, "w") as f: json.dump(empty_json, f) csv_path = tmp_path / "empty_output.csv" - + monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: {}) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: {1: empty_json["rocprofiler-sdk-tool"][0]["agents"][0]}) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: {1: empty_json["rocprofiler-sdk-tool"][0]["agents"][0]}, + ) monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0}) monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: {}) - + utils.v3_json_to_csv(json_path, csv_path) - + assert csv_path.exists() df = pd.read_csv(csv_path) - + assert "Dispatch_ID" in df.columns assert "GPU_ID" in df.columns assert "Kernel_Name" in df.columns assert len(df) == 0 + + def test_v3_json_to_csv_accumulated_counters(tmp_path, monkeypatch): """ Test v3_json_to_csv handling of accumulated counters (with _ACCUM suffix). Should rename them to SQ_ACCUM_PREV_HIRES. - + Args: tmp_path (pathlib.Path): Temporary directory for test files monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ - + json_data = { - "rocprofiler-sdk-tool": [{ - "metadata": {"pid": 12345}, - "agents": [{"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}], - "counters": [ - {"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER_ACCUM"} - ], - "kernel_symbols": {"kernel1": {"formatted_kernel_name": "TestKernel", "private_segment_size": 0}}, - "buffer_records": { - "kernel_dispatch": [{"correlation_id": {"internal": "corr1"}, "start_timestamp": 100, "end_timestamp": 200}] - }, - "callback_records": { - "counter_collection": [{ - "thread_id": 67890, - "lds_block_size_v": 0, - "arch_vgpr_count": 32, - "sgpr_count": 16, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 1, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 2}, - "kernel_id": "kernel1", - "grid_size": {"x": 1, "y": 1, "z": 1}, - "workgroup_size": {"x": 64, "y": 1, "z": 1} - }, - "correlation_id": {"internal": "corr1", "external": "ext1"} - }, - "records": [{"counter_id": {"handle": 101}, "value": 42}] - }] + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 12345}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64} + ], + "counters": [ + { + "id": {"handle": 101}, + "agent_id": {"handle": 1}, + "name": "COUNTER_ACCUM", + } + ], + "kernel_symbols": { + "kernel1": { + "formatted_kernel_name": "TestKernel", + "private_segment_size": 0, + } + }, + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": "corr1"}, + "start_timestamp": 100, + "end_timestamp": 200, + } + ] + }, + "callback_records": { + "counter_collection": [ + { + "thread_id": 67890, + "lds_block_size_v": 0, + "arch_vgpr_count": 32, + "sgpr_count": 16, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 1, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 2}, + "kernel_id": "kernel1", + "grid_size": {"x": 1, "y": 1, "z": 1}, + "workgroup_size": {"x": 64, "y": 1, "z": 1}, + }, + "correlation_id": { + "internal": "corr1", + "external": "ext1", + }, + }, + "records": [{"counter_id": {"handle": 101}, "value": 42}], + } + ] + }, } - }] + ] } - + json_path = tmp_path / "accum.json" with open(json_path, "w") as f: json.dump(json_data, f) - + csv_path = tmp_path / "accum_output.csv" - - monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: {"corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][0]}) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: {1: json_data["rocprofiler-sdk-tool"][0]["agents"][0]}) + + monkeypatch.setattr( + utils, + "v3_json_get_dispatches", + lambda data: { + "corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][0] + }, + ) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: {1: json_data["rocprofiler-sdk-tool"][0]["agents"][0]}, + ) monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0}) - monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: {(1, 101): {"name": "COUNTER_ACCUM"}}) - + monkeypatch.setattr( + utils, "v3_json_get_counters", lambda data: {(1, 101): {"name": "COUNTER_ACCUM"}} + ) + utils.v3_json_to_csv(json_path, csv_path) - + assert csv_path.exists() df = pd.read_csv(csv_path) - + assert "COUNTER_ACCUM" not in df.columns assert "SQ_ACCUM_PREV_HIRES" in df.columns assert df["SQ_ACCUM_PREV_HIRES"][0] == 42 + + def test_v3_json_to_csv_duplicate_counters(tmp_path, monkeypatch): """ Test v3_json_to_csv handling of duplicate counter names. Should sum the values. - + Args: tmp_path (pathlib.Path): Temporary directory for test files monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ - + json_data = { - "rocprofiler-sdk-tool": [{ - "metadata": {"pid": 12345}, - "agents": [{"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}], - "counters": [ - {"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER1"}, - {"id": {"handle": 102}, "agent_id": {"handle": 1}, "name": "COUNTER1"} - ], - "kernel_symbols": {"kernel1": {"formatted_kernel_name": "TestKernel", "private_segment_size": 0}}, - "buffer_records": { - "kernel_dispatch": [{"correlation_id": {"internal": "corr1"}, "start_timestamp": 100, "end_timestamp": 200}] - }, - "callback_records": { - "counter_collection": [{ - "thread_id": 67890, - "lds_block_size_v": 0, - "arch_vgpr_count": 32, - "sgpr_count": 16, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 1, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 2}, - "kernel_id": "kernel1", - "grid_size": {"x": 1, "y": 1, "z": 1}, - "workgroup_size": {"x": 64, "y": 1, "z": 1} - }, - "correlation_id": {"internal": "corr1", "external": "ext1"} + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 12345}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64} + ], + "counters": [ + { + "id": {"handle": 101}, + "agent_id": {"handle": 1}, + "name": "COUNTER1", }, - "records": [ - {"counter_id": {"handle": 101}, "value": 42}, - {"counter_id": {"handle": 102}, "value": 58} + { + "id": {"handle": 102}, + "agent_id": {"handle": 1}, + "name": "COUNTER1", + }, + ], + "kernel_symbols": { + "kernel1": { + "formatted_kernel_name": "TestKernel", + "private_segment_size": 0, + } + }, + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": "corr1"}, + "start_timestamp": 100, + "end_timestamp": 200, + } ] - }] + }, + "callback_records": { + "counter_collection": [ + { + "thread_id": 67890, + "lds_block_size_v": 0, + "arch_vgpr_count": 32, + "sgpr_count": 16, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 1, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 2}, + "kernel_id": "kernel1", + "grid_size": {"x": 1, "y": 1, "z": 1}, + "workgroup_size": {"x": 64, "y": 1, "z": 1}, + }, + "correlation_id": { + "internal": "corr1", + "external": "ext1", + }, + }, + "records": [ + {"counter_id": {"handle": 101}, "value": 42}, + {"counter_id": {"handle": 102}, "value": 58}, + ], + } + ] + }, } - }] + ] } - + json_path = tmp_path / "duplicate.json" with open(json_path, "w") as f: json.dump(json_data, f) - + csv_path = tmp_path / "duplicate_output.csv" - - monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: {"corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][0]}) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: {1: json_data["rocprofiler-sdk-tool"][0]["agents"][0]}) + + monkeypatch.setattr( + utils, + "v3_json_get_dispatches", + lambda data: { + "corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][0] + }, + ) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: {1: json_data["rocprofiler-sdk-tool"][0]["agents"][0]}, + ) monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0}) - monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: { - (1, 101): {"name": "COUNTER1"}, - (1, 102): {"name": "COUNTER1"} - }) - + monkeypatch.setattr( + utils, + "v3_json_get_counters", + lambda data: {(1, 101): {"name": "COUNTER1"}, (1, 102): {"name": "COUNTER1"}}, + ) + utils.v3_json_to_csv(json_path, csv_path) - + assert csv_path.exists() df = pd.read_csv(csv_path) - + assert df["COUNTER1"][0] == 100 # 42 + 58 + + def test_v3_json_to_csv_file_not_found(monkeypatch): """ Test v3_json_to_csv handling of non-existent input file. Should raise FileNotFoundError. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ with pytest.raises(FileNotFoundError): utils.v3_json_to_csv("/nonexistent/path.json", "output.csv") + + def test_v3_json_to_csv_invalid_json(tmp_path): """ Test v3_json_to_csv handling of invalid JSON input. Should raise JSONDecodeError. - + Args: tmp_path (pathlib.Path): Temporary directory for test files """ json_path = tmp_path / "invalid.json" with open(json_path, "w") as f: f.write("{invalid json") - + csv_path = tmp_path / "invalid_output.csv" - + with pytest.raises(json.JSONDecodeError): utils.v3_json_to_csv(json_path, csv_path) + + def test_v3_json_to_csv_missing_required_keys(tmp_path): """ Test v3_json_to_csv handling of JSON missing required keys. Should raise KeyError. - + Args: tmp_path (pathlib.Path): Temporary directory for test files """ - + invalid_json = { - "rocprofiler-sdk-tool": [{ - # Missing "metadata", "agents", etc. - "kernel_symbols": {} - }] + "rocprofiler-sdk-tool": [ + { + # Missing "metadata", "agents", etc. + "kernel_symbols": {} + } + ] } - + json_path = tmp_path / "missing_keys.json" with open(json_path, "w") as f: json.dump(invalid_json, f) - + csv_path = tmp_path / "missing_keys_output.csv" - + with pytest.raises(KeyError): utils.v3_json_to_csv(json_path, csv_path) + + def test_v3_json_to_csv_complex_dispatch(tmp_path, monkeypatch): """ Test v3_json_to_csv with a more complex dispatch scenario including multiple dispatches and 3D grid/workgroup sizes. - + Args: tmp_path (pathlib.Path): Temporary directory for test files monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ - + complex_json = { - "rocprofiler-sdk-tool": [{ - "metadata": {"pid": 12345}, - "agents": [ - {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}, - {"id": {"handle": 2}, "type": 2, "node_id": 1, "wave_front_size": 32} - ], - "counters": [ - {"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER1"}, - {"id": {"handle": 102}, "agent_id": {"handle": 2}, "name": "COUNTER2"} - ], - "kernel_symbols": { - "kernel1": {"formatted_kernel_name": "Kernel1", "private_segment_size": 16}, - "kernel2": {"formatted_kernel_name": "Kernel2", "private_segment_size": 32} - }, - "buffer_records": { - "kernel_dispatch": [ - {"correlation_id": {"internal": "corr1"}, "start_timestamp": 100, "end_timestamp": 200}, - {"correlation_id": {"internal": "corr2"}, "start_timestamp": 300, "end_timestamp": 400} - ] - }, - "callback_records": { - "counter_collection": [ + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 12345}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}, + {"id": {"handle": 2}, "type": 2, "node_id": 1, "wave_front_size": 32}, + ], + "counters": [ { - "thread_id": 67890, - "lds_block_size_v": 64, - "arch_vgpr_count": 32, - "sgpr_count": 16, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 1, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 2}, - "kernel_id": "kernel1", - "grid_size": {"x": 2, "y": 3, "z": 4}, - "workgroup_size": {"x": 8, "y": 4, "z": 2} - }, - "correlation_id": {"internal": "corr1", "external": "ext1"} - }, - "records": [{"counter_id": {"handle": 101}, "value": 42}] + "id": {"handle": 101}, + "agent_id": {"handle": 1}, + "name": "COUNTER1", }, { - "thread_id": 67891, - "lds_block_size_v": 128, - "arch_vgpr_count": 64, - "sgpr_count": 32, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 2, - "agent_id": {"handle": 2}, - "queue_id": {"handle": 3}, - "kernel_id": "kernel2", - "grid_size": {"x": 16, "y": 8, "z": 4}, - "workgroup_size": {"x": 16, "y": 16, "z": 1} - }, - "correlation_id": {"internal": "corr2", "external": "ext2"} + "id": {"handle": 102}, + "agent_id": {"handle": 2}, + "name": "COUNTER2", + }, + ], + "kernel_symbols": { + "kernel1": { + "formatted_kernel_name": "Kernel1", + "private_segment_size": 16, + }, + "kernel2": { + "formatted_kernel_name": "Kernel2", + "private_segment_size": 32, + }, + }, + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": "corr1"}, + "start_timestamp": 100, + "end_timestamp": 200, }, - "records": [{"counter_id": {"handle": 102}, "value": 84}] - } - ] + { + "correlation_id": {"internal": "corr2"}, + "start_timestamp": 300, + "end_timestamp": 400, + }, + ] + }, + "callback_records": { + "counter_collection": [ + { + "thread_id": 67890, + "lds_block_size_v": 64, + "arch_vgpr_count": 32, + "sgpr_count": 16, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 1, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 2}, + "kernel_id": "kernel1", + "grid_size": {"x": 2, "y": 3, "z": 4}, + "workgroup_size": {"x": 8, "y": 4, "z": 2}, + }, + "correlation_id": { + "internal": "corr1", + "external": "ext1", + }, + }, + "records": [{"counter_id": {"handle": 101}, "value": 42}], + }, + { + "thread_id": 67891, + "lds_block_size_v": 128, + "arch_vgpr_count": 64, + "sgpr_count": 32, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 2, + "agent_id": {"handle": 2}, + "queue_id": {"handle": 3}, + "kernel_id": "kernel2", + "grid_size": {"x": 16, "y": 8, "z": 4}, + "workgroup_size": {"x": 16, "y": 16, "z": 1}, + }, + "correlation_id": { + "internal": "corr2", + "external": "ext2", + }, + }, + "records": [{"counter_id": {"handle": 102}, "value": 84}], + }, + ] + }, } - }] + ] } - + json_path = tmp_path / "complex.json" with open(json_path, "w") as f: json.dump(complex_json, f) - + csv_path = tmp_path / "complex_output.csv" - - monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: { - "corr1": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][0], - "corr2": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][1] - }) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: { - 1: complex_json["rocprofiler-sdk-tool"][0]["agents"][0], - 2: complex_json["rocprofiler-sdk-tool"][0]["agents"][1] - }) + + monkeypatch.setattr( + utils, + "v3_json_get_dispatches", + lambda data: { + "corr1": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][0], + "corr2": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][1], + }, + ) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: { + 1: complex_json["rocprofiler-sdk-tool"][0]["agents"][0], + 2: complex_json["rocprofiler-sdk-tool"][0]["agents"][1], + }, + ) monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0, 2: 1}) - monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: { - (1, 101): {"name": "COUNTER1"}, - (2, 102): {"name": "COUNTER2"} - }) - + monkeypatch.setattr( + utils, + "v3_json_get_counters", + lambda data: {(1, 101): {"name": "COUNTER1"}, (2, 102): {"name": "COUNTER2"}}, + ) + utils.v3_json_to_csv(json_path, csv_path) - + assert csv_path.exists() df = pd.read_csv(csv_path) - + assert len(df) == 2 - + assert df["Grid_Size"][0] == 24 assert df["Workgroup_Size"][0] == 64 assert df["Kernel_Name"][0] == "Kernel1" assert df["COUNTER1"][0] == 42 assert df["GPU_ID"][0] == 0 assert df["Wave_Size"][0] == 64 - + assert df["Grid_Size"][1] == 512 assert df["Workgroup_Size"][1] == 256 assert df["Kernel_Name"][1] == "Kernel2" assert df["COUNTER2"][1] == 84 assert df["GPU_ID"][1] == 1 assert df["Wave_Size"][1] == 32 - + + def test_v3_json_to_csv_missing_counters_handling(tmp_path, monkeypatch): """ Test v3_json_to_csv handles cases where different dispatches have different sets of counters. This addresses the DataFrame creation issue where arrays have different lengths. - + Args: tmp_path (pathlib.Path): Temporary directory for test files monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ - + json_data = { - "rocprofiler-sdk-tool": [{ - "metadata": {"pid": 12345}, - "agents": [ - {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}, - {"id": {"handle": 2}, "type": 2, "node_id": 1, "wave_front_size": 32} - ], - "counters": [ - {"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER1"}, - {"id": {"handle": 102}, "agent_id": {"handle": 2}, "name": "COUNTER2"} - ], - "kernel_symbols": { - "kernel1": {"formatted_kernel_name": "Kernel1", "private_segment_size": 16}, - "kernel2": {"formatted_kernel_name": "Kernel2", "private_segment_size": 32} - }, - "buffer_records": { - "kernel_dispatch": [ - {"correlation_id": {"internal": "corr1"}, "start_timestamp": 100, "end_timestamp": 200}, - {"correlation_id": {"internal": "corr2"}, "start_timestamp": 300, "end_timestamp": 400} - ] - }, - "callback_records": { - "counter_collection": [ + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 12345}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}, + {"id": {"handle": 2}, "type": 2, "node_id": 1, "wave_front_size": 32}, + ], + "counters": [ { - "thread_id": 67890, - "lds_block_size_v": 64, - "arch_vgpr_count": 32, - "sgpr_count": 16, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 1, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 2}, - "kernel_id": "kernel1", - "grid_size": {"x": 2, "y": 3, "z": 4}, - "workgroup_size": {"x": 8, "y": 4, "z": 2} - }, - "correlation_id": {"internal": "corr1", "external": "ext1"} - }, - "records": [{"counter_id": {"handle": 101}, "value": 42}] # Only COUNTER1 + "id": {"handle": 101}, + "agent_id": {"handle": 1}, + "name": "COUNTER1", }, { - "thread_id": 67891, - "lds_block_size_v": 128, - "arch_vgpr_count": 64, - "sgpr_count": 32, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 2, - "agent_id": {"handle": 2}, - "queue_id": {"handle": 3}, - "kernel_id": "kernel2", - "grid_size": {"x": 16, "y": 8, "z": 4}, - "workgroup_size": {"x": 16, "y": 16, "z": 1} - }, - "correlation_id": {"internal": "corr2", "external": "ext2"} + "id": {"handle": 102}, + "agent_id": {"handle": 2}, + "name": "COUNTER2", + }, + ], + "kernel_symbols": { + "kernel1": { + "formatted_kernel_name": "Kernel1", + "private_segment_size": 16, + }, + "kernel2": { + "formatted_kernel_name": "Kernel2", + "private_segment_size": 32, + }, + }, + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": "corr1"}, + "start_timestamp": 100, + "end_timestamp": 200, }, - "records": [{"counter_id": {"handle": 102}, "value": 84}] # Only COUNTER2 - } - ] + { + "correlation_id": {"internal": "corr2"}, + "start_timestamp": 300, + "end_timestamp": 400, + }, + ] + }, + "callback_records": { + "counter_collection": [ + { + "thread_id": 67890, + "lds_block_size_v": 64, + "arch_vgpr_count": 32, + "sgpr_count": 16, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 1, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 2}, + "kernel_id": "kernel1", + "grid_size": {"x": 2, "y": 3, "z": 4}, + "workgroup_size": {"x": 8, "y": 4, "z": 2}, + }, + "correlation_id": { + "internal": "corr1", + "external": "ext1", + }, + }, + "records": [ + {"counter_id": {"handle": 101}, "value": 42} + ], # Only COUNTER1 + }, + { + "thread_id": 67891, + "lds_block_size_v": 128, + "arch_vgpr_count": 64, + "sgpr_count": 32, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 2, + "agent_id": {"handle": 2}, + "queue_id": {"handle": 3}, + "kernel_id": "kernel2", + "grid_size": {"x": 16, "y": 8, "z": 4}, + "workgroup_size": {"x": 16, "y": 16, "z": 1}, + }, + "correlation_id": { + "internal": "corr2", + "external": "ext2", + }, + }, + "records": [ + {"counter_id": {"handle": 102}, "value": 84} + ], # Only COUNTER2 + }, + ] + }, } - }] + ] } - + json_path = tmp_path / "missing_counters.json" with open(json_path, "w") as f: json.dump(json_data, f) - + csv_path = tmp_path / "missing_counters_output.csv" - - monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: { - "corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][0], - "corr2": json_data["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][1] - }) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: { - 1: json_data["rocprofiler-sdk-tool"][0]["agents"][0], - 2: json_data["rocprofiler-sdk-tool"][0]["agents"][1] - }) + + monkeypatch.setattr( + utils, + "v3_json_get_dispatches", + lambda data: { + "corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][0], + "corr2": json_data["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][1], + }, + ) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: { + 1: json_data["rocprofiler-sdk-tool"][0]["agents"][0], + 2: json_data["rocprofiler-sdk-tool"][0]["agents"][1], + }, + ) monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0, 2: 1}) - monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: { - (1, 101): {"name": "COUNTER1"}, - (2, 102): {"name": "COUNTER2"} - }) - + monkeypatch.setattr( + utils, + "v3_json_get_counters", + lambda data: {(1, 101): {"name": "COUNTER1"}, (2, 102): {"name": "COUNTER2"}}, + ) + utils.v3_json_to_csv(json_path, csv_path) - + assert csv_path.exists() df = pd.read_csv(csv_path) - + assert len(df) == 2 - + assert "COUNTER1" in df.columns assert "COUNTER2" in df.columns - + assert df["COUNTER1"][0] == 42 assert pd.isna(df["COUNTER2"][0]) - + assert pd.isna(df["COUNTER1"][1]) assert df["COUNTER2"][1] == 84 + # ============================================================================= # RESOURCE ALLOCATION TESTS # ============================================================================= + def test_check_resource_allocation_no_ctest(monkeypatch): """ Test check_resource_allocation when CTEST_RESOURCE_GROUP_COUNT is not set. Should return without setting HIP_VISIBLE_DEVICES. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying environment """ monkeypatch.delenv("CTEST_RESOURCE_GROUP_COUNT", raising=False) monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False) - + from tests.test_utils import check_resource_allocation - + result = check_resource_allocation() - + assert result is None assert "HIP_VISIBLE_DEVICES" not in os.environ + def test_check_resource_allocation_with_gpu_resource(monkeypatch): """ Test check_resource_allocation when CTEST resource allocation is enabled with GPU resource. Should extract GPU ID and set HIP_VISIBLE_DEVICES. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying environment """ @@ -1659,55 +2092,59 @@ def test_check_resource_allocation_with_gpu_resource(monkeypatch): monkeypatch.setenv("CTEST_RESOURCE_GROUP_0_GPUS", "id:2,slots:1") monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False) from tests.test_utils import check_resource_allocation - + result = check_resource_allocation() - + assert result is None assert os.environ["HIP_VISIBLE_DEVICES"] == "2" + def test_check_resource_allocation_no_gpu_resource(monkeypatch): """ Test check_resource_allocation when CTEST is enabled but no GPU resource is specified. Should return without setting HIP_VISIBLE_DEVICES. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying environment """ monkeypatch.setenv("CTEST_RESOURCE_GROUP_COUNT", "1") monkeypatch.delenv("CTEST_RESOURCE_GROUP_0_GPUS", raising=False) monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False) - + from tests.test_utils import check_resource_allocation - + result = check_resource_allocation() - + assert result is None assert "HIP_VISIBLE_DEVICES" not in os.environ + def test_check_resource_allocation_malformed_resource(monkeypatch): """ Test check_resource_allocation with malformed CTEST_RESOURCE_GROUP_0_GPUS format. Should handle gracefully without crashing. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying environment """ monkeypatch.setenv("CTEST_RESOURCE_GROUP_COUNT", "1") monkeypatch.setenv("CTEST_RESOURCE_GROUP_0_GPUS", "malformed_resource_string") monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising=False) - + from tests.test_utils import check_resource_allocation - + try: result = check_resource_allocation() assert result is None except (ValueError, IndexError): pass + # ============================================================================= # FILE PATTERN MATCHING TESTS # ============================================================================= + def test_check_file_pattern_match_found(): """ Test check_file_pattern when the pattern is found in the file. @@ -1715,128 +2152,171 @@ def test_check_file_pattern_match_found(): """ import tempfile - - with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: f.write("This is a test file\nwith multiple lines\nand some pattern text\n") temp_file_path = f.name - + try: result = check_file_pattern("pattern", temp_file_path) assert result is True - + result = check_file_pattern(r"test.*file", temp_file_path) assert result is True - + finally: os.unlink(temp_file_path) + def test_v3_json_to_csv_complex_dispatch(tmp_path, monkeypatch): """ Test v3_json_to_csv with a more complex dispatch scenario including multiple dispatches and 3D grid/workgroup sizes. - + Args: tmp_path (pathlib.Path): Temporary directory for test files monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ - + complex_json = { - "rocprofiler-sdk-tool": [{ - "metadata": {"pid": 12345}, - "agents": [ - {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}, - {"id": {"handle": 2}, "type": 2, "node_id": 1, "wave_front_size": 32} - ], - "counters": [ - {"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER1"}, - {"id": {"handle": 102}, "agent_id": {"handle": 1}, "name": "COUNTER2"} - ], - "kernel_symbols": { - "kernel1": {"formatted_kernel_name": "Kernel1", "private_segment_size": 16}, - "kernel2": {"formatted_kernel_name": "Kernel2", "private_segment_size": 32} - }, - "buffer_records": { - "kernel_dispatch": [ - {"correlation_id": {"internal": "corr1"}, "start_timestamp": 100, "end_timestamp": 200}, - {"correlation_id": {"internal": "corr2"}, "start_timestamp": 300, "end_timestamp": 400} - ] - }, - "callback_records": { - "counter_collection": [ + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 12345}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64}, + {"id": {"handle": 2}, "type": 2, "node_id": 1, "wave_front_size": 32}, + ], + "counters": [ { - "thread_id": 67890, - "lds_block_size_v": 64, - "arch_vgpr_count": 32, - "sgpr_count": 16, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 1, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 2}, - "kernel_id": "kernel1", - "grid_size": {"x": 2, "y": 3, "z": 4}, - "workgroup_size": {"x": 8, "y": 4, "z": 2} - }, - "correlation_id": {"internal": "corr1", "external": "ext1"} - }, - "records": [ - {"counter_id": {"handle": 101}, "value": 42}, - {"counter_id": {"handle": 102}, "value": 24} - ] + "id": {"handle": 101}, + "agent_id": {"handle": 1}, + "name": "COUNTER1", }, { - "thread_id": 67891, - "lds_block_size_v": 128, - "arch_vgpr_count": 64, - "sgpr_count": 32, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 2, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 3}, - "kernel_id": "kernel2", - "grid_size": {"x": 16, "y": 8, "z": 4}, - "workgroup_size": {"x": 16, "y": 16, "z": 1} - }, - "correlation_id": {"internal": "corr2", "external": "ext2"} + "id": {"handle": 102}, + "agent_id": {"handle": 1}, + "name": "COUNTER2", + }, + ], + "kernel_symbols": { + "kernel1": { + "formatted_kernel_name": "Kernel1", + "private_segment_size": 16, + }, + "kernel2": { + "formatted_kernel_name": "Kernel2", + "private_segment_size": 32, + }, + }, + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": "corr1"}, + "start_timestamp": 100, + "end_timestamp": 200, }, - "records": [ - {"counter_id": {"handle": 101}, "value": 84}, - {"counter_id": {"handle": 102}, "value": 36} - ] - } - ] + { + "correlation_id": {"internal": "corr2"}, + "start_timestamp": 300, + "end_timestamp": 400, + }, + ] + }, + "callback_records": { + "counter_collection": [ + { + "thread_id": 67890, + "lds_block_size_v": 64, + "arch_vgpr_count": 32, + "sgpr_count": 16, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 1, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 2}, + "kernel_id": "kernel1", + "grid_size": {"x": 2, "y": 3, "z": 4}, + "workgroup_size": {"x": 8, "y": 4, "z": 2}, + }, + "correlation_id": { + "internal": "corr1", + "external": "ext1", + }, + }, + "records": [ + {"counter_id": {"handle": 101}, "value": 42}, + {"counter_id": {"handle": 102}, "value": 24}, + ], + }, + { + "thread_id": 67891, + "lds_block_size_v": 128, + "arch_vgpr_count": 64, + "sgpr_count": 32, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 2, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 3}, + "kernel_id": "kernel2", + "grid_size": {"x": 16, "y": 8, "z": 4}, + "workgroup_size": {"x": 16, "y": 16, "z": 1}, + }, + "correlation_id": { + "internal": "corr2", + "external": "ext2", + }, + }, + "records": [ + {"counter_id": {"handle": 101}, "value": 84}, + {"counter_id": {"handle": 102}, "value": 36}, + ], + }, + ] + }, } - }] + ] } - + json_path = tmp_path / "complex.json" with open(json_path, "w") as f: json.dump(complex_json, f) - + csv_path = tmp_path / "complex_output.csv" - - monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: { - "corr1": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][0], - "corr2": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][1] - }) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: { - 1: complex_json["rocprofiler-sdk-tool"][0]["agents"][0], - 2: complex_json["rocprofiler-sdk-tool"][0]["agents"][1] - }) + + monkeypatch.setattr( + utils, + "v3_json_get_dispatches", + lambda data: { + "corr1": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][0], + "corr2": complex_json["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][1], + }, + ) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: { + 1: complex_json["rocprofiler-sdk-tool"][0]["agents"][0], + 2: complex_json["rocprofiler-sdk-tool"][0]["agents"][1], + }, + ) monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0, 2: 1}) - monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: { - (1, 101): {"name": "COUNTER1"}, - (1, 102): {"name": "COUNTER2"} - }) - + monkeypatch.setattr( + utils, + "v3_json_get_counters", + lambda data: {(1, 101): {"name": "COUNTER1"}, (1, 102): {"name": "COUNTER2"}}, + ) + utils.v3_json_to_csv(json_path, csv_path) - + assert csv_path.exists() df = pd.read_csv(csv_path) - + assert len(df) == 2 - + assert df["Grid_Size"][0] == 24 assert df["Workgroup_Size"][0] == 64 assert df["Kernel_Name"][0] == "Kernel1" @@ -1844,7 +2324,7 @@ def test_v3_json_to_csv_complex_dispatch(tmp_path, monkeypatch): assert df["COUNTER2"][0] == 24 assert df["GPU_ID"][0] == 0 assert df["Wave_Size"][0] == 64 - + assert df["Grid_Size"][1] == 512 assert df["Workgroup_Size"][1] == 256 assert df["Kernel_Name"][1] == "Kernel2" @@ -1852,114 +2332,165 @@ def test_v3_json_to_csv_complex_dispatch(tmp_path, monkeypatch): assert df["COUNTER2"][1] == 36 assert df["GPU_ID"][1] == 0 assert df["Wave_Size"][1] == 64 + + def test_v3_json_to_csv_missing_counters_handling(tmp_path, monkeypatch): """ Test v3_json_to_csv handles cases where different dispatches have different sets of counters. This addresses the DataFrame creation issue where arrays have different lengths. - + Args: tmp_path (pathlib.Path): Temporary directory for test files monkeypatch (pytest.MonkeyPatch): Pytest fixture for modifying behavior """ - + json_data = { - "rocprofiler-sdk-tool": [{ - "metadata": {"pid": 12345}, - "agents": [ - {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64} - ], - "counters": [ - {"id": {"handle": 101}, "agent_id": {"handle": 1}, "name": "COUNTER1"}, - {"id": {"handle": 102}, "agent_id": {"handle": 1}, "name": "COUNTER2"} - ], - "kernel_symbols": { - "kernel1": {"formatted_kernel_name": "Kernel1", "private_segment_size": 16}, - "kernel2": {"formatted_kernel_name": "Kernel2", "private_segment_size": 32} - }, - "buffer_records": { - "kernel_dispatch": [ - {"correlation_id": {"internal": "corr1"}, "start_timestamp": 100, "end_timestamp": 200}, - {"correlation_id": {"internal": "corr2"}, "start_timestamp": 300, "end_timestamp": 400} - ] - }, - "callback_records": { - "counter_collection": [ + "rocprofiler-sdk-tool": [ + { + "metadata": {"pid": 12345}, + "agents": [ + {"id": {"handle": 1}, "type": 2, "node_id": 0, "wave_front_size": 64} + ], + "counters": [ { - "thread_id": 67890, - "lds_block_size_v": 64, - "arch_vgpr_count": 32, - "sgpr_count": 16, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 1, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 2}, - "kernel_id": "kernel1", - "grid_size": {"x": 2, "y": 3, "z": 4}, - "workgroup_size": {"x": 8, "y": 4, "z": 2} - }, - "correlation_id": {"internal": "corr1", "external": "ext1"} - }, - "records": [{"counter_id": {"handle": 101}, "value": 42}] # Only COUNTER1 + "id": {"handle": 101}, + "agent_id": {"handle": 1}, + "name": "COUNTER1", }, { - "thread_id": 67891, - "lds_block_size_v": 128, - "arch_vgpr_count": 64, - "sgpr_count": 32, - "dispatch_data": { - "dispatch_info": { - "dispatch_id": 2, - "agent_id": {"handle": 1}, - "queue_id": {"handle": 3}, - "kernel_id": "kernel2", - "grid_size": {"x": 16, "y": 8, "z": 4}, - "workgroup_size": {"x": 16, "y": 16, "z": 1} - }, - "correlation_id": {"internal": "corr2", "external": "ext2"} + "id": {"handle": 102}, + "agent_id": {"handle": 1}, + "name": "COUNTER2", + }, + ], + "kernel_symbols": { + "kernel1": { + "formatted_kernel_name": "Kernel1", + "private_segment_size": 16, + }, + "kernel2": { + "formatted_kernel_name": "Kernel2", + "private_segment_size": 32, + }, + }, + "buffer_records": { + "kernel_dispatch": [ + { + "correlation_id": {"internal": "corr1"}, + "start_timestamp": 100, + "end_timestamp": 200, }, - "records": [{"counter_id": {"handle": 102}, "value": 84}] # Only COUNTER2 - } - ] + { + "correlation_id": {"internal": "corr2"}, + "start_timestamp": 300, + "end_timestamp": 400, + }, + ] + }, + "callback_records": { + "counter_collection": [ + { + "thread_id": 67890, + "lds_block_size_v": 64, + "arch_vgpr_count": 32, + "sgpr_count": 16, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 1, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 2}, + "kernel_id": "kernel1", + "grid_size": {"x": 2, "y": 3, "z": 4}, + "workgroup_size": {"x": 8, "y": 4, "z": 2}, + }, + "correlation_id": { + "internal": "corr1", + "external": "ext1", + }, + }, + "records": [ + {"counter_id": {"handle": 101}, "value": 42} + ], # Only COUNTER1 + }, + { + "thread_id": 67891, + "lds_block_size_v": 128, + "arch_vgpr_count": 64, + "sgpr_count": 32, + "dispatch_data": { + "dispatch_info": { + "dispatch_id": 2, + "agent_id": {"handle": 1}, + "queue_id": {"handle": 3}, + "kernel_id": "kernel2", + "grid_size": {"x": 16, "y": 8, "z": 4}, + "workgroup_size": {"x": 16, "y": 16, "z": 1}, + }, + "correlation_id": { + "internal": "corr2", + "external": "ext2", + }, + }, + "records": [ + {"counter_id": {"handle": 102}, "value": 84} + ], # Only COUNTER2 + }, + ] + }, } - }] + ] } - + json_path = tmp_path / "missing_counters.json" with open(json_path, "w") as f: json.dump(json_data, f) - + csv_path = tmp_path / "missing_counters_output.csv" - - monkeypatch.setattr(utils, "v3_json_get_dispatches", lambda data: { - "corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][0], - "corr2": json_data["rocprofiler-sdk-tool"][0]["buffer_records"]["kernel_dispatch"][1] - }) - monkeypatch.setattr(utils, "get_agent_dict", lambda data: { - 1: json_data["rocprofiler-sdk-tool"][0]["agents"][0] - }) + + monkeypatch.setattr( + utils, + "v3_json_get_dispatches", + lambda data: { + "corr1": json_data["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][0], + "corr2": json_data["rocprofiler-sdk-tool"][0]["buffer_records"][ + "kernel_dispatch" + ][1], + }, + ) + monkeypatch.setattr( + utils, + "get_agent_dict", + lambda data: {1: json_data["rocprofiler-sdk-tool"][0]["agents"][0]}, + ) monkeypatch.setattr(utils, "get_gpuid_dict", lambda data: {1: 0}) - monkeypatch.setattr(utils, "v3_json_get_counters", lambda data: { - (1, 101): {"name": "COUNTER1"}, - (1, 102): {"name": "COUNTER2"} - }) - + monkeypatch.setattr( + utils, + "v3_json_get_counters", + lambda data: {(1, 101): {"name": "COUNTER1"}, (1, 102): {"name": "COUNTER2"}}, + ) + try: utils.v3_json_to_csv(json_path, csv_path) - + assert csv_path.exists() df = pd.read_csv(csv_path) - + assert len(df) == 2 - + assert "COUNTER1" in df.columns assert "COUNTER2" in df.columns - + except ValueError as e: if "All arrays must be of the same length" in str(e): - pytest.skip("v3_json_to_csv does not currently handle missing counters gracefully - arrays have different lengths") + pytest.skip( + "v3_json_to_csv does not currently handle missing counters gracefully - arrays have different lengths" + ) else: raise + + def test_check_file_pattern_file_not_found(): """ Test check_file_pattern when the file doesn't exist. @@ -1968,154 +2499,176 @@ def test_check_file_pattern_file_not_found(): with pytest.raises(FileNotFoundError): check_file_pattern("pattern", "/nonexistent/file/path.txt") + # ============================================================================= # TEXT PARSING UTILITIES TESTS # ============================================================================= + def test_parse_text_basic(tmp_path): """Test parse_text with a simple valid input file. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts that counters are correctly extracted from a simple file. """ test_file = tmp_path / "test_counters.txt" test_file.write_text("pmc: counter1 counter2 counter3") - + result = utils.parse_text(str(test_file)) assert result == ["counter1", "counter2", "counter3"] + + def test_parse_text_empty_file(tmp_path): """Test parse_text with an empty file. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts that an empty file returns an empty list. """ test_file = tmp_path / "empty.txt" test_file.write_text("") - + result = utils.parse_text(str(test_file)) assert result == [] + + def test_parse_text_no_pmc_entries(tmp_path): """Test parse_text with a file that doesn't contain any 'pmc:' entries. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts that a file without 'pmc:' returns an empty list. """ test_file = tmp_path / "no_pmc.txt" test_file.write_text("line1\nline2\nline3") - + result = utils.parse_text(str(test_file)) assert result == [] + + def test_parse_text_with_comments(tmp_path): """Test parse_text with lines that have comments after the counters. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts that comments are properly stripped from counter lines. """ test_file = tmp_path / "comments.txt" test_file.write_text("pmc: counter1 counter2 # This is a comment") - + result = utils.parse_text(str(test_file)) assert result == ["counter1", "counter2"] + + def test_parse_text_multiple_lines(tmp_path): """Test parse_text with multiple 'pmc:' lines. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts counters from multiple lines are correctly combined. """ test_file = tmp_path / "multiple_lines.txt" test_file.write_text("pmc: counter1 counter2\npmc: counter3 counter4") - + result = utils.parse_text(str(test_file)) assert result == ["counter1", "counter2", "counter3", "counter4"] + + def test_parse_text_mixed_lines(tmp_path): """Test parse_text with a mix of 'pmc:' and non-'pmc:' lines. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts that only counters from 'pmc:' lines are extracted. """ test_file = tmp_path / "mixed_lines.txt" - test_file.write_text("line1\npmc: counter1 counter2\nline3\npmc: counter3 counter4\nline5") - + test_file.write_text( + "line1\npmc: counter1 counter2\nline3\npmc: counter3 counter4\nline5" + ) + result = utils.parse_text(str(test_file)) assert result == ["counter1", "counter2", "counter3", "counter4"] + + def test_parse_text_whitespace_handling(tmp_path): """Test parse_text with various whitespace combinations. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts that whitespace is properly handled in counter extraction. """ test_file = tmp_path / "whitespace.txt" test_file.write_text("pmc: counter1\t\tcounter2 counter3") - + result = utils.parse_text(str(test_file)) - + result = [item for item in result if item.strip()] - + expected = ["counter1", "counter2", "counter3"] assert result == expected - + test_file.write_text("pmc: counter1 counter2\npmc: counter3 counter4") result = utils.parse_text(str(test_file)) result = [item for item in result if item.strip()] expected = ["counter1", "counter2", "counter3", "counter4"] assert result == expected + + def test_parse_text_edge_cases(tmp_path): """Test parse_text with edge cases like empty 'pmc:' lines. - + Args: tmp_path (pathlib.Path): Temporary path fixture provided by pytest. - + Returns: None: Asserts that edge cases are handled correctly. """ test_file = tmp_path / "edge_cases.txt" test_file.write_text("pmc:\npmc: \npmc: counter1") - + result = utils.parse_text(str(test_file)) result = [item for item in result if item.strip()] assert result == ["counter1"] + + def test_parse_text_file_not_found(): """Test parse_text with a nonexistent file. - + Returns: None: Asserts that FileNotFoundError is raised for nonexistent files. """ with pytest.raises(FileNotFoundError): utils.parse_text("nonexistent_file.txt") + # ============================================================================= # RUN_PROF TESTS # ============================================================================= + def test_run_prof_success_v2(tmp_path, monkeypatch): """ Test run_prof with rocprofv2 successful execution. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts successful execution and file creation. """ @@ -2123,46 +2676,45 @@ def test_run_prof_success_v2(tmp_path, monkeypatch): fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") os.makedirs(workload_dir + "/out/pmc_1", exist_ok=True) - + csv_content = "Dispatch_ID,GPU_ID,Kernel_Name\n0,0,test_kernel" with open(workload_dir + "/out/pmc_1/results_0.csv", "w") as f: f.write(csv_content) - + class MockSpec: def __init__(self): self.gpu_model = "mi250x" self._l2_banks = 32 - + mspec = MockSpec() - + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv2") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: False) monkeypatch.setattr("utils.utils.using_v1", lambda: False) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - monkeypatch.setattr("glob.glob", lambda pattern: [workload_dir + "/out/pmc_1/results_0.csv"]) - - import utils.utils as utils_mod - - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" + monkeypatch.setattr( + "glob.glob", lambda pattern: [workload_dir + "/out/pmc_1/results_0.csv"] ) - + + import utils.utils as utils_mod + + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + assert Path(workload_dir + "/test.csv").exists() + + def test_run_prof_success_v3_csv(tmp_path, monkeypatch): """ Test run_prof with rocprofv3 using CSV format. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts successful execution with v3 CSV processing. """ @@ -2170,92 +2722,85 @@ def test_run_prof_success_v3_csv(tmp_path, monkeypatch): fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") os.makedirs(workload_dir + "/out/pmc_1", exist_ok=True) - + class MockSpec: def __init__(self): self.gpu_model = "mi300x" self._l2_banks = 32 - + mspec = MockSpec() - + csv_files = [workload_dir + "/out/pmc_1/converted.csv"] - + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv3") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: True) monkeypatch.setattr("utils.utils.using_v1", lambda: False) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.process_rocprofv3_output", lambda *a, **k: csv_files) - + mock_df = pd.DataFrame({"Dispatch_ID": [0], "GPU_ID": [0], "Kernel_Name": ["test"]}) monkeypatch.setattr("pandas.read_csv", lambda *a, **k: mock_df) monkeypatch.setattr("pandas.concat", lambda *a, **k: mock_df) - + import utils.utils as utils_mod - - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" - ) + + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + + def test_run_prof_success_rocprofiler_sdk(tmp_path, monkeypatch): """ Test run_prof with rocprofiler-sdk execution. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts successful execution with SDK configuration. """ fname = tmp_path / "test.txt" fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") - + class MockSpec: def __init__(self): self.gpu_model = "mi300x" self._l2_banks = 32 - + mspec = MockSpec() - - profiler_options = { - "APP_CMD": ["./test_app"], - "ROCPROF_OUTPUT_PATH": workload_dir - } - + + profiler_options = {"APP_CMD": ["./test_app"], "ROCPROF_OUTPUT_PATH": workload_dir} + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofiler-sdk") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: True) monkeypatch.setattr("utils.utils.parse_text", lambda f: ["SQ_WAVES"]) monkeypatch.setattr("utils.utils.process_rocprofv3_output", lambda *a, **k: []) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_warning", lambda *a, **k: None) - + import utils.utils as utils_mod - + utils_mod.run_prof( - str(fname), - profiler_options, - workload_dir, - mspec, - logging.INFO, - "csv" + str(fname), profiler_options, workload_dir, mspec, logging.INFO, "csv" ) + + def test_run_prof_with_yaml_config(tmp_path, monkeypatch): """ Test run_prof with additional YAML configuration file. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts YAML config is properly handled. """ @@ -2264,292 +2809,282 @@ def test_run_prof_with_yaml_config(tmp_path, monkeypatch): yaml_file = tmp_path / "test.yaml" yaml_file.write_text("counters:\n - TCC_HIT") workload_dir = str(tmp_path / "workload") - + class MockSpec: def __init__(self): self.gpu_model = "mi300x" self._l2_banks = 32 - + mspec = MockSpec() - + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv3") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: True) monkeypatch.setattr("utils.utils.using_v1", lambda: False) monkeypatch.setattr("utils.utils.process_rocprofv3_output", lambda *a, **k: []) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_warning", lambda *a, **k: None) - + import utils.utils as utils_mod - - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" - ) + + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + + def test_run_prof_failure_subprocess(tmp_path, monkeypatch): """ Test run_prof when subprocess execution fails. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts proper error handling on subprocess failure. """ fname = tmp_path / "test.txt" fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") - + class MockSpec: def __init__(self): self.gpu_model = "mi250x" self._l2_banks = 32 - + mspec = MockSpec() - + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv3") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (False, "error output")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (False, "error output") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: True) monkeypatch.setattr("utils.utils.using_v1", lambda: False) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + def mock_console_error(msg, exit=True): if exit: raise RuntimeError("console_error called") - + monkeypatch.setattr("utils.utils.console_error", mock_console_error) - + import utils.utils as utils_mod - + with pytest.raises(RuntimeError, match="console_error called"): utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" + str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv" ) + + def test_run_prof_mi300_environment_setup(tmp_path, monkeypatch): """ Test run_prof sets proper environment variables for MI300 series GPUs. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts MI300 environment variable is set correctly. """ fname = tmp_path / "test.txt" fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") - + class MockSpec: def __init__(self): - self.gpu_model = "mi300x" + self.gpu_model = "mi300x" self._l2_banks = 32 - + mspec = MockSpec() - + captured_env = {} - + def mock_capture_subprocess_output(cmd, new_env=None, **kwargs): if new_env: captured_env.update(new_env) return (True, "success") - + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv3") - monkeypatch.setattr("utils.utils.capture_subprocess_output", mock_capture_subprocess_output) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", mock_capture_subprocess_output + ) monkeypatch.setattr("utils.utils.using_v3", lambda: True) monkeypatch.setattr("utils.utils.using_v1", lambda: False) monkeypatch.setattr("utils.utils.process_rocprofv3_output", lambda *a, **k: []) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_warning", lambda *a, **k: None) - + import utils.utils as utils_mod - - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" - ) - + + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + assert "ROCPROFILER_INDIVIDUAL_XCC_MODE" in captured_env assert captured_env["ROCPROFILER_INDIVIDUAL_XCC_MODE"] == "1" + + def test_run_prof_timestamps_special_case(tmp_path, monkeypatch): """ Test run_prof handles timestamps.txt special case correctly. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts timestamps processing is handled correctly. """ fname = tmp_path / "timestamps.txt" fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") - + os.makedirs(workload_dir + "/out/pmc_1", exist_ok=True) - + class MockSpec: def __init__(self): self.gpu_model = "mi250x" self._l2_banks = 32 - + mspec = MockSpec() - + csv_content = "Dispatch_ID,Start_Timestamp,End_Timestamp\n0,100,200" with open(workload_dir + "/kernel_trace.csv", "w") as f: f.write(csv_content) - + csv_files = [workload_dir + "/kernel_trace.csv"] - + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv3") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: True) monkeypatch.setattr("utils.utils.using_v1", lambda: False) monkeypatch.setattr("utils.utils.process_rocprofv3_output", lambda *a, **k: csv_files) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_warning", lambda *a, **k: None) - - mock_df = pd.DataFrame({"Dispatch_ID": [0], "Start_Timestamp": [100], "End_Timestamp": [200]}) + + mock_df = pd.DataFrame( + {"Dispatch_ID": [0], "Start_Timestamp": [100], "End_Timestamp": [200]} + ) monkeypatch.setattr("pandas.read_csv", lambda *a, **k: mock_df) monkeypatch.setattr("pandas.concat", lambda *a, **k: mock_df) - + import utils.utils as utils_mod - - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" - ) + + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + + def test_run_prof_no_results_files(tmp_path, monkeypatch): """ Test run_prof when no results files are generated. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts proper handling when no results are found. """ fname = tmp_path / "test.txt" fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") - + class MockSpec: def __init__(self): self.gpu_model = "mi250x" self._l2_banks = 32 - + mspec = MockSpec() - + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv2") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: False) monkeypatch.setattr("utils.utils.using_v1", lambda: False) monkeypatch.setattr("glob.glob", lambda pattern: []) # No files found monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod - - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" - ) + + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + + def test_run_prof_header_standardization(tmp_path, monkeypatch): """ Test run_prof properly standardizes CSV headers. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts CSV headers are standardized correctly. """ fname = tmp_path / "test.txt" fname.write_text("pmc: SQ_WAVES") workload_dir = str(tmp_path / "workload") - + os.makedirs(workload_dir + "/out/pmc_1", exist_ok=True) - + class MockSpec: def __init__(self): self.gpu_model = "mi250x" self._l2_banks = 32 - + mspec = MockSpec() - + csv_content = "KernelName,Index,grd,gpu-id,BeginNs,EndNs\ntest_kernel,0,64,0,100,200" with open(workload_dir + "/out/pmc_1/results_test.csv", "w") as f: f.write(csv_content) - - old_headers_df = pd.DataFrame({ - "KernelName": ["test_kernel"], - "Index": [0], - "grd": [64], - "gpu-id": [0], - "BeginNs": [100], - "EndNs": [200] - }) - + + old_headers_df = pd.DataFrame( + { + "KernelName": ["test_kernel"], + "Index": [0], + "grd": [64], + "gpu-id": [0], + "BeginNs": [100], + "EndNs": [200], + } + ) + monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv2") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: False) monkeypatch.setattr("utils.utils.using_v1", lambda: False) - monkeypatch.setattr("glob.glob", lambda pattern: [workload_dir + "/out/pmc_1/results_test.csv"]) + monkeypatch.setattr( + "glob.glob", lambda pattern: [workload_dir + "/out/pmc_1/results_test.csv"] + ) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + read_calls = [] + def mock_read_csv(path, **kwargs): read_calls.append(path) return old_headers_df.copy() - + write_calls = [] + def mock_to_csv(self, path, **kwargs): write_calls.append((path, self.columns.tolist())) - + monkeypatch.setattr("pandas.read_csv", mock_read_csv) monkeypatch.setattr("pandas.DataFrame.to_csv", mock_to_csv) monkeypatch.setattr("pandas.concat", lambda dfs, **k: old_headers_df.copy()) - + import utils.utils as utils_mod - - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" - ) - + + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + final_headers = write_calls[-1][1] if write_calls else [] assert "Kernel_Name" in final_headers assert "Dispatch_ID" in final_headers @@ -2557,439 +3092,509 @@ def test_run_prof_header_standardization(tmp_path, monkeypatch): assert "GPU_ID" in final_headers assert "Start_Timestamp" in final_headers assert "End_Timestamp" in final_headers + + def test_run_prof_tcc_flattening_mi300(tmp_path, monkeypatch): """ Test run_prof applies TCC flattening for MI300 series GPUs. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts TCC flattening is applied for MI300 GPUs. """ fname = tmp_path / "test.txt" fname.write_text("pmc: TCC_HIT[0]") workload_dir = str(tmp_path / "workload") - + class MockSpec: def __init__(self): self.gpu_model = "mi300x" self.gpu_arch = "gfx942" self.compute_partition = "SPX" self._l2_banks = 32 - + mspec = MockSpec() - + flatten_called = False + def mock_flatten_tcc_info_across_xcds(file, xcds, l2_banks): nonlocal flatten_called flatten_called = True - return pd.DataFrame({"Dispatch_ID": [0], "TCC_HIT[0]": [100], "TCC_HIT[16]": [200]}) - + return pd.DataFrame( + {"Dispatch_ID": [0], "TCC_HIT[0]": [100], "TCC_HIT[16]": [200]} + ) + # Mock functions monkeypatch.setattr("utils.utils.rocprof_cmd", "rocprofv2") - monkeypatch.setattr("utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success")) + monkeypatch.setattr( + "utils.utils.capture_subprocess_output", lambda *a, **k: (True, "success") + ) monkeypatch.setattr("utils.utils.using_v3", lambda: False) monkeypatch.setattr("utils.utils.using_v1", lambda: False) - monkeypatch.setattr("utils.utils.flatten_tcc_info_across_xcds", mock_flatten_tcc_info_across_xcds) + monkeypatch.setattr( + "utils.utils.flatten_tcc_info_across_xcds", mock_flatten_tcc_info_across_xcds + ) monkeypatch.setattr("utils.utils.mi_gpu_specs.get_num_xcds", lambda *a: 2) monkeypatch.setattr("glob.glob", lambda pattern: [workload_dir + "/results_test.csv"]) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + # Mock pandas mock_df = pd.DataFrame({"Dispatch_ID": [0], "TCC_HIT[0]": [100]}) monkeypatch.setattr("pandas.read_csv", lambda *a, **k: mock_df) monkeypatch.setattr("pandas.concat", lambda *a, **k: mock_df) monkeypatch.setattr("pandas.DataFrame.to_csv", lambda self, *a, **k: None) - + import utils.utils as utils_mod - + # Execute function - utils_mod.run_prof( - str(fname), - ["--arg"], - workload_dir, - mspec, - logging.INFO, - "csv" - ) - + utils_mod.run_prof(str(fname), ["--arg"], workload_dir, mspec, logging.INFO, "csv") + assert flatten_called + # ============================================================================= # ROCPROFV3 OUTPUT PROCESSING TESTS # ============================================================================= + def test_process_rocprofv3_output_json_format(tmp_path, monkeypatch): """ Test process_rocprofv3_output with json format converts JSON files to CSV. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts CSV files are created from JSON files. """ workload_dir = str(tmp_path) output_dir = tmp_path / "out" / "pmc_1" / "subdir" output_dir.mkdir(parents=True) - + json_file1 = output_dir / "test1.json" json_file2 = output_dir / "test2.json" json_file1.write_text('{"test": "data1"}') json_file2.write_text('{"test": "data2"}') - + monkeypatch.setattr("glob.glob", lambda pattern: [str(json_file1), str(json_file2)]) - + def mock_v3_json_to_csv(json_path, csv_path): Path(csv_path).write_text("csv,data\ntest,value") - + monkeypatch.setattr("utils.utils.v3_json_to_csv", mock_v3_json_to_csv) - + import utils.utils as utils_mod + result = utils_mod.process_rocprofv3_output("json", workload_dir, False) - + assert len(result) == 2 csv_file1 = output_dir / "test1.csv" csv_file2 = output_dir / "test2.csv" assert csv_file1.exists() assert csv_file2.exists() + + def test_process_rocprofv3_output_csv_format_with_counter_files(tmp_path, monkeypatch): """ Test process_rocprofv3_output with csv format processes counter collection files. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts counter files are converted properly. """ workload_dir = str(tmp_path) output_dir = tmp_path / "out" / "pmc_1" / "subdir" output_dir.mkdir(parents=True) - + counter_file = output_dir / "test_counter_collection.csv" agent_file = output_dir / "test_agent_info.csv" converted_file = output_dir / "test_converted.csv" - + counter_file.write_text("counter,data\ntest,value") agent_file.write_text("agent,data\ntest,value") - + def mock_glob(pattern): if "_counter_collection.csv" in pattern: return [str(counter_file)] elif "_converted.csv" in pattern: return [str(converted_file)] return [] - + monkeypatch.setattr("glob.glob", mock_glob) - + def mock_v3_counter_csv_to_v2_csv(counter_path, agent_path, output_path): Path(output_path).write_text("converted,data\ntest,value") - - monkeypatch.setattr("utils.utils.v3_counter_csv_to_v2_csv", mock_v3_counter_csv_to_v2_csv) - + + monkeypatch.setattr( + "utils.utils.v3_counter_csv_to_v2_csv", mock_v3_counter_csv_to_v2_csv + ) + import utils.utils as utils_mod + result = utils_mod.process_rocprofv3_output("csv", workload_dir, False) - + assert len(result) == 1 assert str(converted_file) in result + + def test_process_rocprofv3_output_csv_format_conversion_error(tmp_path, monkeypatch): """ Test process_rocprofv3_output handles conversion errors gracefully. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts empty list returned when conversion fails. """ workload_dir = str(tmp_path) output_dir = tmp_path / "out" / "pmc_1" / "subdir" output_dir.mkdir(parents=True) - + counter_file = output_dir / "test_counter_collection.csv" agent_file = output_dir / "test_agent_info.csv" - + counter_file.write_text("counter,data\ntest,value") agent_file.write_text("agent,data\ntest,value") - + def mock_glob(pattern): if "_counter_collection.csv" in pattern: return [str(counter_file)] return [] - + monkeypatch.setattr("glob.glob", mock_glob) - + def mock_v3_counter_csv_to_v2_csv(counter_path, agent_path, output_path): raise ValueError("Conversion failed") - - monkeypatch.setattr("utils.utils.v3_counter_csv_to_v2_csv", mock_v3_counter_csv_to_v2_csv) - + + monkeypatch.setattr( + "utils.utils.v3_counter_csv_to_v2_csv", mock_v3_counter_csv_to_v2_csv + ) + warnings = [] monkeypatch.setattr("utils.utils.console_warning", lambda msg: warnings.append(msg)) - + import utils.utils as utils_mod + result = utils_mod.process_rocprofv3_output("csv", workload_dir, False) - + assert result == [] assert len(warnings) == 1 assert "Error converting" in warnings[0] + + def test_process_rocprofv3_output_csv_format_missing_agent_file(tmp_path, monkeypatch): """ Test process_rocprofv3_output raises error when agent info file is missing. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts ValueError is raised for missing agent file. """ workload_dir = str(tmp_path) output_dir = tmp_path / "out" / "pmc_1" / "subdir" output_dir.mkdir(parents=True) - + counter_file = output_dir / "test_counter_collection.csv" counter_file.write_text("counter,data\ntest,value") - + def mock_glob(pattern): if "_counter_collection.csv" in pattern: return [str(counter_file)] return [] - + monkeypatch.setattr("glob.glob", mock_glob) - + import utils.utils as utils_mod + with pytest.raises(ValueError, match='has no coresponding "agent info" file'): utils_mod.process_rocprofv3_output("csv", workload_dir, False) + + def test_process_rocprofv3_output_csv_format_timestamps_fallback(tmp_path, monkeypatch): """ Test process_rocprofv3_output falls back to kernel trace files for timestamps. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts kernel trace files are used when is_timestamps is True. """ workload_dir = str(tmp_path) output_dir = tmp_path / "out" / "pmc_1" / "subdir" output_dir.mkdir(parents=True) - + trace_file = output_dir / "test_kernel_trace.csv" trace_file.write_text("kernel,trace\ntest,data") - + def mock_glob(pattern): if "_counter_collection.csv" in pattern: - return [] + return [] elif "_kernel_trace.csv" in pattern: return [str(trace_file)] return [] - + monkeypatch.setattr("glob.glob", mock_glob) - + import utils.utils as utils_mod + result = utils_mod.process_rocprofv3_output("csv", workload_dir, True) - + assert len(result) == 1 assert str(trace_file) in result -def test_process_rocprofv3_output_csv_format_no_files_non_timestamps(tmp_path, monkeypatch): + + +def test_process_rocprofv3_output_csv_format_no_files_non_timestamps( + tmp_path, monkeypatch +): """ Test process_rocprofv3_output returns empty list when no files found for non-timestamps. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts empty list returned when no counter files exist. """ workload_dir = str(tmp_path) - + monkeypatch.setattr("glob.glob", lambda pattern: []) - + import utils.utils as utils_mod + result = utils_mod.process_rocprofv3_output("csv", workload_dir, False) - + assert result == [] + + def test_process_rocprofv3_output_invalid_format(monkeypatch): """ Test process_rocprofv3_output raises error for invalid output format. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts console_error is called for invalid format. """ + def mock_console_error(msg): raise RuntimeError(f"console_error: {msg}") - + monkeypatch.setattr("utils.utils.console_error", mock_console_error) - + import utils.utils as utils_mod - with pytest.raises(RuntimeError, match="The output file of rocprofv3 can only support json or csv"): + + with pytest.raises( + RuntimeError, match="The output file of rocprofv3 can only support json or csv" + ): utils_mod.process_rocprofv3_output("invalid", "/tmp", False) + + def test_process_rocprofv3_output_json_format_no_files(tmp_path, monkeypatch): """ Test process_rocprofv3_output with json format when no JSON files exist. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts empty list returned when no JSON files found. """ workload_dir = str(tmp_path) - + monkeypatch.setattr("glob.glob", lambda pattern: []) - + import utils.utils as utils_mod + result = utils_mod.process_rocprofv3_output("json", workload_dir, False) - + assert result == [] -def test_process_rocprofv3_output_csv_format_multiple_counter_files(tmp_path, monkeypatch): + + +def test_process_rocprofv3_output_csv_format_multiple_counter_files( + tmp_path, monkeypatch +): """ Test process_rocprofv3_output processes multiple counter collection files. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts multiple counter files are processed correctly. """ workload_dir = str(tmp_path) output_dir = tmp_path / "out" / "pmc_1" / "subdir" output_dir.mkdir(parents=True) - + counter_file1 = output_dir / "test1_counter_collection.csv" agent_file1 = output_dir / "test1_agent_info.csv" converted_file1 = output_dir / "test1_converted.csv" - + counter_file2 = output_dir / "test2_counter_collection.csv" agent_file2 = output_dir / "test2_agent_info.csv" converted_file2 = output_dir / "test2_converted.csv" - + counter_file1.write_text("counter,data\ntest1,value1") agent_file1.write_text("agent,data\ntest1,value1") counter_file2.write_text("counter,data\ntest2,value2") agent_file2.write_text("agent,data\ntest2,value2") - + def mock_glob(pattern): if "_counter_collection.csv" in pattern: return [str(counter_file1), str(counter_file2)] elif "_converted.csv" in pattern: return [str(converted_file1), str(converted_file2)] return [] - + monkeypatch.setattr("glob.glob", mock_glob) - + def mock_v3_counter_csv_to_v2_csv(counter_path, agent_path, output_path): Path(output_path).write_text(f"converted,data\n{Path(counter_path).stem},value") - - monkeypatch.setattr("utils.utils.v3_counter_csv_to_v2_csv", mock_v3_counter_csv_to_v2_csv) - + + monkeypatch.setattr( + "utils.utils.v3_counter_csv_to_v2_csv", mock_v3_counter_csv_to_v2_csv + ) + import utils.utils as utils_mod + result = utils_mod.process_rocprofv3_output("csv", workload_dir, False) - + assert len(result) == 2 assert str(converted_file1) in result assert str(converted_file2) in result + + def test_capture_subprocess_output_failure(monkeypatch): """ Test capture_subprocess_output returns (False, output) when subprocess exits with non-zero code. """ + class DummyProcess: def __init__(self): self.stdout = io.StringIO("error message\n") + def poll(self): return 1 # non-zero exit code + def wait(self): return 1 - + monkeypatch.setattr("subprocess.Popen", lambda *a, **k: DummyProcess()) - monkeypatch.setattr("selectors.DefaultSelector", lambda: mock.Mock(register=mock.Mock(), select=lambda: [], close=mock.Mock())) + monkeypatch.setattr( + "selectors.DefaultSelector", + lambda: mock.Mock(register=mock.Mock(), select=lambda: [], close=mock.Mock()), + ) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + import utils.utils as utils_mod + success, output = utils_mod.capture_subprocess_output(["false"]) - + assert success is False + + def test_capture_subprocess_output_with_logging_disabled(monkeypatch): """ Test capture_subprocess_output with enable_logging=False doesn't call console_log. """ + class DummyProcess: def __init__(self): self.stdout = io.StringIO("test output\n") + def poll(self): return 0 + def wait(self): return 0 - + monkeypatch.setattr("subprocess.Popen", lambda *a, **k: DummyProcess()) - monkeypatch.setattr("selectors.DefaultSelector", lambda: mock.Mock(register=mock.Mock(), select=lambda: [], close=mock.Mock())) - + monkeypatch.setattr( + "selectors.DefaultSelector", + lambda: mock.Mock(register=mock.Mock(), select=lambda: [], close=mock.Mock()), + ) + log_calls = [] - monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: log_calls.append((a, k))) + monkeypatch.setattr( + "utils.utils.console_log", lambda *a, **k: log_calls.append((a, k)) + ) monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + import utils.utils as utils_mod - success, output = utils_mod.capture_subprocess_output(["echo", "test"], enable_logging=False) - + + success, output = utils_mod.capture_subprocess_output( + ["echo", "test"], enable_logging=False + ) + assert success is True assert len(log_calls) == 0 + + # ============================================================================= # KOKKOS TRACE PROCESSING TESTS # ============================================================================= + def test_process_kokkos_trace_output_single_file(tmp_path, monkeypatch): """ Test process_kokkos_trace_output with a single CSV file. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that single file is processed correctly and output files are created. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "single_marker_api_trace.csv" - csv1.write_text("marker_id,marker_name,start_time,end_time\n1,kokkos_begin,1000,1050\n2,kokkos_end,2000,2010\n") - + csv1.write_text( + "marker_id,marker_name,start_time,end_time\n1,kokkos_begin,1000,1050\n2,kokkos_end,2000,2010\n" + ) + fbase = "single_test" - + import utils.utils as utils_mod + utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + # Check output file in pmc_1 directory output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 2 assert df["marker_name"].tolist() == ["kokkos_begin", "kokkos_end"] - + # Check copied file in workload directory copied_file = tmp_path / f"{fbase}_marker_api_trace.csv" assert copied_file.exists() @@ -2999,47 +3604,52 @@ def test_process_kokkos_trace_output_multiple_files(tmp_path, monkeypatch): """ Test process_kokkos_trace_output with multiple valid CSV files. Should concatenate all files and save to both output locations. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that multiple files are concatenated properly. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_warning", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub2 = out_dir / "process2" sub1.mkdir() sub2.mkdir() - + csv1 = sub1 / "test_marker_api_trace.csv" csv2 = sub2 / "test_marker_api_trace.csv" - - csv1.write_text("timestamp,marker_name,duration\n1000,kokkos_malloc,500\n2000,kokkos_parallel_for,300\n") - csv2.write_text("timestamp,marker_name,duration\n3000,kokkos_free,200\n4000,kokkos_parallel_reduce,800\n") - + + csv1.write_text( + "timestamp,marker_name,duration\n1000,kokkos_malloc,500\n2000,kokkos_parallel_for,300\n" + ) + csv2.write_text( + "timestamp,marker_name,duration\n3000,kokkos_free,200\n4000,kokkos_parallel_reduce,800\n" + ) + fbase = "test_workload" - + import utils.utils as utils_mod + utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) - assert len(df) == 4 + assert len(df) == 4 assert df["timestamp"].tolist() == [1000, 2000, 3000, 4000] assert "kokkos_malloc" in df["marker_name"].values assert "kokkos_parallel_reduce" in df["marker_name"].values - + # Check copied file copied_file = tmp_path / f"{fbase}_marker_api_trace.csv" assert copied_file.exists() @@ -3049,101 +3659,105 @@ def test_process_kokkos_trace_output_no_files_found(tmp_path, monkeypatch): """ Test process_kokkos_trace_output when no marker API trace files are found. Should handle empty file list gracefully. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that function handles empty file list without crashing. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + fbase = "no_files" - + def mock_concat(dataframes, **kwargs): if not dataframes: return pd.DataFrame() return pd.concat(dataframes, **kwargs) - + monkeypatch.setattr("pandas.concat", mock_concat) - + def mock_to_csv(self, path, **kwargs): os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: - f.write('') - + with open(path, "w") as f: + f.write("") + monkeypatch.setattr("pandas.DataFrame.to_csv", mock_to_csv) - + import utils.utils as utils_mod - + try: utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + except ValueError as e: # pandas.concat() raises ValueError when passed empty list - pytest.skip("process_kokkos_trace_output doesn't handle empty file list gracefully") + pytest.skip( + "process_kokkos_trace_output doesn't handle empty file list gracefully" + ) def test_process_kokkos_trace_output_mixed_file_states(tmp_path, monkeypatch): """ Test process_kokkos_trace_output with a mix of valid, empty, and corrupted files. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that valid files are processed while invalid ones are handled gracefully. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub2 = out_dir / "process2" sub3 = out_dir / "process3" sub1.mkdir() sub2.mkdir() sub3.mkdir() - + csv1 = sub1 / "valid_marker_api_trace.csv" csv1.write_text("timestamp,marker_name\n1000,kokkos_malloc\n2000,kokkos_free\n") - + csv2 = sub2 / "empty_marker_api_trace.csv" csv2.write_text("") - + csv3 = sub3 / "headers_marker_api_trace.csv" csv3.write_text("timestamp,marker_name\n") - + fbase = "mixed_test" - + original_read_csv = pd.read_csv + def mock_read_csv(filepath, **kwargs): try: return original_read_csv(filepath, **kwargs) except pd.errors.EmptyDataError: # Return empty DataFrame for empty files return pd.DataFrame() - + monkeypatch.setattr("pandas.read_csv", mock_read_csv) - + import utils.utils as utils_mod + utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) >= 0 @@ -3152,37 +3766,38 @@ def test_process_kokkos_trace_output_no_out_directory(tmp_path, monkeypatch): """ Test process_kokkos_trace_output when output directory doesn't exist. Should not copy file to workload directory. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that function handles missing output directory gracefully. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) - + fbase = "no_out_dir" - + monkeypatch.setattr("glob.glob", lambda pattern: []) - + def mock_concat(dataframes, **kwargs): if not dataframes: return pd.DataFrame() return pd.concat(dataframes, **kwargs) - + monkeypatch.setattr("pandas.concat", mock_concat) - + def mock_to_csv(self, path, **kwargs): os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: - f.write('') - + with open(path, "w") as f: + f.write("") + monkeypatch.setattr("pandas.DataFrame.to_csv", mock_to_csv) - + original_path = utils.path + def mock_path_exists(path_str): if path_str == workload_dir + "/out": mock_path_obj = mock.MagicMock() @@ -3190,53 +3805,56 @@ def test_process_kokkos_trace_output_no_out_directory(tmp_path, monkeypatch): return mock_path_obj else: return original_path(path_str) - + monkeypatch.setattr("utils.utils.path", mock_path_exists) - + import utils.utils as utils_mod - + try: utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + # Should not copy file to workload directory since /out doesn't exist copied_file = tmp_path / f"{fbase}_marker_api_trace.csv" assert not copied_file.exists() - + except ValueError: - pytest.skip("process_kokkos_trace_output doesn't handle missing output directory gracefully") + pytest.skip( + "process_kokkos_trace_output doesn't handle missing output directory gracefully" + ) def test_process_kokkos_trace_output_csv_with_only_headers(tmp_path, monkeypatch): """ Test process_kokkos_trace_output with CSV files that contain only headers but no data. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that header-only files result in empty DataFrame. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "headers_only_marker_api_trace.csv" csv1.write_text("timestamp,marker_name,duration,thread_id\n") - + fbase = "headers_only" - + import utils.utils as utils_mod + utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 0 assert list(df.columns) == ["timestamp", "marker_name", "duration", "thread_id"] @@ -3245,41 +3863,48 @@ def test_process_kokkos_trace_output_csv_with_only_headers(tmp_path, monkeypatch def test_process_kokkos_trace_output_large_files(tmp_path, monkeypatch): """ Test process_kokkos_trace_output with larger CSV files to ensure memory handling. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that large files are processed correctly. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "large_marker_api_trace.csv" - + content = "timestamp,marker_name,duration,thread_id\n" - kokkos_markers = ["kokkos_malloc", "kokkos_free", "kokkos_parallel_for", "kokkos_parallel_reduce", "kokkos_fence"] + kokkos_markers = [ + "kokkos_malloc", + "kokkos_free", + "kokkos_parallel_for", + "kokkos_parallel_reduce", + "kokkos_fence", + ] for i in range(1000): marker_name = kokkos_markers[i % len(kokkos_markers)] content += f"{i},{marker_name},{i%100},{i%10}\n" - + csv1.write_text(content) - + fbase = "large_test" - + import utils.utils as utils_mod + utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 1000 assert "kokkos_malloc" in df["marker_name"].values @@ -3289,34 +3914,38 @@ def test_process_kokkos_trace_output_large_files(tmp_path, monkeypatch): def test_process_kokkos_trace_output_unicode_content(tmp_path, monkeypatch): """ Test process_kokkos_trace_output with CSV files containing unicode characters. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that unicode content is handled properly. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "unicode_marker_api_trace.csv" - csv1.write_text("timestamp,marker_name,duration\n1000,kokkos_α_kernel,500\n2000,kokkos_β_operation,300\n", encoding='utf-8') - + csv1.write_text( + "timestamp,marker_name,duration\n1000,kokkos_α_kernel,500\n2000,kokkos_β_operation,300\n", + encoding="utf-8", + ) + fbase = "unicode_test" - + import utils.utils as utils_mod + utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 2 assert "kokkos_α_kernel" in df["marker_name"].values @@ -3326,40 +3955,45 @@ def test_process_kokkos_trace_output_unicode_content(tmp_path, monkeypatch): def test_process_kokkos_trace_output_different_schemas(tmp_path, monkeypatch): """ Test process_kokkos_trace_output with CSV files having different column schemas. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that files with different schemas are concatenated properly. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub2 = out_dir / "process2" sub1.mkdir() sub2.mkdir() - + csv1 = sub1 / "schema1_marker_api_trace.csv" csv2 = sub2 / "schema2_marker_api_trace.csv" - + # Different column order and types - csv1.write_text("marker_id,marker_name,start_time\n1,kokkos_begin,1000\n2,kokkos_end,2000\n") - csv2.write_text("marker_name,duration,thread_id\nkokkos_malloc,500,0\nkokkos_free,200,1\n") - + csv1.write_text( + "marker_id,marker_name,start_time\n1,kokkos_begin,1000\n2,kokkos_end,2000\n" + ) + csv2.write_text( + "marker_name,duration,thread_id\nkokkos_malloc,500,0\nkokkos_free,200,1\n" + ) + fbase = "schema_test" - + import utils.utils as utils_mod + utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_marker_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 4 # Should have union of all columns with NaN for missing values @@ -3370,38 +4004,39 @@ def test_process_kokkos_trace_output_different_schemas(tmp_path, monkeypatch): def test_process_kokkos_trace_output_permission_error(tmp_path, monkeypatch): """ Test process_kokkos_trace_output when there are permission errors during file operations. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that permission errors are handled gracefully. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "test_marker_api_trace.csv" csv1.write_text("timestamp,marker_name\n1000,kokkos_malloc\n") - + fbase = "permission_test" - + def mock_to_csv_permission_error(self, path, **kwargs): raise PermissionError("Permission denied") - + monkeypatch.setattr("pandas.DataFrame.to_csv", mock_to_csv_permission_error) - + import utils.utils as utils_mod - + with pytest.raises(PermissionError): utils_mod.process_kokkos_trace_output(workload_dir, fbase) - + + # ============================================================================= # HIP TRACE PROCESSING TESTS # ============================================================================= @@ -3435,6 +4070,7 @@ Missing output directory for copy operation File I/O errors """ + def test_process_hip_trace_output_multiple_files(tmp_path, monkeypatch): """ Test process_hip_trace_output with multiple valid CSV files. @@ -3443,36 +4079,41 @@ def test_process_hip_trace_output_multiple_files(tmp_path, monkeypatch): monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_warning", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub2 = out_dir / "process2" sub1.mkdir() sub2.mkdir() - + csv1 = sub1 / "test_hip_api_trace.csv" csv2 = sub2 / "test_hip_api_trace.csv" - - csv1.write_text("timestamp,api_name,duration\n1000,hipMalloc,500\n2000,hipMemcpy,300\n") - csv2.write_text("timestamp,api_name,duration\n3000,hipFree,200\n4000,hipLaunchKernel,800\n") - + + csv1.write_text( + "timestamp,api_name,duration\n1000,hipMalloc,500\n2000,hipMemcpy,300\n" + ) + csv2.write_text( + "timestamp,api_name,duration\n3000,hipFree,200\n4000,hipLaunchKernel,800\n" + ) + fbase = "test_workload" - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) - assert len(df) == 4 + assert len(df) == 4 assert df["timestamp"].tolist() == [1000, 2000, 3000, 4000] assert "hipMalloc" in df["api_name"].values assert "hipLaunchKernel" in df["api_name"].values - + copied_file = tmp_path / f"{fbase}_hip_api_trace.csv" assert copied_file.exists() df_copy = pd.read_csv(copied_file) @@ -3485,25 +4126,28 @@ def test_process_hip_trace_output_single_file(tmp_path, monkeypatch): """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "single_hip_api_trace.csv" - csv1.write_text("api_id,function_name,start_time,end_time\n1,hipDeviceSynchronize,1000,1050\n2,hipStreamCreate,2000,2010\n") - + csv1.write_text( + "api_id,function_name,start_time,end_time\n1,hipDeviceSynchronize,1000,1050\n2,hipStreamCreate,2000,2010\n" + ) + fbase = "single_test" - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 2 assert df["function_name"].tolist() == ["hipDeviceSynchronize", "hipStreamCreate"] @@ -3516,34 +4160,34 @@ def test_process_hip_trace_output_no_files_found(tmp_path, monkeypatch): """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + fbase = "no_files" - + def mock_concat(dataframes, **kwargs): if not dataframes: return pd.DataFrame() return pd.concat(dataframes, **kwargs) - + monkeypatch.setattr("pandas.concat", mock_concat) - + def mock_to_csv(self, path, **kwargs): - with open(path, 'w') as f: - f.write('') - + with open(path, "w") as f: + f.write("") + monkeypatch.setattr("pandas.DataFrame.to_csv", mock_to_csv) - + import utils.utils as utils_mod - + try: utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + except (ValueError, pd.errors.EmptyDataError): pytest.skip("process_hip_trace_output doesn't handle empty file list gracefully") @@ -3553,48 +4197,50 @@ def test_process_hip_trace_output_files_not_exist(tmp_path, monkeypatch): Test process_hip_trace_output when glob finds files but they don't actually exist. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + fake_files = [ str(out_dir / "fake1" / "test_hip_api_trace.csv"), - str(out_dir / "fake2" / "test_hip_api_trace.csv") + str(out_dir / "fake2" / "test_hip_api_trace.csv"), ] - + monkeypatch.setattr("glob.glob", lambda pattern: fake_files) - + fbase = "nonexistent" - + def mock_is_file(self): return False - + monkeypatch.setattr("pathlib.Path.is_file", mock_is_file) - + def mock_concat(dataframes, **kwargs): if not dataframes: return pd.DataFrame() return pd.concat(dataframes, **kwargs) - + monkeypatch.setattr("pandas.concat", mock_concat) - + def mock_to_csv(self, path, **kwargs): - with open(path, 'w') as f: - f.write('') - + with open(path, "w") as f: + f.write("") + monkeypatch.setattr("pandas.DataFrame.to_csv", mock_to_csv) - + import utils.utils as utils_mod - + try: utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + except ValueError: - pytest.skip("process_hip_trace_output doesn't handle empty file filtering gracefully") + pytest.skip( + "process_hip_trace_output doesn't handle empty file filtering gracefully" + ) def test_process_hip_trace_output_empty_csv_files(tmp_path, monkeypatch): @@ -3602,31 +4248,33 @@ def test_process_hip_trace_output_empty_csv_files(tmp_path, monkeypatch): Test process_hip_trace_output with empty CSV files. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "empty_hip_api_trace.csv" csv1.write_text("") - + fbase = "empty_test" - + original_read_csv = pd.read_csv + def mock_read_csv(filepath, **kwargs): try: return original_read_csv(filepath, **kwargs) except pd.errors.EmptyDataError: return pd.DataFrame() - + monkeypatch.setattr("pandas.read_csv", mock_read_csv) - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() @@ -3636,30 +4284,31 @@ def test_process_hip_trace_output_different_schemas(tmp_path, monkeypatch): Test process_hip_trace_output with CSV files having different column schemas. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub2 = out_dir / "process2" sub1.mkdir() sub2.mkdir() - + csv1 = sub1 / "schema1_hip_api_trace.csv" csv2 = sub2 / "schema2_hip_api_trace.csv" - + csv1.write_text("timestamp,api_name\n1000,hipMalloc\n") csv2.write_text("time,function,thread_id\n2000,hipFree,123\n") - + fbase = "mixed_schema" - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 2 @@ -3670,28 +4319,29 @@ def test_process_hip_trace_output_no_out_directory(tmp_path, monkeypatch): Should not copy file to workload directory. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) - + fbase = "no_out_dir" - + monkeypatch.setattr("glob.glob", lambda pattern: []) - + def mock_concat(dataframes, **kwargs): if not dataframes: return pd.DataFrame() return pd.concat(dataframes, **kwargs) - + monkeypatch.setattr("pandas.concat", mock_concat) - + def mock_to_csv(self, path, **kwargs): os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: - f.write('') - + with open(path, "w") as f: + f.write("") + monkeypatch.setattr("pandas.DataFrame.to_csv", mock_to_csv) - + original_path = utils.path + def mock_path_exists(path_str): if path_str == workload_dir + "/out": mock_path_obj = mock.MagicMock() @@ -3699,19 +4349,21 @@ def test_process_hip_trace_output_no_out_directory(tmp_path, monkeypatch): return mock_path_obj else: return original_path(path_str) - + monkeypatch.setattr("utils.utils.path", mock_path_exists) - + import utils.utils as utils_mod - + try: utils_mod.process_hip_trace_output(workload_dir, fbase) - + copied_file = tmp_path / f"{fbase}_hip_api_trace.csv" assert not copied_file.exists() - + except ValueError: - pytest.skip("process_hip_trace_output doesn't handle missing output directory gracefully") + pytest.skip( + "process_hip_trace_output doesn't handle missing output directory gracefully" + ) def test_process_hip_trace_output_file_permission_error(tmp_path, monkeypatch): @@ -3719,26 +4371,26 @@ def test_process_hip_trace_output_file_permission_error(tmp_path, monkeypatch): Test process_hip_trace_output when file operations fail due to permissions. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "perm_test_hip_api_trace.csv" csv1.write_text("api_name,duration\nhipMalloc,100\n") - + fbase = "permission_test" - + def mock_copyfile(src, dst): raise PermissionError("Permission denied") - + monkeypatch.setattr("shutil.copyfile", mock_copyfile) - + import utils.utils as utils_mod - + with pytest.raises(PermissionError): utils_mod.process_hip_trace_output(workload_dir, fbase) @@ -3748,27 +4400,29 @@ def test_process_hip_trace_output_corrupted_csv_files(tmp_path, monkeypatch): Test process_hip_trace_output with corrupted CSV files. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "corrupted_hip_api_trace.csv" - csv1.write_text("timestamp,api_name,duration\n1000,hipMalloc\n2000,hipFree,invalid_number,extra_column\n") - + csv1.write_text( + "timestamp,api_name,duration\n1000,hipMalloc\n2000,hipFree,invalid_number,extra_column\n" + ) + fbase = "corrupted_test" - + import utils.utils as utils_mod - + try: utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + except (pd.errors.ParserError, ValueError): pytest.skip("process_hip_trace_output doesn't handle corrupted CSV gracefully") @@ -3778,32 +4432,39 @@ def test_process_hip_trace_output_large_files(tmp_path, monkeypatch): Test process_hip_trace_output with larger CSV files to ensure memory handling. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "large_hip_api_trace.csv" - + content = "timestamp,api_name,duration,thread_id\n" - hip_apis = ["hipMalloc", "hipFree", "hipMemcpy", "hipLaunchKernel", "hipDeviceSynchronize"] + hip_apis = [ + "hipMalloc", + "hipFree", + "hipMemcpy", + "hipLaunchKernel", + "hipDeviceSynchronize", + ] for i in range(1000): api_name = hip_apis[i % len(hip_apis)] content += f"{i},{api_name},{i%100},{i%10}\n" - + csv1.write_text(content) - + fbase = "large_test" - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 1000 assert "hipMalloc" in df["api_name"].values @@ -3815,25 +4476,29 @@ def test_process_hip_trace_output_unicode_content(tmp_path, monkeypatch): Test process_hip_trace_output with CSV files containing unicode characters. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "unicode_hip_api_trace.csv" - csv1.write_text("api_name,description\nhipMalloc,内存分配\nhipKernel,核函数执行\n", encoding='utf-8') - + csv1.write_text( + "api_name,description\nhipMalloc,内存分配\nhipKernel,核函数执行\n", + encoding="utf-8", + ) + fbase = "unicode_test" - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 2 assert "内存分配" in df["description"].values @@ -3845,25 +4510,26 @@ def test_process_hip_trace_output_csv_with_only_headers(tmp_path, monkeypatch): Test process_hip_trace_output with CSV files that contain only headers but no data. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "headers_only_hip_api_trace.csv" csv1.write_text("timestamp,api_name,duration,thread_id\n") - + fbase = "headers_only" - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) == 0 assert list(df.columns) == ["timestamp", "api_name", "duration", "thread_id"] @@ -3874,44 +4540,46 @@ def test_process_hip_trace_output_mixed_file_states(tmp_path, monkeypatch): Test process_hip_trace_output with a mix of valid, empty, and corrupted files. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub2 = out_dir / "process2" sub3 = out_dir / "process3" sub1.mkdir() sub2.mkdir() sub3.mkdir() - + csv1 = sub1 / "valid_hip_api_trace.csv" csv1.write_text("timestamp,api_name\n1000,hipMalloc\n2000,hipFree\n") - + csv2 = sub2 / "empty_hip_api_trace.csv" csv2.write_text("") - + csv3 = sub3 / "headers_hip_api_trace.csv" csv3.write_text("timestamp,api_name\n") - + fbase = "mixed_test" - + original_read_csv = pd.read_csv + def mock_read_csv(filepath, **kwargs): try: return original_read_csv(filepath, **kwargs) except pd.errors.EmptyDataError: return pd.DataFrame() - + monkeypatch.setattr("pandas.read_csv", mock_read_csv) - + import utils.utils as utils_mod + utils_mod.process_hip_trace_output(workload_dir, fbase) - + output_file = out_dir / f"results_{fbase}_hip_api_trace.csv" assert output_file.exists() - + df = pd.read_csv(output_file) assert len(df) >= 0 @@ -3921,481 +4589,501 @@ def test_process_hip_trace_output_invalid_fbase_characters(tmp_path, monkeypatch Test process_hip_trace_output with invalid fbase containing special characters. """ monkeypatch.setattr("utils.utils.console_debug", lambda *a, **k: None) - + workload_dir = str(tmp_path) out_dir = tmp_path / "out" / "pmc_1" out_dir.mkdir(parents=True) - + sub1 = out_dir / "process1" sub1.mkdir() - + csv1 = sub1 / "special_hip_api_trace.csv" csv1.write_text("api_name\nhipMalloc\n") - + fbase = "test\x00invalid" - + import utils.utils as utils_mod - + with pytest.raises((OSError, ValueError)): utils_mod.process_hip_trace_output(workload_dir, fbase) - + + # ============================================================================== # ROOFLINE DETECTION TESTS # ============================================================================== + def test_ubuntu_22_04_detection(monkeypatch): """ Test Ubuntu 22.04 detection. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching - + Returns: Verifies that the function correctly identifies Ubuntu 22.04 and returns the appropriate distro """ mock_os_release = 'VERSION_ID="22.04"\nNAME="Ubuntu"' - + def mock_path_read_text(self): return mock_os_release - + monkeypatch.setattr("os.environ", {"keys": lambda: []}) - + monkeypatch.setattr("pathlib.Path.read_text", mock_path_read_text) - + def mock_search(pattern, text): - if 'VERSION_ID' in pattern: + if "VERSION_ID" in pattern: return "22.04" return None - + monkeypatch.setattr("utils.specs.search", mock_search) - + import utils.utils as utils_mod + result = utils_mod.detect_roofline({}) - + assert result == {"distro": "22.04"} def test_ubuntu_24_04_detection(monkeypatch): """ Test Ubuntu 24.04 detection. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching - + Returns: Verifies that the function correctly identifies Ubuntu 24.04 and returns the appropriate distro """ mock_os_release = 'VERSION_ID="24.04"\nNAME="Ubuntu"' - + def mock_path_read_text(self): return mock_os_release - + monkeypatch.setattr("os.environ", {"keys": lambda: []}) - + monkeypatch.setattr("pathlib.Path.read_text", mock_path_read_text) - + def mock_search(pattern, text): - if 'VERSION_ID' in pattern: + if "VERSION_ID" in pattern: return "24.04" return None - + monkeypatch.setattr("utils.specs.search", mock_search) - + import utils.utils as utils_mod + result = utils_mod.detect_roofline({}) - + assert result == {"distro": "22.04"} def test_rhel_detection(monkeypatch): """ Test RHEL distro detection. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching - + Returns: Verifies that the function correctly identifies RHEL and returns the appropriate distro """ mock_os_release = 'PLATFORM_ID="platform:el9"\nNAME="Red Hat Enterprise Linux"' - + def mock_path_read_text(self): return mock_os_release - + monkeypatch.setattr("os.environ", {"keys": lambda: []}) - + monkeypatch.setattr("pathlib.Path.read_text", mock_path_read_text) - + def mock_search(pattern, text): - if 'PLATFORM_ID' in pattern: + if "PLATFORM_ID" in pattern: return "platform:el9" return None - + monkeypatch.setattr("utils.specs.search", mock_search) - + import utils.utils as utils_mod + result = utils_mod.detect_roofline({}) - + assert result == {"distro": "platform:el8"} def test_sles_15_6_detection(monkeypatch): """ Test SLES 15.6 detection. - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching - + Returns: Verifies that the function correctly identifies SLES 15.6 and returns the appropriate distro """ mock_os_release = 'VERSION_ID="15.6"\nNAME="SLES"' - + def mock_path_read_text(self): return mock_os_release - + monkeypatch.setattr("os.environ", {"keys": lambda: []}) - + monkeypatch.setattr("pathlib.Path.read_text", mock_path_read_text) - + def mock_search(pattern, text): - if 'VERSION_ID' in pattern: + if "VERSION_ID" in pattern: return "15.6" return None - + monkeypatch.setattr("utils.specs.search", mock_search) - + import utils.utils as utils_mod + result = utils_mod.detect_roofline({}) - + assert result == {"distro": "15.6"} def test_sles_15_7_detection(monkeypatch): """ Test SLES 15.7 detection (edge case with higher service pack). - + Args: monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching - + Returns: Verifies that the function correctly handles newer SLES service packs """ mock_os_release = 'VERSION_ID="15.7"\nNAME="SLES"' - + def mock_path_read_text(self): return mock_os_release - + monkeypatch.setattr("os.environ", {"keys": lambda: []}) - + monkeypatch.setattr("pathlib.Path.read_text", mock_path_read_text) - + def mock_search(pattern, text): - if 'VERSION_ID' in pattern: + if "VERSION_ID" in pattern: return "15.7" return None - + monkeypatch.setattr("utils.specs.search", mock_search) - + import utils.utils as utils_mod + result = utils_mod.detect_roofline({}) - + assert result == {"distro": "15.6"} - + + # ============================================================================= # TESTS FOR MIBENCH OUTPUT # ============================================================================= + def test_mibench_override_distro_success(tmp_path, monkeypatch): """ Test mibench with override distro that successfully finds and executes binary. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that override path is used and subprocess is called correctly. """ + class MockArgs: path = str(tmp_path) device = 0 quiet = False - + class MockMspec: pass - + override_binary_path = tmp_path / "custom_roofline" override_binary_path.write_text("#!/bin/bash\necho 'roofline executed'") override_binary_path.chmod(0o755) - + def mock_detect_roofline(mspec): return {"distro": "override", "path": str(override_binary_path)} - + subprocess_calls = [] + def mock_subprocess_run(args, check=True): subprocess_calls.append((args, check)) - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod + utils_mod.mibench(MockArgs(), MockMspec()) - + assert len(subprocess_calls) == 1 expected_args = [ str(override_binary_path), "-o", str(tmp_path) + "/roofline.csv", "-d", - "0" + "0", ] assert subprocess_calls[0][0] == expected_args assert subprocess_calls[0][1] is True + def test_mibench_standard_distro_first_path_exists(tmp_path, monkeypatch): """ Test mibench with standard distro where first potential path exists. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that first path is used when it exists. """ + class MockArgs: path = str(tmp_path) device = 1 quiet = True - + class MockMspec: pass - + rocprof_home = tmp_path / "rocprof_home" install_root = tmp_path / "install_root" rocprof_home.mkdir(parents=True) install_root.mkdir(parents=True) - + first_path = rocprof_home / "utils" / "rooflines" first_path.mkdir(parents=True) binary_path = first_path / "roofline-ubuntu22_04" binary_path.write_text("#!/bin/bash\necho 'roofline executed'") binary_path.chmod(0o755) - + class MockConfig: def __init__(self): self.rocprof_compute_home = self.MockPath(rocprof_home, install_root) - + class MockPath: def __init__(self, home_path, install_path): self._home_path = home_path self._install_path = install_path self.parent = self.MockParent(install_path) - + def __str__(self): return str(self._home_path) - + def __truediv__(self, other): return self._home_path / other - + class MockParent: def __init__(self, install_path): self.parent = install_path - + def __truediv__(self, other): return self.parent / other - + mock_config = MockConfig() - + def mock_detect_roofline(mspec): return {"distro": "22.04"} - + subprocess_calls = [] + def mock_subprocess_run(args, check=True): subprocess_calls.append((args, check)) - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("utils.utils.config", mock_config) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod + utils_mod.mibench(MockArgs(), MockMspec()) - + assert len(subprocess_calls) == 1 - + actual_cmd = subprocess_calls[0][0] - + assert actual_cmd[0] == str(binary_path) assert "-o" in actual_cmd assert str(tmp_path) + "/roofline.csv" in actual_cmd assert "-d" in actual_cmd assert "1" in actual_cmd - + cmd_str = "".join(str(arg) for arg in actual_cmd) - assert "--quiet" in cmd_str or all(c in cmd_str for c in ['-', 'q', 'u', 'i', 'e', 't']) + assert "--quiet" in cmd_str or all( + c in cmd_str for c in ["-", "q", "u", "i", "e", "t"] + ) + def test_mibench_standard_distro_second_path_exists(tmp_path, monkeypatch): """ Test mibench with standard distro where second potential path exists. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that second path is used when first doesn't exist. """ + class MockArgs: path = str(tmp_path) device = 2 quiet = False - + class MockMspec: pass - + rocprof_home = tmp_path / "rocprof_home" install_root = tmp_path / "install_root" rocprof_home.mkdir(parents=True) install_root.mkdir(parents=True) - + second_path = install_root / "bin" second_path.mkdir(parents=True) binary_path = second_path / "roofline-rhel8" binary_path.write_text("#!/bin/bash\necho 'roofline executed'") binary_path.chmod(0o755) - + class MockConfig: def __init__(self): self.rocprof_compute_home = self.MockPath(rocprof_home, install_root) - + class MockPath: def __init__(self, home_path, install_path): self._home_path = home_path self._install_path = install_path self.parent = self.MockParent(install_path) - + def __str__(self): return str(self._home_path) - + def __truediv__(self, other): return self._home_path / other - + class MockParent: def __init__(self, install_path): self.parent = install_path - + def __truediv__(self, other): return self.parent / other - + mock_config = MockConfig() - + def mock_detect_roofline(mspec): return {"distro": "platform:el8"} - + subprocess_calls = [] + def mock_subprocess_run(args, check=True): subprocess_calls.append((args, check)) - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("utils.utils.config", mock_config) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod + utils_mod.mibench(MockArgs(), MockMspec()) - + assert len(subprocess_calls) == 1 - expected_args = [ - str(binary_path), - "-o", - str(tmp_path) + "/roofline.csv", - "-d", - "2" - ] + expected_args = [str(binary_path), "-o", str(tmp_path) + "/roofline.csv", "-d", "2"] assert subprocess_calls[0][0] == expected_args + def test_mibench_no_binary_found_error(tmp_path, monkeypatch): """ Test mibench when no binary paths exist, should call console_error. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that console_error is called when no binaries are found. """ + class MockArgs: path = str(tmp_path) device = 0 quiet = False - + class MockMspec: pass - + rocprof_home = tmp_path / "rocprof_home" install_root = tmp_path / "install_root" rocprof_home.mkdir(parents=True) install_root.mkdir(parents=True) - + class MockConfig: def __init__(self): self.rocprof_compute_home = self.MockPath(rocprof_home, install_root) - + class MockPath: def __init__(self, home_path, install_path): self._home_path = home_path self._install_path = install_path self.parent = self.MockParent(install_path) - + def __str__(self): return str(self._home_path) - + def __truediv__(self, other): return self._home_path / other - + class MockParent: def __init__(self, install_path): self.parent = install_path - + def __truediv__(self, other): return self.parent / other - + mock_config = MockConfig() - + def mock_detect_roofline(mspec): return {"distro": "15.6"} - + console_error_calls = [] + def mock_console_error(category, msg): console_error_calls.append((category, msg)) raise RuntimeError("console_error called") - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("utils.utils.config", mock_config) monkeypatch.setattr("utils.utils.console_error", mock_console_error) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod - + with pytest.raises(RuntimeError, match="console_error called"): utils_mod.mibench(MockArgs(), MockMspec()) - + assert len(console_error_calls) == 1 assert console_error_calls[0][0] == "roofline" assert "Unable to locate expected binary" in console_error_calls[0][1] + def test_mibench_quiet_flag_handling_bug(tmp_path, monkeypatch): """ Test mibench quiet flag handling demonstrates the bug where += splits the string. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that the bug exists and characters are split. """ @@ -4403,352 +5091,365 @@ def test_mibench_quiet_flag_handling_bug(tmp_path, monkeypatch): install_root = tmp_path / "install_root" rocprof_home.mkdir(parents=True) install_root.mkdir(parents=True) - + first_path = rocprof_home / "utils" / "rooflines" first_path.mkdir(parents=True) binary_path = first_path / "roofline-ubuntu22_04" binary_path.write_text("#!/bin/bash\necho 'roofline executed'") binary_path.chmod(0o755) - + class MockConfig: def __init__(self): self.rocprof_compute_home = self.MockPath(rocprof_home, install_root) - + class MockPath: def __init__(self, home_path, install_path): self._home_path = home_path self._install_path = install_path self.parent = self.MockParent(install_path) - + def __str__(self): return str(self._home_path) - + def __truediv__(self, other): return self._home_path / other - + class MockParent: def __init__(self, install_path): self.parent = install_path - + def __truediv__(self, other): return self.parent / other - + mock_config = MockConfig() - + def mock_detect_roofline(mspec): return {"distro": "22.04"} - + subprocess_calls = [] + def mock_subprocess_run(args, check=True): subprocess_calls.append((args, check)) - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("utils.utils.config", mock_config) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod - + class MockArgsQuiet: path = str(tmp_path) device = 0 quiet = True - + class MockMspecQuiet: pass - + utils_mod.mibench(MockArgsQuiet(), MockMspecQuiet()) - + actual_cmd = subprocess_calls[0][0] expected_base_args = [ str(binary_path), "-o", str(tmp_path) + "/roofline.csv", "-d", - "0" + "0", ] - expected_full_args = expected_base_args + ['-', '-', 'q', 'u', 'i', 'e', 't'] + expected_full_args = expected_base_args + ["-", "-", "q", "u", "i", "e", "t"] assert actual_cmd == expected_full_args - + subprocess_calls.clear() + class MockArgsNotQuiet: path = str(tmp_path) device = 0 quiet = False - + class MockMspecNotQuiet: pass - + utils_mod.mibench(MockArgsNotQuiet(), MockMspecNotQuiet()) - + actual_cmd = subprocess_calls[0][0] - expected_args = [ - str(binary_path), - "-o", - str(tmp_path) + "/roofline.csv", - "-d", - "0" - ] + expected_args = [str(binary_path), "-o", str(tmp_path) + "/roofline.csv", "-d", "0"] assert actual_cmd == expected_args + def test_mibench_sles_distro_mapping(tmp_path, monkeypatch): """ Test mibench with SLES distro mapping. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that SLES distro is correctly mapped. """ + class MockArgs: path = str(tmp_path) device = 3 quiet = False - + class MockMspec: pass - + rocprof_home = tmp_path / "rocprof_home" install_root = tmp_path / "install_root" rocprof_home.mkdir(parents=True) install_root.mkdir(parents=True) - + first_path = rocprof_home / "utils" / "rooflines" first_path.mkdir(parents=True) binary_path = first_path / "roofline-sles15sp6" binary_path.write_text("#!/bin/bash\necho 'roofline executed'") binary_path.chmod(0o755) - + class MockConfig: def __init__(self): self.rocprof_compute_home = self.MockPath(rocprof_home, install_root) - + class MockPath: def __init__(self, home_path, install_path): self._home_path = home_path self._install_path = install_path self.parent = self.MockParent(install_path) - + def __str__(self): return str(self._home_path) - + def __truediv__(self, other): return self._home_path / other - + class MockParent: def __init__(self, install_path): self.parent = install_path - + def __truediv__(self, other): return self.parent / other - + mock_config = MockConfig() - + def mock_detect_roofline(mspec): return {"distro": "15.6"} - + subprocess_calls = [] + def mock_subprocess_run(args, check=True): subprocess_calls.append((args, check)) - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("utils.utils.config", mock_config) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod + utils_mod.mibench(MockArgs(), MockMspec()) - + assert len(subprocess_calls) == 1 assert str(binary_path) in subprocess_calls[0][0] + def test_mibench_subprocess_run_failure(tmp_path, monkeypatch): """ Test mibench when subprocess.run raises an exception. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that subprocess exceptions are properly propagated. """ + class MockArgs: path = str(tmp_path) device = 0 quiet = False - + class MockMspec: pass - + override_binary_path = tmp_path / "failing_roofline" override_binary_path.write_text("#!/bin/bash\nexit 1") override_binary_path.chmod(0o755) - + def mock_detect_roofline(mspec): return {"distro": "override", "path": str(override_binary_path)} - + def mock_subprocess_run(args, check=True): raise subprocess.CalledProcessError(1, args) - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod - + with pytest.raises(subprocess.CalledProcessError): utils_mod.mibench(MockArgs(), MockMspec()) + def test_mibench_device_string_conversion(tmp_path, monkeypatch): """ Test mibench correctly converts device ID to string. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that device ID is converted to string in subprocess args. """ + class MockArgs: path = str(tmp_path) device = 42 quiet = False - + class MockMspec: pass - + override_binary_path = tmp_path / "test_roofline" override_binary_path.write_text("#!/bin/bash\necho 'success'") override_binary_path.chmod(0o755) - + def mock_detect_roofline(mspec): return {"distro": "override", "path": str(override_binary_path)} - + subprocess_calls = [] + def mock_subprocess_run(args, check=True): subprocess_calls.append(args) - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod + utils_mod.mibench(MockArgs(), MockMspec()) - + assert len(subprocess_calls) == 1 device_arg_index = subprocess_calls[0].index("-d") + 1 assert subprocess_calls[0][device_arg_index] == "42" assert isinstance(subprocess_calls[0][device_arg_index], str) + def test_mibench_unknown_distro_mapping(tmp_path, monkeypatch): """ Test mibench behavior with unknown distro (should cause KeyError). - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that KeyError is raised for unknown distro. """ + class MockArgs: path = str(tmp_path) device = 0 quiet = False - + class MockMspec: pass - + rocprof_home = tmp_path / "rocprof_home" install_root = tmp_path / "install_root" rocprof_home.mkdir(parents=True) install_root.mkdir(parents=True) - + class MockConfig: def __init__(self): self.rocprof_compute_home = self.MockPath(rocprof_home, install_root) - + class MockPath: def __init__(self, home_path, install_path): self._home_path = home_path self._install_path = install_path self.parent = self.MockParent(install_path) - + def __str__(self): return str(self._home_path) - + def __truediv__(self, other): return self._home_path / other - + class MockParent: def __init__(self, install_path): self.parent = install_path - + def __truediv__(self, other): return self.parent / other - + mock_config = MockConfig() - + def mock_detect_roofline(mspec): return {"distro": "unknown_distro"} # Not in distro_map - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("utils.utils.config", mock_config) monkeypatch.setattr("utils.utils.console_log", lambda *a, **k: None) - + import utils.utils as utils_mod - + with pytest.raises(KeyError): utils_mod.mibench(MockArgs(), MockMspec()) + def test_mibench_console_log_called(tmp_path, monkeypatch): """ Test mibench calls console_log with correct message. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. monkeypatch (pytest.MonkeyPatch): Pytest fixture for patching. - + Returns: None: Asserts that console_log is called with expected message. """ + class MockArgs: path = str(tmp_path) device = 0 quiet = False - + class MockMspec: pass - + override_binary_path = tmp_path / "test_roofline" override_binary_path.write_text("#!/bin/bash\necho 'success'") override_binary_path.chmod(0o755) - + def mock_detect_roofline(mspec): return {"distro": "override", "path": str(override_binary_path)} - + console_log_calls = [] + def mock_console_log(category, message): console_log_calls.append((category, message)) - + def mock_subprocess_run(args, check=True): pass - + monkeypatch.setattr("utils.utils.detect_roofline", mock_detect_roofline) monkeypatch.setattr("subprocess.run", mock_subprocess_run) monkeypatch.setattr("utils.utils.console_log", mock_console_log) - + import utils.utils as utils_mod + utils_mod.mibench(MockArgs(), MockMspec()) - + assert len(console_log_calls) == 1 assert console_log_calls[0][0] == "roofline" assert console_log_calls[0][1] == "No roofline data found. Generating..." - + + # ============================================================================= # TESTS FOR flatten_tcc_info_across_xcds # ============================================================================= @@ -4782,87 +5483,97 @@ Large dataset handling Data preservation validation Regex pattern validation """ + + def test_flatten_tcc_info_across_xcds_zero_xcds(tmp_path): """ Test edge case with zero XCDs. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles zero XCDs edge case by raising ValueError. """ - columns = ['Kernel_Name', 'TCC_HIT[0]'] - data = [['kernel1', 100]] - + columns = ["Kernel_Name", "TCC_HIT[0]"] + data = [["kernel1", 100]] + df = pd.DataFrame(data, columns=columns) csv_file = tmp_path / "test_zero_xcds.csv" df.to_csv(csv_file, index=False) - + import utils.utils as utils_mod - + with pytest.raises(ValueError, match="range\\(\\) arg 3 must not be zero"): - utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=0, tcc_channel_per_xcd=4) + utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=0, tcc_channel_per_xcd=4 + ) def test_flatten_tcc_info_across_xcds_insufficient_data(tmp_path): """ Test when there's insufficient data for the specified XCDs. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function raises ValueError when trying to process insufficient data. """ - columns = ['Kernel_Name', 'TCC_HIT[0]'] - data = [['kernel1', 100]] - + columns = ["Kernel_Name", "TCC_HIT[0]"] + data = [["kernel1", 100]] + df = pd.DataFrame(data, columns=columns) csv_file = tmp_path / "test_insufficient.csv" df.to_csv(csv_file, index=False) - + import utils.utils as utils_mod - + with pytest.raises(ValueError, match="cannot set a row with mismatched columns"): - utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=3, tcc_channel_per_xcd=4) + utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=3, tcc_channel_per_xcd=4 + ) def test_flatten_tcc_info_across_xcds_irregular_tcc_column_names(tmp_path): """ Test with irregular TCC column naming patterns. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles various TCC column name patterns but may fail with pandas Series ambiguity. """ - columns = ['Kernel_Name', 'TCC_HIT_SPECIAL[0]', 'NOT_TCC_BUT_HAS_TCC', 'TCC_MISS[0]'] + columns = ["Kernel_Name", "TCC_HIT_SPECIAL[0]", "NOT_TCC_BUT_HAS_TCC", "TCC_MISS[0]"] data = [ - ['kernel1', 100, 50, 10], - ['kernel1', 200, 60, 20], + ["kernel1", 100, 50, 10], + ["kernel1", 200, 60, 20], ] - + df = pd.DataFrame(data, columns=columns) csv_file = tmp_path / "test_irregular.csv" df.to_csv(csv_file, index=False) - + import utils.utils as utils_mod - + try: - result = utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=2, tcc_channel_per_xcd=4) - + result = utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=2, tcc_channel_per_xcd=4 + ) + assert len(result) == 1 - assert 'TCC_HIT_SPECIAL[0]' in result.columns - assert 'TCC_HIT_SPECIAL[4]' in result.columns - assert 'TCC_MISS[0]' in result.columns - assert 'TCC_MISS[4]' in result.columns - assert result.iloc[0]['NOT_TCC_BUT_HAS_TCC'] == 50 - + assert "TCC_HIT_SPECIAL[0]" in result.columns + assert "TCC_HIT_SPECIAL[4]" in result.columns + assert "TCC_MISS[0]" in result.columns + assert "TCC_MISS[4]" in result.columns + assert result.iloc[0]["NOT_TCC_BUT_HAS_TCC"] == 50 + except ValueError as e: if "The truth value of a Series is ambiguous" in str(e): - pytest.skip("Function has pandas Series ambiguity issue in boolean evaluation") + pytest.skip( + "Function has pandas Series ambiguity issue in boolean evaluation" + ) else: raise @@ -4870,41 +5581,45 @@ def test_flatten_tcc_info_across_xcds_irregular_tcc_column_names(tmp_path): def test_flatten_tcc_info_across_xcds_regex_pattern_validation(tmp_path): """ Test that regex pattern correctly identifies channel indices. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts regex pattern works for various channel index formats but may fail with pandas Series ambiguity. """ - columns = ['TCC_HIT[0]', 'TCC_MISS[10]', 'TCC_REQ[255]', 'TCC_INVALID_NO_BRACKET'] + columns = ["TCC_HIT[0]", "TCC_MISS[10]", "TCC_REQ[255]", "TCC_INVALID_NO_BRACKET"] data = [ [100, 200, 300, 400], # XCD 0 [500, 600, 700, 800], # XCD 1 ] - + df = pd.DataFrame(data, columns=columns) csv_file = tmp_path / "test_regex.csv" df.to_csv(csv_file, index=False) - + import utils.utils as utils_mod - + try: - result = utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=2, tcc_channel_per_xcd=128) - + result = utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=2, tcc_channel_per_xcd=128 + ) + assert len(result) == 1 - assert 'TCC_HIT[0]' in result.columns - assert 'TCC_HIT[128]' in result.columns # 0 + 1*128 - assert 'TCC_MISS[10]' in result.columns - assert 'TCC_MISS[138]' in result.columns # 10 + 1*128 - assert 'TCC_REQ[255]' in result.columns - assert 'TCC_REQ[383]' in result.columns # 255 + 1*128 - - assert result.iloc[0]['TCC_INVALID_NO_BRACKET'] == 400 - + assert "TCC_HIT[0]" in result.columns + assert "TCC_HIT[128]" in result.columns # 0 + 1*128 + assert "TCC_MISS[10]" in result.columns + assert "TCC_MISS[138]" in result.columns # 10 + 1*128 + assert "TCC_REQ[255]" in result.columns + assert "TCC_REQ[383]" in result.columns # 255 + 1*128 + + assert result.iloc[0]["TCC_INVALID_NO_BRACKET"] == 400 + except ValueError as e: if "The truth value of a Series is ambiguous" in str(e): - pytest.skip("Function has pandas Series ambiguity issue in boolean evaluation") + pytest.skip( + "Function has pandas Series ambiguity issue in boolean evaluation" + ) else: raise @@ -4912,67 +5627,78 @@ def test_flatten_tcc_info_across_xcds_regex_pattern_validation(tmp_path): def test_flatten_tcc_info_across_xcds_edge_case_validation(tmp_path): """ Test edge cases and validation scenarios for flatten_tcc_info_across_xcds. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function behavior with various edge cases. """ import utils.utils as utils_mod - - columns = ['Kernel_Name', 'TCC_HIT[0]'] - data = [['kernel1', 100]] + + columns = ["Kernel_Name", "TCC_HIT[0]"] + data = [["kernel1", 100]] df = pd.DataFrame(data, columns=columns) csv_file = tmp_path / "test_zero_xcds.csv" df.to_csv(csv_file, index=False) - + with pytest.raises(ValueError): - utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=0, tcc_channel_per_xcd=4) - + utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=0, tcc_channel_per_xcd=4 + ) + try: - result = utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=-1, tcc_channel_per_xcd=4) + result = utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=-1, tcc_channel_per_xcd=4 + ) assert len(result) == 0 except ValueError: pass - + with pytest.raises(FileNotFoundError): - utils_mod.flatten_tcc_info_across_xcds('nonexistent.csv', xcds=2, tcc_channel_per_xcd=4) + utils_mod.flatten_tcc_info_across_xcds( + "nonexistent.csv", xcds=2, tcc_channel_per_xcd=4 + ) + def test_flatten_tcc_info_across_xcds_pandas_filter_issue(tmp_path): """ Test demonstrating the pandas filter regex issue that causes Series ambiguity error. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Documents the pandas boolean evaluation issue in the function. """ - columns = ['Kernel_Name', 'TCC_HIT[0]', 'SQ_WAVES'] + columns = ["Kernel_Name", "TCC_HIT[0]", "SQ_WAVES"] data = [ - ['kernel1', 100, 50], - ['kernel1', 200, 60], + ["kernel1", 100, 50], + ["kernel1", 200, 60], ] - + df = pd.DataFrame(data, columns=columns) csv_file = tmp_path / "test_pandas_issue.csv" df.to_csv(csv_file, index=False) - + import utils.utils as utils_mod - + try: - result = utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=2, tcc_channel_per_xcd=4) - + result = utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=2, tcc_channel_per_xcd=4 + ) + assert len(result) == 1 - assert 'Kernel_Name' in result.columns - assert 'TCC_HIT[0]' in result.columns - assert 'TCC_HIT[4]' in result.columns - assert 'SQ_WAVES' in result.columns - + assert "Kernel_Name" in result.columns + assert "TCC_HIT[0]" in result.columns + assert "TCC_HIT[4]" in result.columns + assert "SQ_WAVES" in result.columns + except ValueError as e: if "The truth value of a Series is ambiguous" in str(e): - pytest.skip("Known issue: pandas .filter() with regex causes Series boolean ambiguity") + pytest.skip( + "Known issue: pandas .filter() with regex causes Series boolean ambiguity" + ) else: raise @@ -4980,35 +5706,38 @@ def test_flatten_tcc_info_across_xcds_pandas_filter_issue(tmp_path): def test_flatten_tcc_info_across_xcds_successful_cases_only(tmp_path): """ Test only the cases that are expected to work successfully. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts successful operation for known working scenarios. """ import utils.utils as utils_mod - - columns = ['TCC_HIT[0]', 'TCC_MISS[0]'] + + columns = ["TCC_HIT[0]", "TCC_MISS[0]"] data = [ [100, 10], # XCD 0 [200, 20], # XCD 1 ] - + df = pd.DataFrame(data, columns=columns) csv_file = tmp_path / "test_simple_success.csv" df.to_csv(csv_file, index=False) - - result = utils_mod.flatten_tcc_info_across_xcds(str(csv_file), xcds=2, tcc_channel_per_xcd=4) - + + result = utils_mod.flatten_tcc_info_across_xcds( + str(csv_file), xcds=2, tcc_channel_per_xcd=4 + ) + assert len(result) == 1 - assert 'TCC_HIT[0]' in result.columns - assert 'TCC_HIT[4]' in result.columns - assert 'TCC_MISS[0]' in result.columns - assert 'TCC_MISS[4]' in result.columns - assert result.iloc[0]['TCC_HIT[0]'] == 100 - assert result.iloc[0]['TCC_HIT[4]'] == 200 - + assert "TCC_HIT[0]" in result.columns + assert "TCC_HIT[4]" in result.columns + assert "TCC_MISS[0]" in result.columns + assert "TCC_MISS[4]" in result.columns + assert result.iloc[0]["TCC_HIT[0]"] == 100 + assert result.iloc[0]["TCC_HIT[4]"] == 200 + + # ============================================================================= # TESTS FOR flatten_tcc_info_across_xcds # ============================================================================= @@ -5046,52 +5775,56 @@ Return type consistency Docstring verification Behavior validation """ + + def test_get_submodules_basic_functionality(): """ Test basic functionality with a real package that has submodules. - + Returns: None: Asserts function correctly lists submodules from a real package. """ - import utils.utils as utils_mod from unittest.mock import MagicMock, patch - + + import utils.utils as utils_mod + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_parse', False), - (None, 'module_request', False), - (None, 'module_error', False), + (None, "module_parse", False), + (None, "module_request", False), + (None, "module_error", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + assert isinstance(result, list) assert len(result) == 3 - expected = ['parse', 'request', 'error'] + expected = ["parse", "request", "error"] assert result == expected def test_get_submodules_empty_package(): """ Test with a package that has no submodules. - + Returns: None: Asserts function returns empty list for packages without submodules. """ - from unittest.mock import patch, MagicMock + from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=[]): - result = utils_mod.get_submodules('empty_package') - + mock_package.__path__ = ["/fake/path"] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=[]): + result = utils_mod.get_submodules("empty_package") + assert isinstance(result, list) assert len(result) == 0 @@ -5099,146 +5832,151 @@ def test_get_submodules_empty_package(): def test_get_submodules_package_not_found(): """ Test behavior when package doesn't exist. - + Returns: None: Asserts ModuleNotFoundError is raised for non-existent packages. """ import utils.utils as utils_mod - + with pytest.raises(ModuleNotFoundError): - utils_mod.get_submodules('nonexistent_package_12345') + utils_mod.get_submodules("nonexistent_package_12345") def test_get_submodules_name_processing_single_underscore(): """ Test name processing with single underscore pattern. - + Returns: None: Asserts correct name processing for submodules with single underscore. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_parser', False), - (None, 'module_request', False), - (None, 'module_error', False), + (None, "module_parser", False), + (None, "module_request", False), + (None, "module_error", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - - expected = ['parser', 'request', 'error'] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + + expected = ["parser", "request", "error"] assert result == expected def test_get_submodules_name_processing_multiple_underscores(): """ Test name processing with multiple underscores in submodule names. - + Returns: None: Asserts correct name processing for complex underscore patterns. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_some_complex_name', False), - (None, 'module_another_test_case', False), - (None, 'module_simple', False), + (None, "module_some_complex_name", False), + (None, "module_another_test_case", False), + (None, "module_simple", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - - expected = ['somecomplexname', 'anothertestcase', 'simple'] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + + expected = ["somecomplexname", "anothertestcase", "simple"] assert result == expected def test_get_submodules_base_module_filtered(): """ Test that 'base' submodule is properly filtered out. - + Returns: None: Asserts 'base' submodules are excluded from results. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_base', False), - (None, 'module_parser', False), - (None, 'module_handler', False), + (None, "module_base", False), + (None, "module_parser", False), + (None, "module_handler", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - - expected = ['parser', 'handler'] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + + expected = ["parser", "handler"] assert result == expected - assert 'base' not in result + assert "base" not in result def test_get_submodules_no_underscore_in_name(): """ Test behavior with submodule names that don't follow the expected pattern. - + Returns: None: Asserts function handles names without underscores by raising IndexError. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'simplemodule', False), - (None, 'anothermodule', False), + (None, "simplemodule", False), + (None, "anothermodule", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): with pytest.raises(IndexError): - utils_mod.get_submodules('test_package') + utils_mod.get_submodules("test_package") def test_get_submodules_empty_name_parts(): """ Test behavior with empty name parts after splitting. - + Returns: None: Asserts function handles edge cases in name processing. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_', False), # ends with underscore - (None, '_module', False), # starts with underscore - this will cause IndexError - (None, 'module__double', False), # double underscore + (None, "module_", False), # ends with underscore + (None, "_module", False), # starts with underscore - this will cause IndexError + (None, "module__double", False), # double underscore ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): try: - result = utils_mod.get_submodules('test_package') - expected = ['', '', 'double'] # Empty strings for edge cases + result = utils_mod.get_submodules("test_package") + expected = ["", "", "double"] # Empty strings for edge cases assert len(result) == 3 except IndexError: pytest.skip("Function doesn't handle edge case module names gracefully") @@ -5247,95 +5985,99 @@ def test_get_submodules_empty_name_parts(): def test_get_submodules_package_without_path_attribute(): """ Test behavior when package doesn't have __path__ attribute. - + Returns: None: Asserts AttributeError is raised for packages without __path__. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - del mock_package.__path__ - - with patch('importlib.import_module', return_value=mock_package): + del mock_package.__path__ + + with patch("importlib.import_module", return_value=mock_package): with pytest.raises(AttributeError): - utils_mod.get_submodules('test_package') + utils_mod.get_submodules("test_package") def test_get_submodules_pkgutil_walk_packages_exception(): """ Test behavior when pkgutil.walk_packages raises an exception. - + Returns: None: Asserts exceptions from pkgutil.walk_packages are properly handled. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', side_effect=ImportError("Mock error")): + mock_package.__path__ = ["/fake/path"] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", side_effect=ImportError("Mock error")): with pytest.raises(ImportError): - utils_mod.get_submodules('test_package') + utils_mod.get_submodules("test_package") def test_get_submodules_mixed_module_types(): """ Test with a mix of different module types and names. - + Returns: None: Asserts function correctly processes various submodule patterns. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_base', False), # Should be filtered out - (None, 'module_parser', False), # Normal case - (None, 'module_test_case', False), # Multiple underscores - (None, 'module_simple', False), # Simple case - (None, 'module_another_base', False), # Contains 'base' but not exactly 'base' + (None, "module_base", False), # Should be filtered out + (None, "module_parser", False), # Normal case + (None, "module_test_case", False), # Multiple underscores + (None, "module_simple", False), # Simple case + (None, "module_another_base", False), # Contains 'base' but not exactly 'base' ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - - expected = ['parser', 'testcase', 'simple', 'anotherbase'] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + + expected = ["parser", "testcase", "simple", "anotherbase"] assert result == expected - assert 'base' not in result + assert "base" not in result def test_get_submodules_large_number_of_submodules(): """ Test performance and correctness with a large number of submodules. - + Returns: None: Asserts function handles large numbers of submodules correctly. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [] expected_results = [] - + for i in range(100): - module_name = f'module_test{i}' + module_name = f"module_test{i}" mock_submodules.append((None, module_name, False)) - expected_results.append(f'test{i}') - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - + expected_results.append(f"test{i}") + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + assert len(result) == 100 assert result == expected_results @@ -5343,45 +6085,46 @@ def test_get_submodules_large_number_of_submodules(): def test_get_submodules_string_input_validation(): """ Test input validation for package_name parameter. - + Returns: None: Asserts function handles invalid input types but may not validate properly. """ import utils.utils as utils_mod - + with pytest.raises((TypeError, AttributeError)): utils_mod.get_submodules(None) - + with pytest.raises((TypeError, AttributeError)): utils_mod.get_submodules(123) - + with pytest.raises((TypeError, AttributeError)): - utils_mod.get_submodules(['list', 'input']) + utils_mod.get_submodules(["list", "input"]) def test_get_submodules_return_type_consistency(): """ Test that function always returns a list, even in edge cases. - + Returns: None: Asserts return type is always a list. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=[]): - result = utils_mod.get_submodules('test_package') + mock_package.__path__ = ["/fake/path"] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=[]): + result = utils_mod.get_submodules("test_package") assert isinstance(result, list) assert len(result) == 0 - - mock_submodules = [(None, 'module_base', False)] - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') + + mock_submodules = [(None, "module_base", False)] + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") assert isinstance(result, list) assert len(result) == 0 @@ -5389,115 +6132,120 @@ def test_get_submodules_return_type_consistency(): def test_get_submodules_special_characters_in_names(): """ Test handling of special characters in submodule names. - + Returns: None: Asserts function processes special characters in names correctly. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_test-case', False), - (None, 'module_test.case', False), - (None, 'module_test123', False), + (None, "module_test-case", False), + (None, "module_test.case", False), + (None, "module_test123", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - - expected = ['test-case', 'test.case', 'test123'] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + + expected = ["test-case", "test.case", "test123"] assert result == expected def test_get_submodules_imports_isolation(): """ Test that imports are properly isolated and don't affect global state. - + Returns: None: Asserts function imports don't pollute global namespace. """ import sys - import utils.utils as utils_mod from unittest.mock import MagicMock, patch - - original_importlib = sys.modules.get('importlib') - original_pkgutil = sys.modules.get('pkgutil') - + + import utils.utils as utils_mod + + original_importlib = sys.modules.get("importlib") + original_pkgutil = sys.modules.get("pkgutil") + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - mock_submodules = [(None, 'module_test', False)] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - - assert sys.modules.get('importlib') == original_importlib - assert sys.modules.get('pkgutil') == original_pkgutil - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [(None, "module_test", False)] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + + assert sys.modules.get("importlib") == original_importlib + assert sys.modules.get("pkgutil") == original_pkgutil + assert isinstance(result, list) - assert result == ['test'] + assert result == ["test"] def test_get_submodules_unicode_names(): """ Test handling of Unicode characters in package and submodule names. - + Returns: None: Asserts function handles Unicode characters appropriately. """ from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_tëst', False), - (None, 'module_测试', False), - (None, 'module_тест', False), + (None, "module_tëst", False), + (None, "module_测试", False), + (None, "module_тест", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - - expected = ['tëst', '测试', 'тест'] + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + + expected = ["tëst", "测试", "тест"] assert result == expected def test_get_submodules_docstring_verification(): """ Test that function behavior matches its docstring description. - + Returns: None: Asserts function behavior aligns with documented purpose. """ - import utils.utils as utils_mod from unittest.mock import MagicMock, patch - + + import utils.utils as utils_mod + assert utils_mod.get_submodules.__doc__ is not None assert "List all submodules for a target package" in utils_mod.get_submodules.__doc__ - + mock_package = MagicMock() - mock_package.__path__ = ['/fake/path'] - + mock_package.__path__ = ["/fake/path"] + mock_submodules = [ - (None, 'module_submodule1', False), - (None, 'module_submodule2', False), + (None, "module_submodule1", False), + (None, "module_submodule2", False), ] - - with patch('importlib.import_module', return_value=mock_package): - with patch('pkgutil.walk_packages', return_value=mock_submodules): - result = utils_mod.get_submodules('test_package') - + + with patch("importlib.import_module", return_value=mock_package): + with patch("pkgutil.walk_packages", return_value=mock_submodules): + result = utils_mod.get_submodules("test_package") + assert isinstance(result, list) - assert 'submodule1' in result - assert 'submodule2' in result - + assert "submodule1" in result + assert "submodule2" in result + + # ============================================================================= # TESTS FOR EMPTY WORKLOAD # ============================================================================= @@ -5536,69 +6284,74 @@ Large datasets with performance implications Different input path types """ + def test_is_workload_empty_valid_data_file(tmp_path): """ Test is_workload_empty with a valid pmc_perf.csv file containing data. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles valid data files without errors. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" valid_data = """Kernel_Name,GPU_ID,Counter1,Counter2 kernel1,0,100,200 kernel2,1,150,250 kernel3,0,120,220""" pmc_perf_file.write_text(valid_data) - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 0 def test_is_workload_empty_file_with_nan_values(tmp_path): """ Test is_workload_empty with pmc_perf.csv containing NaN values. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function detects and reports empty cells after dropping NaN. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" nan_data = """Kernel_Name,GPU_ID,Counter1,Counter2 ,,NaN, ,NaN,,NaN NaN,,,""" pmc_perf_file.write_text(nan_data) - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 1 error_args = console_error_calls[0][0] assert "profilingFound empty cells" in error_args[0] @@ -5609,27 +6362,29 @@ NaN,,,""" def test_is_workload_empty_completely_empty_csv(tmp_path): """ Test is_workload_empty with completely empty pmc_perf.csv file. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function detects empty CSV file. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" pmc_perf_file.write_text("") - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): try: utils_mod.is_workload_empty(str(workload_dir)) except Exception: @@ -5639,30 +6394,32 @@ def test_is_workload_empty_completely_empty_csv(tmp_path): def test_is_workload_empty_headers_only_csv(tmp_path): """ Test is_workload_empty with CSV containing only headers. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function detects CSV with headers but no data. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" headers_only = "Kernel_Name,GPU_ID,Counter1,Counter2" pmc_perf_file.write_text(headers_only) - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 1 error_args = console_error_calls[0][0] assert "profilingFound empty cells" in error_args[0] @@ -5671,26 +6428,28 @@ def test_is_workload_empty_headers_only_csv(tmp_path): def test_is_workload_empty_no_pmc_perf_file(tmp_path): """ Test is_workload_empty when pmc_perf.csv file doesn't exist. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function detects missing profiling data file. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 1 error_args = console_error_calls[0][0] assert error_args[0] == "analysis" @@ -5700,20 +6459,22 @@ def test_is_workload_empty_no_pmc_perf_file(tmp_path): def test_is_workload_empty_nonexistent_directory(): """ Test is_workload_empty with nonexistent directory path. - + Returns: None: Asserts function handles nonexistent directories. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty("/nonexistent/path") - + assert len(console_error_calls) == 1 error_args = console_error_calls[0][0] assert error_args[0] == "analysis" @@ -5723,31 +6484,33 @@ def test_is_workload_empty_nonexistent_directory(): def test_is_workload_empty_malformed_csv(tmp_path): """ Test is_workload_empty with malformed CSV that causes pandas read error. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles pandas CSV reading errors gracefully. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" malformed_data = """Kernel_Name,GPU_ID,Counter1,Counter2 kernel1,0,100,200,extra_column_data kernel2,1,150 incomplete_row""" pmc_perf_file.write_text(malformed_data) - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): try: utils_mod.is_workload_empty(str(workload_dir)) except Exception: @@ -5757,19 +6520,20 @@ incomplete_row""" def test_is_workload_empty_mixed_valid_invalid_data(tmp_path): """ Test is_workload_empty with CSV containing mix of valid and invalid (NaN) data. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles mixed data correctly. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" mixed_data = """Kernel_Name,GPU_ID,Counter1,Counter2 kernel1,0,100,200 @@ -5779,31 +6543,33 @@ kernel3,1,120, pmc_perf_file.write_text(mixed_data) console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 0 def test_is_workload_empty_large_dataset_with_nans(tmp_path): """ Test is_workload_empty with large dataset that becomes empty after dropping NaNs. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function correctly processes large datasets. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" headers = "Kernel_Name,GPU_ID,Counter1,Counter2\n" nan_rows = [] @@ -5811,14 +6577,15 @@ def test_is_workload_empty_large_dataset_with_nans(tmp_path): nan_rows.append("NaN,NaN,NaN,NaN") large_nan_data = headers + "\n".join(nan_rows) pmc_perf_file.write_text(large_nan_data) - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 1 error_args = console_error_calls[0][0] assert "profilingFound empty cells" in error_args[0] @@ -5827,97 +6594,103 @@ def test_is_workload_empty_large_dataset_with_nans(tmp_path): def test_is_workload_empty_unicode_content(tmp_path): """ Test is_workload_empty with CSV containing Unicode characters. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles Unicode content correctly. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" unicode_data = """Kernel_Name,GPU_ID,Counter1,Counter2 kernel_测试,0,100,200 kernel_тест,1,150,250 kernel_tëst,0,120,220""" - pmc_perf_file.write_text(unicode_data, encoding='utf-8') - + pmc_perf_file.write_text(unicode_data, encoding="utf-8") + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 0 def test_is_workload_empty_special_path_characters(tmp_path): """ Test is_workload_empty with directory paths containing special characters. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles special characters in paths. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload-test_dir.with.dots" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" valid_data = """Kernel_Name,GPU_ID,Counter1,Counter2 kernel1,0,100,200""" pmc_perf_file.write_text(valid_data) - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 0 def test_is_workload_empty_csv_read_permission_error(tmp_path): """ Test is_workload_empty when CSV file exists but cannot be read due to permissions. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function handles file permission errors. """ - import utils.utils as utils_mod - from unittest.mock import patch import os - - if os.name == 'nt': + from unittest.mock import patch + + import utils.utils as utils_mod + + if os.name == "nt": pytest.skip("Permission test not applicable on Windows") - + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" pmc_perf_file.write_text("Kernel_Name,GPU_ID\nkernel1,0") pmc_perf_file.chmod(0o000) # Remove all permissions - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - + try: - with patch('utils.utils.console_error', side_effect=mock_console_error): + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) except PermissionError: pass @@ -5928,20 +6701,22 @@ def test_is_workload_empty_csv_read_permission_error(tmp_path): def test_is_workload_empty_string_path_input(): """ Test is_workload_empty with string path input vs pathlib.Path. - + Returns: None: Asserts function handles different path input types. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty("/nonexistent/string/path") - + assert len(console_error_calls) == 1 error_args = console_error_calls[0][0] assert error_args[0] == "analysis" @@ -5951,29 +6726,31 @@ def test_is_workload_empty_string_path_input(): def test_is_workload_empty_console_error_string_formatting(tmp_path): """ Test is_workload_empty string formatting in console_error messages. - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts console_error messages are properly formatted. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" pmc_perf_file.write_text("Kernel_Name,GPU_ID\nNaN,NaN") - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.is_workload_empty(str(workload_dir)) - + assert len(console_error_calls) == 1 error_args = console_error_calls[0][0] expected_path = str(workload_dir / "pmc_perf.csv") @@ -5985,60 +6762,63 @@ def test_is_workload_empty_console_error_string_formatting(tmp_path): def test_is_workload_empty_function_return_value(tmp_path): """ Test that is_workload_empty function return behavior (implicitly returns None). - + Args: tmp_path (pathlib.Path): Temporary directory for test files. - + Returns: None: Asserts function return value consistency. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + workload_dir = tmp_path / "workload" workload_dir.mkdir() - + pmc_perf_file = workload_dir / "pmc_perf.csv" pmc_perf_file.write_text("Kernel_Name,GPU_ID\nkernel1,0") - - with patch('utils.utils.console_error'): + + with patch("utils.utils.console_error"): result = utils_mod.is_workload_empty(str(workload_dir)) - + assert result is None - + workload_dir2 = tmp_path / "workload2" workload_dir2.mkdir() - - with patch('utils.utils.console_error'): + + with patch("utils.utils.console_error"): result2 = utils_mod.is_workload_empty(str(workload_dir2)) - + assert result2 is None def test_is_workload_empty_pandas_import_dependency(): """ Test is_workload_empty dependency on pandas module. - + Returns: None: Asserts function properly uses pandas functionality. """ + from unittest.mock import MagicMock, patch + import utils.utils as utils_mod - from unittest.mock import patch, MagicMock - + mock_pandas = MagicMock() mock_df = MagicMock() mock_df.dropna.return_value.empty = False mock_pandas.read_csv.return_value = mock_df - - with patch.dict('sys.modules', {'pandas': mock_pandas}): - with patch('utils.utils.pd', mock_pandas): - with patch('utils.utils.console_error'): - with patch('pathlib.Path.is_file', return_value=True): + + with patch.dict("sys.modules", {"pandas": mock_pandas}): + with patch("utils.utils.pd", mock_pandas): + with patch("utils.utils.console_error"): + with patch("pathlib.Path.is_file", return_value=True): utils_mod.is_workload_empty("/test/path") - + mock_pandas.read_csv.assert_called_once() mock_df.dropna.assert_called_once() - + + # ============================================================================= # TESTS FOR LOCAL ENCODING FUNCTION # ============================================================================= @@ -6076,26 +6856,29 @@ Comprehensive error path coverage Module import dependencies """ + def test_set_locale_encoding_successful_c_utf8(): """ Test set_locale_encoding when C.UTF-8 locale is available and can be set successfully. - + Returns: None: Asserts function sets C.UTF-8 locale without errors. """ - import utils.utils as utils_mod from unittest.mock import patch - + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.return_value = None - + utils_mod.set_locale_encoding() - + mock_setlocale.assert_called_once_with(locale.LC_ALL, "C.UTF-8") assert len(console_error_calls) == 0 @@ -6103,181 +6886,210 @@ def test_set_locale_encoding_successful_c_utf8(): def test_set_locale_encoding_c_utf8_fails_fallback_to_current_utf8(): """ Test set_locale_encoding when C.UTF-8 fails but current locale is UTF-8 based. - + Returns: None: Asserts function falls back to current UTF-8 locale successfully. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = [locale.Error("C.UTF-8 not available"), None] - mock_getdefaultlocale.return_value = ('en_US', 'UTF-8') - + mock_getdefaultlocale.return_value = ("en_US", "UTF-8") + utils_mod.set_locale_encoding() - + assert mock_setlocale.call_count == 2 mock_setlocale.assert_any_call(locale.LC_ALL, "C.UTF-8") - mock_setlocale.assert_any_call(locale.LC_ALL, 'en_US') + mock_setlocale.assert_any_call(locale.LC_ALL, "en_US") assert len(console_error_calls) == 0 def test_set_locale_encoding_c_utf8_fails_fallback_also_fails(): """ Test set_locale_encoding when both C.UTF-8 and fallback locale fail. - + Returns: None: Asserts function calls console_error when fallback locale fails. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): fallback_error = locale.Error("Fallback locale failed") mock_setlocale.side_effect = [ - locale.Error("C.UTF-8 not available"), - fallback_error + locale.Error("C.UTF-8 not available"), + fallback_error, ] - mock_getdefaultlocale.return_value = ('en_US', 'UTF-8') - + mock_getdefaultlocale.return_value = ("en_US", "UTF-8") + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 2 - assert "Failed to set locale to the current UTF-8-based locale." in console_error_calls[0][0][0] - assert console_error_calls[0][1]['exit'] == False + assert ( + "Failed to set locale to the current UTF-8-based locale." + in console_error_calls[0][0][0] + ) + assert console_error_calls[0][1]["exit"] == False assert console_error_calls[1][0][0] == fallback_error def test_set_locale_encoding_no_utf8_locale_available(): """ Test set_locale_encoding when no UTF-8 locale is available. - + Returns: None: Asserts function calls console_error when no UTF-8 locale found. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = locale.Error("C.UTF-8 not available") - mock_getdefaultlocale.return_value = ('en_US', 'ISO-8859-1') - + mock_getdefaultlocale.return_value = ("en_US", "ISO-8859-1") + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 1 - assert "Please ensure that a UTF-8-based locale is available on your system." in console_error_calls[0][0][0] - assert console_error_calls[0][1]['exit'] == False + assert ( + "Please ensure that a UTF-8-based locale is available on your system." + in console_error_calls[0][0][0] + ) + assert console_error_calls[0][1]["exit"] == False def test_set_locale_encoding_getdefaultlocale_returns_none(): """ Test set_locale_encoding when getdefaultlocale returns None. - + Returns: None: Asserts function handles None return from getdefaultlocale. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = locale.Error("C.UTF-8 not available") mock_getdefaultlocale.return_value = None - + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 1 - assert "Please ensure that a UTF-8-based locale is available on your system." in console_error_calls[0][0][0] + assert ( + "Please ensure that a UTF-8-based locale is available on your system." + in console_error_calls[0][0][0] + ) def test_set_locale_encoding_getdefaultlocale_partial_none(): """ Test set_locale_encoding when getdefaultlocale returns partial None values. - + Returns: None: Asserts function handles partial None values from getdefaultlocale. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = locale.Error("C.UTF-8 not available") - - mock_getdefaultlocale.return_value = ('en_US', None) - + + mock_getdefaultlocale.return_value = ("en_US", None) + try: utils_mod.set_locale_encoding() except TypeError as e: if "argument of type 'NoneType' is not iterable" in str(e): - pytest.skip("Function doesn't handle None encoding gracefully - needs null check") + pytest.skip( + "Function doesn't handle None encoding gracefully - needs null check" + ) else: raise - + assert len(console_error_calls) == 1 - assert "Please ensure that a UTF-8-based locale is available on your system." in console_error_calls[0][0][0] + assert ( + "Please ensure that a UTF-8-based locale is available on your system." + in console_error_calls[0][0][0] + ) def test_set_locale_encoding_utf8_case_variations(): """ Test set_locale_encoding with various UTF-8 case variations in encoding. - + Returns: None: Asserts function handles different UTF-8 case formats. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - - utf8_variations = ['UTF-8', 'utf-8', 'UTF8', 'utf8'] - + from unittest.mock import patch + + import utils.utils as utils_mod + + utf8_variations = ["UTF-8", "utf-8", "UTF8", "utf8"] + for utf8_variant in utf8_variations: console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): - mock_setlocale.side_effect = [locale.Error("C.UTF-8 not available"), None] - mock_getdefaultlocale.return_value = ('en_US', utf8_variant) - + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): + mock_setlocale.side_effect = [ + locale.Error("C.UTF-8 not available"), + None, + ] + mock_getdefaultlocale.return_value = ("en_US", utf8_variant) + utils_mod.set_locale_encoding() - - if 'UTF-8' in utf8_variant: + + if "UTF-8" in utf8_variant: assert len(console_error_calls) == 0 assert mock_setlocale.call_count == 2 else: @@ -6287,53 +7099,63 @@ def test_set_locale_encoding_utf8_case_variations(): def test_set_locale_encoding_empty_encoding(): """ Test set_locale_encoding when getdefaultlocale returns empty encoding. - + Returns: None: Asserts function handles empty encoding string. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = locale.Error("C.UTF-8 not available") - mock_getdefaultlocale.return_value = ('en_US', '') - + mock_getdefaultlocale.return_value = ("en_US", "") + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 1 - assert "Please ensure that a UTF-8-based locale is available on your system." in console_error_calls[0][0][0] + assert ( + "Please ensure that a UTF-8-based locale is available on your system." + in console_error_calls[0][0][0] + ) def test_set_locale_encoding_locale_with_utf8_substring(): """ Test set_locale_encoding with encoding that contains UTF-8 as substring. - + Returns: None: Asserts function correctly identifies UTF-8 in encoding names. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = [locale.Error("C.UTF-8 not available"), None] - mock_getdefaultlocale.return_value = ('en_US', 'ISO-8859-1.UTF-8.EXTENDED') - + mock_getdefaultlocale.return_value = ( + "en_US", + "ISO-8859-1.UTF-8.EXTENDED", + ) + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 0 assert mock_setlocale.call_count == 2 @@ -6341,38 +7163,40 @@ def test_set_locale_encoding_locale_with_utf8_substring(): def test_set_locale_encoding_different_locale_error_types(): """ Test set_locale_encoding with different types of locale.Error exceptions. - + Returns: None: Asserts function handles various locale error scenarios. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + error_scenarios = [ "Locale not supported", "Invalid locale specification", "System locale database corrupted", "", # Empty error message ] - + for error_msg in error_scenarios: console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): fallback_error = locale.Error(error_msg) mock_setlocale.side_effect = [ - locale.Error("C.UTF-8 not available"), - fallback_error + locale.Error("C.UTF-8 not available"), + fallback_error, ] - mock_getdefaultlocale.return_value = ('en_US', 'UTF-8') - + mock_getdefaultlocale.return_value = ("en_US", "UTF-8") + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 2 assert console_error_calls[1][0][0] == fallback_error @@ -6380,35 +7204,40 @@ def test_set_locale_encoding_different_locale_error_types(): def test_set_locale_encoding_unusual_locale_names(): """ Test set_locale_encoding with unusual but valid locale names. - + Returns: None: Asserts function handles unusual locale name formats. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + unusual_locales = [ - ('C', 'UTF-8'), - ('POSIX', 'UTF-8'), - ('en_US.UTF-8', 'UTF-8'), - ('zh_CN.UTF-8', 'UTF-8'), - ('', 'UTF-8'), # Empty locale name + ("C", "UTF-8"), + ("POSIX", "UTF-8"), + ("en_US.UTF-8", "UTF-8"), + ("zh_CN.UTF-8", "UTF-8"), + ("", "UTF-8"), # Empty locale name ] - + for locale_name, encoding in unusual_locales: console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): - mock_setlocale.side_effect = [locale.Error("C.UTF-8 not available"), None] + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): + mock_setlocale.side_effect = [ + locale.Error("C.UTF-8 not available"), + None, + ] mock_getdefaultlocale.return_value = (locale_name, encoding) - + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 0 assert mock_setlocale.call_count == 2 mock_setlocale.assert_any_call(locale.LC_ALL, locale_name) @@ -6417,24 +7246,26 @@ def test_set_locale_encoding_unusual_locale_names(): def test_set_locale_encoding_getdefaultlocale_exception(): """ Test set_locale_encoding when getdefaultlocale raises an exception. - + Returns: None: Asserts function handles getdefaultlocale exceptions gracefully. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = locale.Error("C.UTF-8 not available") mock_getdefaultlocale.side_effect = Exception("getdefaultlocale failed") - + try: utils_mod.set_locale_encoding() except Exception: @@ -6444,57 +7275,60 @@ def test_set_locale_encoding_getdefaultlocale_exception(): def test_set_locale_encoding_console_error_parameters(): """ Test set_locale_encoding console_error call parameters are correct. - + Returns: None: Asserts console_error is called with correct parameters. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.side_effect = locale.Error("C.UTF-8 not available") - mock_getdefaultlocale.return_value = ('en_US', 'ISO-8859-1') - + mock_getdefaultlocale.return_value = ("en_US", "ISO-8859-1") + utils_mod.set_locale_encoding() - + assert len(console_error_calls) == 1 args, kwargs = console_error_calls[0] assert len(args) == 1 - assert 'exit' in kwargs - assert kwargs['exit'] == False + assert "exit" in kwargs + assert kwargs["exit"] == False def test_set_locale_encoding_return_value(): """ Test that set_locale_encoding returns None (implicit return). - + Returns: None: Asserts function returns None in all scenarios. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - - with patch('locale.setlocale') as mock_setlocale: - with patch('utils.utils.console_error'): + from unittest.mock import patch + + import utils.utils as utils_mod + + with patch("locale.setlocale") as mock_setlocale: + with patch("utils.utils.console_error"): mock_setlocale.return_value = None - + result = utils_mod.set_locale_encoding() assert result is None - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error'): + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error"): mock_setlocale.side_effect = locale.Error("C.UTF-8 not available") - mock_getdefaultlocale.return_value = ('en_US', 'ISO-8859-1') - + mock_getdefaultlocale.return_value = ("en_US", "ISO-8859-1") + result = utils_mod.set_locale_encoding() assert result is None @@ -6502,83 +7336,88 @@ def test_set_locale_encoding_return_value(): def test_set_locale_encoding_locale_module_import(): """ Test set_locale_encoding dependency on locale module. - + Returns: None: Asserts function properly uses locale module functionality. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale + from unittest.mock import patch + + import utils.utils as utils_mod setlocale_calls = [] getdefaultlocale_calls = [] - + def mock_setlocale(category, locale_name): setlocale_calls.append((category, locale_name)) return None - + def mock_getdefaultlocale(): getdefaultlocale_calls.append(True) - return ('en_US', 'UTF-8') - + return ("en_US", "UTF-8") + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale', side_effect=mock_setlocale): - with patch('locale.getdefaultlocale', side_effect=mock_getdefaultlocale): - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale", side_effect=mock_setlocale): + with patch("locale.getdefaultlocale", side_effect=mock_getdefaultlocale): + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.set_locale_encoding() - + assert len(setlocale_calls) == 1 assert setlocale_calls[0] == (locale.LC_ALL, "C.UTF-8") - assert len(getdefaultlocale_calls) == 0 + assert len(getdefaultlocale_calls) == 0 assert len(console_error_calls) == 0 - + setlocale_calls.clear() getdefaultlocale_calls.clear() console_error_calls.clear() - + def mock_setlocale_with_error(category, locale_name): setlocale_calls.append((category, locale_name)) if locale_name == "C.UTF-8": raise locale.Error("C.UTF-8 not available") return None - - with patch('locale.setlocale', side_effect=mock_setlocale_with_error): - with patch('locale.getdefaultlocale', side_effect=mock_getdefaultlocale): - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale", side_effect=mock_setlocale_with_error): + with patch("locale.getdefaultlocale", side_effect=mock_getdefaultlocale): + with patch("utils.utils.console_error", side_effect=mock_console_error): utils_mod.set_locale_encoding() - + assert len(setlocale_calls) == 2 assert setlocale_calls[0] == (locale.LC_ALL, "C.UTF-8") assert setlocale_calls[1] == (locale.LC_ALL, "en_US") assert len(getdefaultlocale_calls) == 1 assert len(console_error_calls) == 0 - + + def test_set_locale_encoding_multiple_calls(): """ Test set_locale_encoding behavior when called multiple times. - + Returns: None: Asserts function behaves consistently across multiple calls. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale') as mock_setlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): + + with patch("locale.setlocale") as mock_setlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): mock_setlocale.return_value = None - + utils_mod.set_locale_encoding() utils_mod.set_locale_encoding() utils_mod.set_locale_encoding() - + assert mock_setlocale.call_count == 3 assert len(console_error_calls) == 0 @@ -6586,33 +7425,36 @@ def test_set_locale_encoding_multiple_calls(): def test_set_locale_encoding_thread_safety_simulation(): """ Test set_locale_encoding behavior in simulated concurrent scenarios. - + Returns: None: Asserts function handles concurrent-like access patterns. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + call_count = 0 + def side_effect_setlocale(*args, **kwargs): nonlocal call_count call_count += 1 if call_count == 1: raise locale.Error("First call fails") return None - + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - - with patch('locale.setlocale', side_effect=side_effect_setlocale): - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): - mock_getdefaultlocale.return_value = ('en_US', 'UTF-8') - + + with patch("locale.setlocale", side_effect=side_effect_setlocale): + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): + mock_getdefaultlocale.return_value = ("en_US", "UTF-8") + utils_mod.set_locale_encoding() - + assert call_count == 2 assert len(console_error_calls) == 0 @@ -6620,58 +7462,68 @@ def test_set_locale_encoding_thread_safety_simulation(): def test_set_locale_encoding_comprehensive_error_handling(): """ Test set_locale_encoding comprehensive error handling across all code paths. - + Returns: None: Asserts all error paths are properly handled. """ - import utils.utils as utils_mod - from unittest.mock import patch import locale - + from unittest.mock import patch + + import utils.utils as utils_mod + console_error_calls = [] + def mock_console_error(*args, **kwargs): console_error_calls.append((args, kwargs)) - + test_scenarios = [ { - 'name': 'C.UTF-8 success', - 'setlocale_side_effect': [None], - 'getdefaultlocale_return': ('en_US', 'UTF-8'), - 'expected_errors': 0 + "name": "C.UTF-8 success", + "setlocale_side_effect": [None], + "getdefaultlocale_return": ("en_US", "UTF-8"), + "expected_errors": 0, }, { - 'name': 'C.UTF-8 fails, fallback success', - 'setlocale_side_effect': [locale.Error("C.UTF-8 fail"), None], - 'getdefaultlocale_return': ('en_US', 'UTF-8'), - 'expected_errors': 0 + "name": "C.UTF-8 fails, fallback success", + "setlocale_side_effect": [locale.Error("C.UTF-8 fail"), None], + "getdefaultlocale_return": ("en_US", "UTF-8"), + "expected_errors": 0, }, { - 'name': 'Both fail with UTF-8 locale', - 'setlocale_side_effect': [locale.Error("C.UTF-8 fail"), locale.Error("Fallback fail")], - 'getdefaultlocale_return': ('en_US', 'UTF-8'), - 'expected_errors': 2 + "name": "Both fail with UTF-8 locale", + "setlocale_side_effect": [ + locale.Error("C.UTF-8 fail"), + locale.Error("Fallback fail"), + ], + "getdefaultlocale_return": ("en_US", "UTF-8"), + "expected_errors": 2, }, { - 'name': 'No UTF-8 locale available', - 'setlocale_side_effect': [locale.Error("C.UTF-8 fail")], - 'getdefaultlocale_return': ('en_US', 'ISO-8859-1'), - 'expected_errors': 1 + "name": "No UTF-8 locale available", + "setlocale_side_effect": [locale.Error("C.UTF-8 fail")], + "getdefaultlocale_return": ("en_US", "ISO-8859-1"), + "expected_errors": 1, }, ] - + for scenario in test_scenarios: console_error_calls.clear() - - with patch('locale.setlocale') as mock_setlocale: - with patch('locale.getdefaultlocale') as mock_getdefaultlocale: - with patch('utils.utils.console_error', side_effect=mock_console_error): - mock_setlocale.side_effect = scenario['setlocale_side_effect'] - mock_getdefaultlocale.return_value = scenario['getdefaultlocale_return'] - + + with patch("locale.setlocale") as mock_setlocale: + with patch("locale.getdefaultlocale") as mock_getdefaultlocale: + with patch("utils.utils.console_error", side_effect=mock_console_error): + mock_setlocale.side_effect = scenario["setlocale_side_effect"] + mock_getdefaultlocale.return_value = scenario[ + "getdefaultlocale_return" + ] + utils_mod.set_locale_encoding() - - assert len(console_error_calls) == scenario['expected_errors'], f"Failed scenario: {scenario['name']}" - + + assert ( + len(console_error_calls) == scenario["expected_errors"] + ), f"Failed scenario: {scenario['name']}" + + # ============================================================================= # TESTS FOR reverse_multi_index_df_pmc FUNCTION # ============================================================================= @@ -6714,314 +7566,326 @@ Proper DataFrame structure in results Consistent length of returned lists """ + def test_reverse_multi_index_df_pmc_basic_functionality(): """ Test reverse_multi_index_df_pmc with a basic multi-index DataFrame. - + Returns: None: Asserts function correctly decomposes multi-index DataFrame. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [1, 2, 3], - ('file1', 'col2'): [4, 5, 6], - ('file2', 'col1'): [7, 8, 9], - ('file2', 'col3'): [10, 11, 12] + ("file1", "col1"): [1, 2, 3], + ("file1", "col2"): [4, 5, 6], + ("file2", "col1"): [7, 8, 9], + ("file2", "col3"): [10, 11, 12], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 2 assert len(coll_levels) == 2 - assert 'file1' in coll_levels - assert 'file2' in coll_levels - - assert list(dfs[0].columns) == ['col1', 'col2'] - assert list(dfs[0]['col1']) == [1, 2, 3] - assert list(dfs[0]['col2']) == [4, 5, 6] - - assert list(dfs[1].columns) == ['col1', 'col3'] - assert list(dfs[1]['col1']) == [7, 8, 9] - assert list(dfs[1]['col3']) == [10, 11, 12] + assert "file1" in coll_levels + assert "file2" in coll_levels + + assert list(dfs[0].columns) == ["col1", "col2"] + assert list(dfs[0]["col1"]) == [1, 2, 3] + assert list(dfs[0]["col2"]) == [4, 5, 6] + + assert list(dfs[1].columns) == ["col1", "col3"] + assert list(dfs[1]["col1"]) == [7, 8, 9] + assert list(dfs[1]["col3"]) == [10, 11, 12] + def test_reverse_multi_index_df_pmc_empty_dataframe(): """ Test reverse_multi_index_df_pmc with empty multi-index DataFrame. - + Returns: None: Asserts function handles empty DataFrames correctly. """ - import utils.utils as utils_mod import pandas as pd - - columns = pd.MultiIndex.from_tuples([('file1', 'col1'), ('file1', 'col2')]) + + import utils.utils as utils_mod + + columns = pd.MultiIndex.from_tuples([("file1", "col1"), ("file1", "col2")]) df = pd.DataFrame(columns=columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 1 assert len(coll_levels) == 1 - assert coll_levels[0] == 'file1' + assert coll_levels[0] == "file1" assert len(dfs[0]) == 0 - assert list(dfs[0].columns) == ['col1', 'col2'] + assert list(dfs[0].columns) == ["col1", "col2"] def test_reverse_multi_index_df_pmc_single_column_per_level(): """ Test reverse_multi_index_df_pmc with single column per level. - + Returns: None: Asserts function handles single column per level correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('level1', 'col1'): [1, 2, 3], - ('level2', 'col1'): [4, 5, 6], - ('level3', 'col1'): [7, 8, 9] + ("level1", "col1"): [1, 2, 3], + ("level2", "col1"): [4, 5, 6], + ("level3", "col1"): [7, 8, 9], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 3 assert len(coll_levels) == 3 - assert set(coll_levels) == {'level1', 'level2', 'level3'} - + assert set(coll_levels) == {"level1", "level2", "level3"} + for i, df_result in enumerate(dfs): assert len(df_result.columns) == 1 - assert df_result.columns[0] == 'col1' + assert df_result.columns[0] == "col1" assert len(df_result) == 3 def test_reverse_multi_index_df_pmc_uneven_column_distribution(): """ Test reverse_multi_index_df_pmc with uneven column distribution across levels. - + Returns: None: Asserts function handles uneven column distributions correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [1, 2, 3], - ('file1', 'col2'): [4, 5, 6], - ('file1', 'col3'): [7, 8, 9], - ('file2', 'col1'): [10, 11, 12], - ('file3', 'col1'): [13, 14, 15], - ('file3', 'col2'): [16, 17, 18], + ("file1", "col1"): [1, 2, 3], + ("file1", "col2"): [4, 5, 6], + ("file1", "col3"): [7, 8, 9], + ("file2", "col1"): [10, 11, 12], + ("file3", "col1"): [13, 14, 15], + ("file3", "col2"): [16, 17, 18], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 3 assert len(coll_levels) == 3 - assert set(coll_levels) == {'file1', 'file2', 'file3'} - - file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file1') + assert set(coll_levels) == {"file1", "file2", "file3"} + + file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file1") assert len(file1_df.columns) == 3 - - file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file2') + + file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file2") assert len(file2_df.columns) == 1 - - file3_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file3') + + file3_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file3") assert len(file3_df.columns) == 2 def test_reverse_multi_index_df_pmc_duplicate_level_names(): """ Test reverse_multi_index_df_pmc with duplicate level names (should handle unique() correctly). - + Returns: None: Asserts function handles duplicate level names correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [1, 2, 3], - ('file1', 'col2'): [4, 5, 6], - ('file1', 'col3'): [7, 8, 9], + ("file1", "col1"): [1, 2, 3], + ("file1", "col2"): [4, 5, 6], + ("file1", "col3"): [7, 8, 9], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 1 assert len(coll_levels) == 1 - assert coll_levels[0] == 'file1' + assert coll_levels[0] == "file1" assert len(dfs[0].columns) == 3 - assert list(dfs[0].columns) == ['col1', 'col2', 'col3'] + assert list(dfs[0].columns) == ["col1", "col2", "col3"] def test_reverse_multi_index_df_pmc_mixed_data_types(): """ Test reverse_multi_index_df_pmc with mixed data types in columns. - + Returns: None: Asserts function handles mixed data types correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'integers'): [1, 2, 3], - ('file1', 'floats'): [1.1, 2.2, 3.3], - ('file1', 'strings'): ['a', 'b', 'c'], - ('file2', 'booleans'): [True, False, True], - ('file2', 'mixed'): [1, 'text', 3.14], + ("file1", "integers"): [1, 2, 3], + ("file1", "floats"): [1.1, 2.2, 3.3], + ("file1", "strings"): ["a", "b", "c"], + ("file2", "booleans"): [True, False, True], + ("file2", "mixed"): [1, "text", 3.14], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 2 assert len(coll_levels) == 2 - - file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file1') - assert file1_df['integers'].dtype == 'int64' - assert file1_df['floats'].dtype == 'float64' - assert file1_df['strings'].dtype == 'object' - - file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file2') - assert file2_df['booleans'].dtype == 'bool' - assert file2_df['mixed'].dtype == 'object' + + file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file1") + assert file1_df["integers"].dtype == "int64" + assert file1_df["floats"].dtype == "float64" + assert file1_df["strings"].dtype == "object" + + file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file2") + assert file2_df["booleans"].dtype == "bool" + assert file2_df["mixed"].dtype == "object" def test_reverse_multi_index_df_pmc_nan_values(): """ Test reverse_multi_index_df_pmc with NaN values in data. - + Returns: None: Asserts function handles NaN values correctly. """ - import utils.utils as utils_mod - import pandas as pd import numpy as np - + import pandas as pd + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [1, np.nan, 3], - ('file1', 'col2'): [np.nan, 5, 6], - ('file2', 'col1'): [7, 8, np.nan], + ("file1", "col1"): [1, np.nan, 3], + ("file1", "col2"): [np.nan, 5, 6], + ("file2", "col1"): [7, 8, np.nan], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 2 - - file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file1') - assert pd.isna(file1_df.iloc[1, 0]) + + file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file1") + assert pd.isna(file1_df.iloc[1, 0]) assert pd.isna(file1_df.iloc[0, 1]) - - file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file2') + + file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file2") assert pd.isna(file2_df.iloc[2, 0]) def test_reverse_multi_index_df_pmc_special_column_names(): """ Test reverse_multi_index_df_pmc with special characters in column names. - + Returns: None: Asserts function handles special characters in column names. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file-1', 'col_1'): [1, 2, 3], - ('file-1', 'col.2'): [4, 5, 6], - ('file 2', 'col@3'): [7, 8, 9], - ('file 2', 'col#4'): [10, 11, 12], + ("file-1", "col_1"): [1, 2, 3], + ("file-1", "col.2"): [4, 5, 6], + ("file 2", "col@3"): [7, 8, 9], + ("file 2", "col#4"): [10, 11, 12], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 2 - assert 'file-1' in coll_levels - assert 'file 2' in coll_levels - - file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file-1') - assert 'col_1' in file1_df.columns - assert 'col.2' in file1_df.columns - - file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file 2') - assert 'col@3' in file2_df.columns - assert 'col#4' in file2_df.columns + assert "file-1" in coll_levels + assert "file 2" in coll_levels + + file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file-1") + assert "col_1" in file1_df.columns + assert "col.2" in file1_df.columns + + file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file 2") + assert "col@3" in file2_df.columns + assert "col#4" in file2_df.columns def test_reverse_multi_index_df_pmc_numeric_level_names(): """ Test reverse_multi_index_df_pmc with numeric level names. - + Returns: None: Asserts function handles numeric level names correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - (1, 'col1'): [1, 2, 3], - (1, 'col2'): [4, 5, 6], - (2, 'col1'): [7, 8, 9], - (3.5, 'col1'): [10, 11, 12], + (1, "col1"): [1, 2, 3], + (1, "col2"): [4, 5, 6], + (2, "col1"): [7, 8, 9], + (3.5, "col1"): [10, 11, 12], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 3 assert set(coll_levels) == {1, 2, 3.5} - + for level in [1, 2, 3.5]: level_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == level) assert len(level_df.columns) >= 1 - assert 'col1' in level_df.columns + assert "col1" in level_df.columns def test_reverse_multi_index_df_pmc_large_dataframe(): """ Test reverse_multi_index_df_pmc with large DataFrame. - + Returns: None: Asserts function handles large DataFrames efficiently. """ - import utils.utils as utils_mod - import pandas as pd import numpy as np - + import pandas as pd + + import utils.utils as utils_mod + num_rows = 1000 num_levels = 5 num_cols_per_level = 10 - + data = {} for level in range(num_levels): for col in range(num_cols_per_level): - data[(f'level_{level}', f'col_{col}')] = np.random.randint(0, 100, num_rows) - + data[(f"level_{level}", f"col_{col}")] = np.random.randint(0, 100, num_rows) + df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == num_levels assert len(coll_levels) == num_levels - + for i, df_result in enumerate(dfs): assert len(df_result) == num_rows assert len(df_result.columns) == num_cols_per_level @@ -7030,50 +7894,52 @@ def test_reverse_multi_index_df_pmc_large_dataframe(): def test_reverse_multi_index_df_pmc_three_level_index(): """ Test reverse_multi_index_df_pmc with three-level MultiIndex (should still work). - + Returns: None: Asserts function handles three-level MultiIndex correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'group1', 'col1'): [1, 2, 3], - ('file1', 'group1', 'col2'): [4, 5, 6], - ('file1', 'group2', 'col1'): [7, 8, 9], - ('file2', 'group1', 'col1'): [10, 11, 12], + ("file1", "group1", "col1"): [1, 2, 3], + ("file1", "group1", "col2"): [4, 5, 6], + ("file1", "group2", "col1"): [7, 8, 9], + ("file2", "group1", "col1"): [10, 11, 12], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 2 - assert set(coll_levels) == {'file1', 'file2'} - - file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file1') + assert set(coll_levels) == {"file1", "file2"} + + file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file1") assert len(file1_df.columns.levels) == 2 def test_reverse_multi_index_df_pmc_return_type_validation(): """ Test reverse_multi_index_df_pmc return types are correct. - + Returns: None: Asserts function returns correct types. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [1, 2, 3], - ('file2', 'col1'): [4, 5, 6], + ("file1", "col1"): [1, 2, 3], + ("file2", "col1"): [4, 5, 6], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert isinstance(dfs, list) assert isinstance(coll_levels, list) assert all(isinstance(df, pd.DataFrame) for df in dfs) @@ -7083,169 +7949,176 @@ def test_reverse_multi_index_df_pmc_return_type_validation(): def test_reverse_multi_index_df_pmc_column_order_preservation(): """ Test reverse_multi_index_df_pmc preserves column order within levels. - + Returns: None: Asserts function preserves column order correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'z_col'): [1, 2, 3], - ('file1', 'a_col'): [4, 5, 6], - ('file1', 'm_col'): [7, 8, 9], - ('file2', 'b_col'): [10, 11, 12], - ('file2', 'y_col'): [13, 14, 15], + ("file1", "z_col"): [1, 2, 3], + ("file1", "a_col"): [4, 5, 6], + ("file1", "m_col"): [7, 8, 9], + ("file2", "b_col"): [10, 11, 12], + ("file2", "y_col"): [13, 14, 15], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - - file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file1') - assert list(file1_df.columns) == ['z_col', 'a_col', 'm_col'] - - file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file2') - assert list(file2_df.columns) == ['b_col', 'y_col'] + + file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file1") + assert list(file1_df.columns) == ["z_col", "a_col", "m_col"] + + file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file2") + assert list(file2_df.columns) == ["b_col", "y_col"] def test_reverse_multi_index_df_pmc_index_preservation(): """ Test reverse_multi_index_df_pmc preserves DataFrame index. - + Returns: None: Asserts function preserves original DataFrame index. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [1, 2, 3], - ('file1', 'col2'): [4, 5, 6], - ('file2', 'col1'): [7, 8, 9], + ("file1", "col1"): [1, 2, 3], + ("file1", "col2"): [4, 5, 6], + ("file2", "col1"): [7, 8, 9], } - df = pd.DataFrame(data, index=['row_a', 'row_b', 'row_c']) + df = pd.DataFrame(data, index=["row_a", "row_b", "row_c"]) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + for df_result in dfs: - assert list(df_result.index) == ['row_a', 'row_b', 'row_c'] + assert list(df_result.index) == ["row_a", "row_b", "row_c"] def test_reverse_multi_index_df_pmc_memory_efficiency(): """ Test reverse_multi_index_df_pmc memory usage patterns. - + Returns: None: Asserts function doesn't create unnecessary copies. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [1, 2, 3], - ('file2', 'col1'): [4, 5, 6], + ("file1", "col1"): [1, 2, 3], + ("file2", "col1"): [4, 5, 6], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + original_memory = df.memory_usage(deep=True).sum() - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + total_result_memory = sum(df.memory_usage(deep=True).sum() for df in dfs) - + assert total_result_memory < original_memory * 3 def test_reverse_multi_index_df_pmc_edge_case_single_row(): """ Test reverse_multi_index_df_pmc with single row DataFrame. - + Returns: None: Asserts function handles single row DataFrames correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'col1'): [100], - ('file1', 'col2'): [200], - ('file2', 'col1'): [300], + ("file1", "col1"): [100], + ("file1", "col2"): [200], + ("file2", "col1"): [300], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + dfs, coll_levels = utils_mod.reverse_multi_index_df_pmc(df) - + assert len(dfs) == 2 assert len(coll_levels) == 2 - + for df_result in dfs: assert len(df_result) == 1 - - file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file1') - assert file1_df.iloc[0]['col1'] == 100 - assert file1_df.iloc[0]['col2'] == 200 - - file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == 'file2') - assert file2_df.iloc[0]['col1'] == 300 - + + file1_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file1") + assert file1_df.iloc[0]["col1"] == 100 + assert file1_df.iloc[0]["col2"] == 200 + + file2_df = next(df for i, df in enumerate(dfs) if coll_levels[i] == "file2") + assert file2_df.iloc[0]["col1"] == 300 + + # ============================================================================= # TESTS FOR merge_counters_spatial_multiplex FUNCTION # ============================================================================= + def test_merge_counters_spatial_multiplex_basic_functionality(): """ Test merge_counters_spatial_multiplex with basic multi-index DataFrame. - + Returns: None: Asserts function correctly merges counter values for spatial multiplexing. """ - import utils.utils as utils_mod - import pandas as pd import numpy as np - + import pandas as pd + + import utils.utils as utils_mod + data = { - ('file1', 'Dispatch_ID'): [1, 2, 3], - ('file1', 'GPU_ID'): [0, 0, 1], - ('file1', 'Grid_Size'): [64, 128, 256], - ('file1', 'Workgroup_Size'): [16, 32, 64], - ('file1', 'LDS_Per_Workgroup'): [1024, 2048, 4096], - ('file1', 'Scratch_Per_Workitem'): [0, 0, 0], - ('file1', 'Arch_VGPR'): [32, 64, 96], - ('file1', 'Accum_VGPR'): [0, 0, 0], - ('file1', 'SGPR'): [16, 32, 48], - ('file1', 'Wave_Size'): [64, 64, 64], - ('file1', 'Correlation_ID'): [1001, 1002, 1003], - ('file1', 'Kernel_ID'): [501, 502, 503], - ('file1', 'Kernel_Name'): ['kernel_a', 'kernel_a', 'kernel_b'], - ('file1', 'Start_Timestamp'): [1000, 1100, 2000], - ('file1', 'End_Timestamp'): [1200, 1300, 2500], - ('file1', 'Counter1'): [100, 200, 300], - ('file2', 'Dispatch_ID'): [4, 5, 6], - ('file2', 'GPU_ID'): [1, 2, 2], - ('file2', 'Grid_Size'): [512, 1024, 2048], - ('file2', 'Workgroup_Size'): [32, 64, 128], - ('file2', 'LDS_Per_Workgroup'): [2048, 4096, 8192], - ('file2', 'Scratch_Per_Workitem'): [0, 0, 0], - ('file2', 'Arch_VGPR'): [64, 96, 128], - ('file2', 'Accum_VGPR'): [0, 0, 0], - ('file2', 'SGPR'): [32, 48, 64], - ('file2', 'Wave_Size'): [64, 64, 64], - ('file2', 'Correlation_ID'): [2001, 2002, 2003], - ('file2', 'Kernel_ID'): [601, 602, 603], - ('file2', 'Kernel_Name'): ['kernel_c', 'kernel_c', 'kernel_d'], - ('file2', 'Start_Timestamp'): [3000, 3100, 4000], - ('file2', 'End_Timestamp'): [3400, 3500, 4800], - ('file2', 'Counter1'): [400, 500, 600], + ("file1", "Dispatch_ID"): [1, 2, 3], + ("file1", "GPU_ID"): [0, 0, 1], + ("file1", "Grid_Size"): [64, 128, 256], + ("file1", "Workgroup_Size"): [16, 32, 64], + ("file1", "LDS_Per_Workgroup"): [1024, 2048, 4096], + ("file1", "Scratch_Per_Workitem"): [0, 0, 0], + ("file1", "Arch_VGPR"): [32, 64, 96], + ("file1", "Accum_VGPR"): [0, 0, 0], + ("file1", "SGPR"): [16, 32, 48], + ("file1", "Wave_Size"): [64, 64, 64], + ("file1", "Correlation_ID"): [1001, 1002, 1003], + ("file1", "Kernel_ID"): [501, 502, 503], + ("file1", "Kernel_Name"): ["kernel_a", "kernel_a", "kernel_b"], + ("file1", "Start_Timestamp"): [1000, 1100, 2000], + ("file1", "End_Timestamp"): [1200, 1300, 2500], + ("file1", "Counter1"): [100, 200, 300], + ("file2", "Dispatch_ID"): [4, 5, 6], + ("file2", "GPU_ID"): [1, 2, 2], + ("file2", "Grid_Size"): [512, 1024, 2048], + ("file2", "Workgroup_Size"): [32, 64, 128], + ("file2", "LDS_Per_Workgroup"): [2048, 4096, 8192], + ("file2", "Scratch_Per_Workitem"): [0, 0, 0], + ("file2", "Arch_VGPR"): [64, 96, 128], + ("file2", "Accum_VGPR"): [0, 0, 0], + ("file2", "SGPR"): [32, 48, 64], + ("file2", "Wave_Size"): [64, 64, 64], + ("file2", "Correlation_ID"): [2001, 2002, 2003], + ("file2", "Kernel_ID"): [601, 602, 603], + ("file2", "Kernel_Name"): ["kernel_c", "kernel_c", "kernel_d"], + ("file2", "Start_Timestamp"): [3000, 3100, 4000], + ("file2", "End_Timestamp"): [3400, 3500, 4800], + ("file2", "Counter1"): [400, 500, 600], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + result = utils_mod.merge_counters_spatial_multiplex(df) - + assert isinstance(result, pd.DataFrame) assert isinstance(result.columns, pd.MultiIndex) assert len(result.columns.levels) == 2 @@ -7254,81 +8127,86 @@ def test_merge_counters_spatial_multiplex_basic_functionality(): def test_merge_counters_spatial_multiplex_kernel_name_fallback(): """ Test merge_counters_spatial_multiplex when Kernel_Name is missing but Name exists. - + Returns: None: Asserts function uses Name column when Kernel_Name is not available. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'Dispatch_ID'): [1, 2], - ('file1', 'GPU_ID'): [0, 0], - ('file1', 'Grid_Size'): [64, 128], - ('file1', 'Workgroup_Size'): [16, 32], - ('file1', 'LDS_Per_Workgroup'): [1024, 2048], - ('file1', 'Scratch_Per_Workitem'): [0, 0], - ('file1', 'Arch_VGPR'): [32, 64], - ('file1', 'Accum_VGPR'): [0, 0], - ('file1', 'SGPR'): [16, 32], - ('file1', 'Wave_Size'): [64, 64], - ('file1', 'Correlation_ID'): [1001, 1002], - ('file1', 'Kernel_ID'): [501, 502], - ('file1', 'Name'): ['kernel_a', 'kernel_a'], - ('file1', 'Start_Timestamp'): [1000, 1100], - ('file1', 'End_Timestamp'): [1200, 1300], - ('file1', 'Counter1'): [100, 200], + ("file1", "Dispatch_ID"): [1, 2], + ("file1", "GPU_ID"): [0, 0], + ("file1", "Grid_Size"): [64, 128], + ("file1", "Workgroup_Size"): [16, 32], + ("file1", "LDS_Per_Workgroup"): [1024, 2048], + ("file1", "Scratch_Per_Workitem"): [0, 0], + ("file1", "Arch_VGPR"): [32, 64], + ("file1", "Accum_VGPR"): [0, 0], + ("file1", "SGPR"): [16, 32], + ("file1", "Wave_Size"): [64, 64], + ("file1", "Correlation_ID"): [1001, 1002], + ("file1", "Kernel_ID"): [501, 502], + ("file1", "Name"): ["kernel_a", "kernel_a"], + ("file1", "Start_Timestamp"): [1000, 1100], + ("file1", "End_Timestamp"): [1200, 1300], + ("file1", "Counter1"): [100, 200], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - - # The function currently has a bug where it doesn't properly check for 'Kernel_Name' + + # The function currently has a bug where it doesn't properly check for 'Kernel_Name' # existence before accessing it, even though it has fallback logic for 'Name' try: result = utils_mod.merge_counters_spatial_multiplex(df) - + assert isinstance(result, pd.DataFrame) assert len(result) > 0 - + except KeyError as e: if "'Kernel_Name'" in str(e): - pytest.skip("Function doesn't properly check for Kernel_Name existence before accessing - needs to validate column presence in the check condition") + pytest.skip( + "Function doesn't properly check for Kernel_Name existence before accessing - needs to validate column presence in the check condition" + ) else: raise - + + def test_merge_counters_spatial_multiplex_single_kernel_occurrence(): """ Test merge_counters_spatial_multiplex with kernels that appear only once. - + Returns: None: Asserts function handles single kernel occurrences correctly. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'Dispatch_ID'): [1, 2, 3], - ('file1', 'GPU_ID'): [0, 1, 2], - ('file1', 'Grid_Size'): [64, 128, 256], - ('file1', 'Workgroup_Size'): [16, 32, 64], - ('file1', 'LDS_Per_Workgroup'): [1024, 2048, 4096], - ('file1', 'Scratch_Per_Workitem'): [0, 0, 0], - ('file1', 'Arch_VGPR'): [32, 64, 96], - ('file1', 'Accum_VGPR'): [0, 0, 0], - ('file1', 'SGPR'): [16, 32, 48], - ('file1', 'Wave_Size'): [64, 64, 64], - ('file1', 'Correlation_ID'): [1001, 1002, 1003], - ('file1', 'Kernel_ID'): [501, 502, 503], - ('file1', 'Kernel_Name'): ['kernel_a', 'kernel_b', 'kernel_c'], - ('file1', 'Start_Timestamp'): [1000, 2000, 3000], - ('file1', 'End_Timestamp'): [1200, 2500, 3800], - ('file1', 'Counter1'): [100, 200, 300], + ("file1", "Dispatch_ID"): [1, 2, 3], + ("file1", "GPU_ID"): [0, 1, 2], + ("file1", "Grid_Size"): [64, 128, 256], + ("file1", "Workgroup_Size"): [16, 32, 64], + ("file1", "LDS_Per_Workgroup"): [1024, 2048, 4096], + ("file1", "Scratch_Per_Workitem"): [0, 0, 0], + ("file1", "Arch_VGPR"): [32, 64, 96], + ("file1", "Accum_VGPR"): [0, 0, 0], + ("file1", "SGPR"): [16, 32, 48], + ("file1", "Wave_Size"): [64, 64, 64], + ("file1", "Correlation_ID"): [1001, 1002, 1003], + ("file1", "Kernel_ID"): [501, 502, 503], + ("file1", "Kernel_Name"): ["kernel_a", "kernel_b", "kernel_c"], + ("file1", "Start_Timestamp"): [1000, 2000, 3000], + ("file1", "End_Timestamp"): [1200, 2500, 3800], + ("file1", "Counter1"): [100, 200, 300], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + result = utils_mod.merge_counters_spatial_multiplex(df) - + assert isinstance(result, pd.DataFrame) assert len(result) == 3 @@ -7336,36 +8214,44 @@ def test_merge_counters_spatial_multiplex_single_kernel_occurrence(): def test_merge_counters_spatial_multiplex_multiple_duplicate_kernels(): """ Test merge_counters_spatial_multiplex with multiple kernels having duplicates. - + Returns: None: Asserts function correctly handles multiple kernel duplicates. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'Dispatch_ID'): [1, 2, 3, 4, 5, 6], - ('file1', 'GPU_ID'): [0, 0, 1, 1, 2, 2], - ('file1', 'Grid_Size'): [64, 64, 128, 128, 256, 256], - ('file1', 'Workgroup_Size'): [16, 16, 32, 32, 64, 64], - ('file1', 'LDS_Per_Workgroup'): [1024, 1024, 2048, 2048, 4096, 4096], - ('file1', 'Scratch_Per_Workitem'): [0, 0, 0, 0, 0, 0], - ('file1', 'Arch_VGPR'): [32, 32, 64, 64, 96, 96], - ('file1', 'Accum_VGPR'): [0, 0, 0, 0, 0, 0], - ('file1', 'SGPR'): [16, 16, 32, 32, 48, 48], - ('file1', 'Wave_Size'): [64, 64, 64, 64, 64, 64], - ('file1', 'Correlation_ID'): [1001, 1002, 1003, 1004, 1005, 1006], - ('file1', 'Kernel_ID'): [501, 502, 503, 504, 505, 506], - ('file1', 'Kernel_Name'): ['kernel_a', 'kernel_a', 'kernel_b', 'kernel_b', 'kernel_c', 'kernel_c'], - ('file1', 'Start_Timestamp'): [1000, 1100, 2000, 2100, 3000, 3100], - ('file1', 'End_Timestamp'): [1200, 1300, 2500, 2600, 3800, 3900], - ('file1', 'Counter1'): [100, 200, 300, 400, 500, 600], + ("file1", "Dispatch_ID"): [1, 2, 3, 4, 5, 6], + ("file1", "GPU_ID"): [0, 0, 1, 1, 2, 2], + ("file1", "Grid_Size"): [64, 64, 128, 128, 256, 256], + ("file1", "Workgroup_Size"): [16, 16, 32, 32, 64, 64], + ("file1", "LDS_Per_Workgroup"): [1024, 1024, 2048, 2048, 4096, 4096], + ("file1", "Scratch_Per_Workitem"): [0, 0, 0, 0, 0, 0], + ("file1", "Arch_VGPR"): [32, 32, 64, 64, 96, 96], + ("file1", "Accum_VGPR"): [0, 0, 0, 0, 0, 0], + ("file1", "SGPR"): [16, 16, 32, 32, 48, 48], + ("file1", "Wave_Size"): [64, 64, 64, 64, 64, 64], + ("file1", "Correlation_ID"): [1001, 1002, 1003, 1004, 1005, 1006], + ("file1", "Kernel_ID"): [501, 502, 503, 504, 505, 506], + ("file1", "Kernel_Name"): [ + "kernel_a", + "kernel_a", + "kernel_b", + "kernel_b", + "kernel_c", + "kernel_c", + ], + ("file1", "Start_Timestamp"): [1000, 1100, 2000, 2100, 3000, 3100], + ("file1", "End_Timestamp"): [1200, 1300, 2500, 2600, 3800, 3900], + ("file1", "Counter1"): [100, 200, 300, 400, 500, 600], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + result = utils_mod.merge_counters_spatial_multiplex(df) - + assert isinstance(result, pd.DataFrame) assert len(result) == 3 @@ -7373,46 +8259,49 @@ def test_merge_counters_spatial_multiplex_multiple_duplicate_kernels(): def test_merge_counters_spatial_multiplex_timestamp_median_calculation(): """ Test merge_counters_spatial_multiplex timestamp median calculations. - + Returns: None: Asserts function correctly calculates median timestamps. """ - import utils.utils as utils_mod import pandas as pd - + + import utils.utils as utils_mod + data = { - ('file1', 'Dispatch_ID'): [1, 2, 3], - ('file1', 'GPU_ID'): [0, 0, 0], - ('file1', 'Grid_Size'): [64, 64, 64], - ('file1', 'Workgroup_Size'): [16, 16, 16], - ('file1', 'LDS_Per_Workgroup'): [1024, 1024, 1024], - ('file1', 'Scratch_Per_Workitem'): [0, 0, 0], - ('file1', 'Arch_VGPR'): [32, 32, 32], - ('file1', 'Accum_VGPR'): [0, 0, 0], - ('file1', 'SGPR'): [16, 16, 16], - ('file1', 'Wave_Size'): [64, 64, 64], - ('file1', 'Correlation_ID'): [1001, 1002, 1003], - ('file1', 'Kernel_ID'): [501, 502, 503], - ('file1', 'Kernel_Name'): ['kernel_a', 'kernel_a', 'kernel_a'], - ('file1', 'Start_Timestamp'): [1000, 1200, 1400], - ('file1', 'End_Timestamp'): [1500, 1700, 1900], - ('file1', 'Counter1'): [100, 200, 300], + ("file1", "Dispatch_ID"): [1, 2, 3], + ("file1", "GPU_ID"): [0, 0, 0], + ("file1", "Grid_Size"): [64, 64, 64], + ("file1", "Workgroup_Size"): [16, 16, 16], + ("file1", "LDS_Per_Workgroup"): [1024, 1024, 1024], + ("file1", "Scratch_Per_Workitem"): [0, 0, 0], + ("file1", "Arch_VGPR"): [32, 32, 32], + ("file1", "Accum_VGPR"): [0, 0, 0], + ("file1", "SGPR"): [16, 16, 16], + ("file1", "Wave_Size"): [64, 64, 64], + ("file1", "Correlation_ID"): [1001, 1002, 1003], + ("file1", "Kernel_ID"): [501, 502, 503], + ("file1", "Kernel_Name"): ["kernel_a", "kernel_a", "kernel_a"], + ("file1", "Start_Timestamp"): [1000, 1200, 1400], + ("file1", "End_Timestamp"): [1500, 1700, 1900], + ("file1", "Counter1"): [100, 200, 300], } df = pd.DataFrame(data) df.columns = pd.MultiIndex.from_tuples(df.columns) - + result = utils_mod.merge_counters_spatial_multiplex(df) - + assert isinstance(result, pd.DataFrame) assert len(result) == 1 - + + # ============================================================================= # Tests for convert_metric_id_to_panel_idx function # ============================================================================ + def test_convert_metric_id_to_panel_idx_zero_values(): """Test convert_metric_id_to_panel_idx with zero values in different positions. - + Args: None Returns: @@ -7426,7 +8315,7 @@ def test_convert_metric_id_to_panel_idx_zero_values(): def test_convert_metric_id_to_panel_idx_leading_zeros(): """Test convert_metric_id_to_panel_idx with leading zeros in metric IDs. - + Args: None Returns: @@ -7439,7 +8328,7 @@ def test_convert_metric_id_to_panel_idx_leading_zeros(): def test_convert_metric_id_to_panel_idx_invalid_empty_string(): """Test convert_metric_id_to_panel_idx with empty string raises exception. - + Args: None Returns: @@ -7451,7 +8340,7 @@ def test_convert_metric_id_to_panel_idx_invalid_empty_string(): def test_convert_metric_id_to_panel_idx_invalid_too_many_parts(): """Test convert_metric_id_to_panel_idx with more than two parts raises exception. - + Args: None Returns: @@ -7469,7 +8358,7 @@ def test_convert_metric_id_to_panel_idx_invalid_too_many_parts(): def test_convert_metric_id_to_panel_idx_invalid_non_numeric(): """Test convert_metric_id_to_panel_idx with non-numeric values raises exception. - + Args: None Returns: @@ -7487,9 +8376,10 @@ def test_convert_metric_id_to_panel_idx_invalid_non_numeric(): with pytest.raises(ValueError): utils.convert_metric_id_to_panel_idx("4.02abc") + def test_convert_metric_id_to_panel_idx_invalid_floating_point(): """Test convert_metric_id_to_panel_idx with floating point numbers in unexpected format. - + Args: None Returns: @@ -7504,7 +8394,7 @@ def test_convert_metric_id_to_panel_idx_invalid_floating_point(): def test_convert_metric_id_to_panel_idx_edge_case_whitespace(): """Test convert_metric_id_to_panel_idx with whitespace in metric IDs. - + Args: None Returns: @@ -7517,9 +8407,10 @@ def test_convert_metric_id_to_panel_idx_edge_case_whitespace(): assert utils.convert_metric_id_to_panel_idx("4. 02") == 402 assert utils.convert_metric_id_to_panel_idx(" 4 . 02 ") == 402 + def test_convert_metric_id_to_panel_idx_edge_case_dot_only(): """Test convert_metric_id_to_panel_idx with only dot character raises exception. - + Args: None Returns: @@ -7535,4 +8426,4 @@ def test_convert_metric_id_to_panel_idx_edge_case_dot_only(): utils.convert_metric_id_to_panel_idx("4.") with pytest.raises(ValueError): - utils.convert_metric_id_to_panel_idx(".02") \ No newline at end of file + utils.convert_metric_id_to_panel_idx(".02")