From dbb361c60633f160a783cfd831e11573b5f308dd Mon Sep 17 00:00:00 2001 From: vedithal-amd Date: Mon, 3 Nov 2025 09:23:22 -0500 Subject: [PATCH] [rocprofiler-compute] fix parser to prevent missing metrics in analysis mode (#1613) * fix parser * fix parser * fix parser --------- Co-authored-by: fei.zheng Co-authored-by: ywang103-amd --- .../rocprofiler-compute/src/utils/parser.py | 76 +++++++++---------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/projects/rocprofiler-compute/src/utils/parser.py b/projects/rocprofiler-compute/src/utils/parser.py index a5b27ffd6a..e8093071f7 100755 --- a/projects/rocprofiler-compute/src/utils/parser.py +++ b/projects/rocprofiler-compute/src/utils/parser.py @@ -315,33 +315,13 @@ class MetricEvaluator: self.raw_pmc_df = raw_pmc_df self.sys_vars = sys_vars self.empirical_peaks = empirical_peaks - self._prepare_df_cache() - - def _prepare_df_cache(self) -> None: - """Prepare cached dataframe access for performance.""" - if isinstance(self.raw_pmc_df, dict): - self.df_cache = { - f"raw_pmc_df_{key}": self.raw_pmc_df[key] - for key in self.raw_pmc_df.keys() - } - elif isinstance(self.raw_pmc_df, pd.DataFrame): - raw_pmc_df_keys = set(self.raw_pmc_df.columns.get_level_values(0)) - self.df_cache = { - f"raw_pmc_df_{key}": self.raw_pmc_df[key] for key in raw_pmc_df_keys - } - else: - raise ValueError(f'Unknown `raw_pmc_df` type: "{type(self.raw_pmc_df)}".') def eval_expression(self, expr: str) -> Union[str, float, int]: """Evaluate a single expression with proper local context.""" try: - # Optimize dataframe access by replacing dict notation with dir_path - # variable access - opt_expr = re.sub(r"raw_pmc_df\['(.*?)'\]", r"raw_pmc_df_\1", expr) - # Create comprehensive local context local_expr_context = {} - local_expr_context.update(self.df_cache) + local_expr_context.update({"raw_pmc_df": self.raw_pmc_df}) local_expr_context.update(self.sys_vars) local_expr_context.update(self.empirical_peaks) @@ -361,12 +341,12 @@ class MetricEvaluator: }) eval_result = eval( - compile(opt_expr, "", "eval"), + compile(expr, "", "eval"), {}, local_expr_context, ) - if np.isnan(eval_result): + if np.isnan(eval_result).any(): return "" else: return eval_result @@ -378,10 +358,14 @@ class MetricEvaluator: ) return "" else: + console_warning(f"Failed to evaluate expression '{expr}': {exception}.") return "" except AttributeError as attribute_error: if str(attribute_error) == "'NoneType' object has no attribute 'get'": + console_warning( + f"Failed to evaluate expression '{expr}': {attribute_error}." + ) return "" else: console_error("analysis", str(attribute_error)) @@ -477,8 +461,17 @@ def build_eval_string(equation: str, coll_level: str, config: dict) -> str: equation_string, ) else: + # Use pmc_perf.csv for all counters equation_string = re.sub( - r"raw_pmc_df", f"raw_pmc_df['{coll_level}']", equation_string + r"raw_pmc_df", + f"raw_pmc_df['{schema.PMC_PERF_FILE_PREFIX}']", + equation_string, + ) + # Use coll_level csv for SQ_ACCUM_PREV_HIRES counter only + equation_string = re.sub( + rf"raw_pmc_df['{schema.PMC_PERF_FILE_PREFIX}']['SQ_ACCUM_PREV_HIRES']", + f"raw_pmc_df['{coll_level}']['SQ_ACCUM_PREV_HIRES']", + equation_string, ) return equation_string @@ -911,7 +904,9 @@ def create_sys_vars(sys_info: pd.Series) -> dict[str, Union[int, float]]: def calc_builtin_vars( - raw_pmc_df: Union[pd.DataFrame, dict], config: dict + raw_pmc_df: Union[pd.DataFrame, dict], + config: dict, + sys_vars: dict[str, Union[int, float]], ) -> dict[str, Optional[Union[str, float, int]]]: """Calculate built-in variables""" # TODO: fix all $normUnit in Unit column or title @@ -929,7 +924,8 @@ def calc_builtin_vars( ) try: # Create temporary evaluator for this calculation - temporary_evaluator = MetricEvaluator(raw_pmc_df, {}, {}) + # Pass sys_vars so that $num_xcd and other system variables are available + temporary_evaluator = MetricEvaluator(raw_pmc_df, sys_vars, {}) calculation_result = temporary_evaluator.eval_expression(eval_string) builtin_vars_collection[f"ammolite__{variable_key}"] = calculation_result except (TypeError, NameError, KeyError, AttributeError): @@ -944,9 +940,9 @@ def calc_builtin_vars( variable_value, schema.PMC_PERF_FILE_PREFIX, config ) try: - temporary_evaluator = MetricEvaluator( - raw_pmc_df, builtin_vars_collection, {} - ) + # Merge sys_vars with builtin_vars_collection for second pass + combined_vars = {**sys_vars, **builtin_vars_collection} + temporary_evaluator = MetricEvaluator(raw_pmc_df, combined_vars, {}) calculation_result = temporary_evaluator.eval_expression(eval_string) builtin_vars_collection[f"ammolite__{variable_key}"] = calculation_result except (TypeError, NameError, KeyError, AttributeError): @@ -981,7 +977,7 @@ def eval_metric( sys_vars = create_sys_vars(sys_info) empirical_peaks = create_empirical_peaks_dict(empirical_peaks_df) - builtin_vars = calc_builtin_vars(raw_pmc_df, config) + builtin_vars = calc_builtin_vars(raw_pmc_df, config, sys_vars) sys_vars.update(builtin_vars) # Create metric evaluator @@ -1323,28 +1319,28 @@ def search_pc_sampling_record( console_warning("PC sampling: no pc sampling record found!") return None - # Prepare sorted output list + # Convert to sorted list of tuples: + # (code_object_id, inst_index, code_object_offset, count, count_issued, + # count_stalled, stall_reason) sorted_counts = sorted( [ ( code_object_id, - code_object_offset, - inst_index, - info[0], # total_count + info[3], # inst_index + offset, + info[0], # count info[1], # count_issued info[2], # count_stalled + # For info[4] (stall_reason dict), remove the zero entries, + # sorting the remaining items by their values in descending order sorted( - ((k, v) for k, v in info[3].items() if v > 0), + ((k, v) for k, v in info[4].items() if v > 0), key=lambda item: item[1], reverse=True, ), # sorted stall reasons sorted(info[4]), # sorted dispatch_ids list ) - for ( - code_object_id, - code_object_offset, - inst_index, - ), info in grouped_data.items() + for (code_object_id, offset), info in grouped_data.items() ], key=lambda x: (x[0], x[1], x[2]), )