Names shortened/demangled after join_prof

-  Added kernelVerbose flag in profile_group
-  Added KernelVerbose flag in analyze_group
-  Analyze replaces csv with shortened/demangled name
-  csv_converter uses llvm-cxxfilt

Signed-off-by: josantos <josantos@amd.com>


[ROCm/rocprofiler-compute commit: 59d77f9d81]

projects/rocprofiler-compute/src/omniperf

@@ -446,6 +446,21 @@ def characterize_app(args, VER):
     # Manually join each pmc_perf*.csv output
     if args.use_rocscope == False:
         join_prof(workload_dir, args.join_type, log, args.verbose)
+        #demangle
+        for filename in os.listdir(workload_dir):
+                try:
+                    # fileName = file[0 : file.find(".")]
+                    # Only shorten KernelNames if instructed to
+                    if args.kernelVerbose < 5:
+                        t1 = pd.read_csv(
+                            os.path.join(workload_dir, filename),
+                            on_bad_lines="skip",
+                            engine="python",
+                        )
+                        t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose)
+                        t2.to_csv(fname, index=False)
+                except pd.errors.EmptyDataError:
+                    print("Skipping empty csv " + filename)
 
     # Close log
     log.close()
@@ -660,6 +675,7 @@ def omniperf_profile(args, VER):
                 run_rocscope(args, fname)
             else:
                 run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose)
+                
 
         # Update timestamps
         replace_timestamps(workload_dir, log)
@@ -667,6 +683,22 @@ def omniperf_profile(args, VER):
         # Manually join each pmc_perf*.csv output
         if args.use_rocscope == False:
             join_prof(workload_dir, args.join_type, log, args.verbose)
+            #demangle
+            for filename in os.listdir(workload_dir):
+                if filename.endswith('.csv'):
+                    try:
+                        # fileName = file[0 : file.find(".")]
+                        # Only shorten KernelNames if instructed to
+                        if args.kernelVerbose < 5:
+                            t1 = pd.read_csv(
+                                os.path.join(workload_dir, filename),
+                                on_bad_lines="skip",
+                                engine="python",
+                            )
+                            t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose)
+                            t2.to_csv(os.path.join(workload_dir, filename), index=False)
+                    except pd.errors.EmptyDataError:
+                        print("Skipping empty csv " + filename)
 
     # Generate sysinfo
     gen_sysinfo(args.name, workload_dir, args.ipblocks, args.remaining, args.no_roof)
@@ -739,7 +771,7 @@ def main():
     # PROFILE MODE
     ##############
     if args.mode == "profile":
-        Extractionlvl = 3 #args.extraction_level
+        Extractionlvl = args.kernelVerbose
         print("Resolving rocprof")
         resolve_rocprof()
         # Cannot access parent directories
@@ -777,43 +809,28 @@ def main():
             roof_setup(args, my_parser, VER)
             # Generate roofline
             roofline_only(args.path, args.device, args.sort, args.mem_level, args.kernel_names, args.verbose)
-            #demangle
-            for file in os.listdir(args.path):
-                if file.endswith(".csv"):
-                    try:
-                        fileName = file[0 : file.find(".")]
-                        # Only shorten KernelNames if instructed to
-                        if Extractionlvl < 5:
-                            t1 = pd.read_csv(
-                                os.listdir(args.path) + "/" + file,
-                                on_bad_lines="skip",
-                                engine="python",
-                            )
-
-                            t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl)
-                    except pd.errors.EmptyDataError:
-                        print("Skipping empty csv " + file)
 
         # Profile only
         else:
             print("\n-------------\nProfile only\n-------------\n")
             omniperf_profile(args, VER)
+            workload_dir = args.path
             #demangle
-            for file in os.listdir(args.path):
-                if file.endswith(".csv"):
+            for filename in os.listdir(workload_dir):
+                if filename.endswith('.csv'):
                     try:
-                        fileName = file[0 : file.find(".")]
+                        # fileName = file[0 : file.find(".")]
                         # Only shorten KernelNames if instructed to
-                        if Extractionlvl < 5:
+                        if args.kernelVerbose < 5:
                             t1 = pd.read_csv(
-                                os.listdir(args.path) + "/" + file,
+                                os.path.join(workload_dir, filename),
                                 on_bad_lines="skip",
                                 engine="python",
                             )
-
-                            t2 = csv_converter.kernel_name_shortener(t1, cache, level=Extractionlvl)
+                            t2 = csv_converter.kernel_name_shortener(t1, cache, level=args.kernelVerbose)
+                            t2.to_csv(os.path.join(workload_dir, filename), index=False)
                     except pd.errors.EmptyDataError:
-                        print("Skipping empty csv " + file)
+                        print("Skipping empty csv " + filename)
 
     ##############
     # DATABASE MODE

projects/rocprofiler-compute/src/omniperf_analyze/omniperf_analyze.py

@@ -45,6 +45,8 @@ import os.path
 from pathlib import Path
 from omniperf_analyze.utils import parser, file_io
 from omniperf_analyze.utils.gui_components.roofline import get_roofline
+from utils import csv_converter
+import pandas as pd
 
 archConfigs = {}
 
@@ -220,7 +222,15 @@ def run_cli(args, runs):
     # If we assume the panel layout for all archs are similar, it doesn't matter
     # which archConfig passed into show_all function.
     # After decide to how to manage kernels display patterns, we can revisit it.
+    cache =dict()
     for d in args.path:
+        #demangle
+        for filename in os.listdir(d[0]):
+            if filename.endswith('.csv'):
+                df = pd.read_csv(os.path.join(d[0],filename))
+                new_df = csv_converter.kernel_name_shortener(df, cache, args.kernelVerbose)
+                new_df.to_csv(os.path.join(d[0],filename), index=False)
+            
         file_io.create_df_kernel_top_stats(
             d[0],
             runs[d[0]].filter_gpu_ids,

projects/rocprofiler-compute/src/parser.py

@@ -204,6 +204,15 @@ def parse(my_parser):
         nargs=argparse.REMAINDER,
         help="\t\t\tProvide command for profiling after double dash.",
     )
+    profile_group.add_argument(
+        "-f",
+        "--kernelVerbose",
+        required=False,
+        metavar="",
+        help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)",
+        default=2,
+        type=int,
+    )
 
     ## Roofline Command Line Options
     roofline_group.add_argument(
@@ -514,3 +523,12 @@ def parse(my_parser):
         action="store_true",
         help="\t\tRandomly generate a port to launch GUI application.\n\t\tRegistered Ports range inclusive (1024-49151).",
     )
+    analyze_group.add_argument(
+        "-f",
+        "--kernelVerbose",
+        required=False,
+        metavar="",
+        help="\t\t\t\tSpecify Kernel Name verbose level 1-5. Lower the level, shorter the kernel name. (DEFAULT: 2) (DISABLE: 5)",
+        default=2,
+        type=int,
+    )

projects/rocprofiler-compute/src/utils/csv_converter.py

@@ -25,6 +25,7 @@
 import argparse
 import collections
 import os
+import subprocess
 import sys
 import re
 import pandas as pd
@@ -33,11 +34,11 @@ from pymongo import MongoClient
 from tqdm import tqdm
 import shutil
 
+
 cache = dict()
 supported_arch = {"gfx906": "mi50", "gfx908": "mi100", "gfx90a": "mi200"}
 MAX_SERVER_SEL_DELAY = 5000  # 5 sec connection timeout
 
-
 def kernel_name_shortener(df, cache, level):
     if level >= 5:
         return df
@@ -55,6 +56,13 @@ def kernel_name_shortener(df, cache, level):
             if original_name in cache:
                 continue
 
+            cmd = ["llvm-cxxfilt", original_name]
+
+            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+            demangled_name, e = proc.communicate()
+            demangled_name = str(demangled_name, "UTF-8").strip()
+
             # cache miss, add the shortened name to the dictionary
             new_name = ""
             matches = ""
@@ -62,15 +70,14 @@ def kernel_name_shortener(df, cache, level):
             names_and_args = re.compile(r"(?P<name>[( )A-Za-z0-9_]+)([ ,*<>()]+)(::)?")
 
             # works for name Kokkos::namespace::init_lock_array_kernel_threadid(int) [clone .kd]
-            if names_and_args.search(original_name):
-                matches = names_and_args.findall(original_name)
+            if names_and_args.search(demangled_name):
+                matches = names_and_args.findall(demangled_name)
             else:
                 # Works for first case  '__amd_rocclr_fillBuffer.kd'
-                # remove .kd and then parse through original regex
-                first_case = re.compile(r"([^\s]+)(.kd)")
-                Mod_name_and_args = re.compile(r"(?P<name>[( )A-Za-z0-9_]+)([ ,*<>()]*)")
-                interim_name = first_case.search(original_name).group(1)
-                matches = Mod_name_and_args.findall(interim_name)
+                cache[original_name] = new_name
+                if new_name == None or new_name == "":
+                    cache[original_name] = demangled_name
+                continue
 
             current_level = 0
             for name in matches:
@@ -103,13 +110,12 @@ def kernel_name_shortener(df, cache, level):
 
             cache[original_name] = new_name
             if new_name == None or new_name == "":
-                cache[original_name] = original_name
+                cache[original_name] = demangled_name
 
         df[columnName] = df[columnName].map(cache)
 
     return df
 
-
 # Verify target directory and setup connection
 def parse(args, profileAndExport):
     host = args.host