rocm-systems/src/omniperf_profile/profiler_base.py

##############################################################################bl
# MIT License
#
# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All Rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
##############################################################################el

from abc import ABC, abstractmethod
import logging
import glob
import sys
import os
import re
from utils.utils import capture_subprocess_output, run_prof, gen_sysinfo, run_rocscope, error, demarcate
import config
import pandas as pd

class OmniProfiler_Base():
    def __init__(self, args, profiler_mode, soc):
        self.__args = args
        self.__profiler = profiler_mode
        self._soc = soc # OmniSoC obj
        self.__perfmon_dir = os.path.join(str(config.omniperf_home), "omniperf_soc", "profile_configs")

    def get_args(self):
        return self.__args
    def get_profiler_options(self, fname):
        """Fetch any version specific arguments required by profiler
        """
        # assume no SoC specific options and return empty list by default
        return []
    
    @demarcate
    def pmc_perf_split(self):
        """Avoid default rocprof join utility by spliting each line into a separate input file
        """
        workload_perfmon_dir = os.path.join(self.__args.path, "perfmon")
        lines = open(os.path.join(workload_perfmon_dir, "pmc_perf.txt"), "r").read().splitlines()

        # Iterate over each line in pmc_perf.txt
        mpattern = r"^pmc:(.*)"
        i = 0
        for line in lines:
            # Verify no comments
            stext = line.split("#")[0].strip()
            if not stext:
                continue

            # all pmc counters start with  "pmc:"
            m = re.match(mpattern, stext)
            if m is None:
                continue

            # Create separate file for each line
            fd = open(workload_perfmon_dir + "/pmc_perf_" + str(i) + ".txt", "w")
            fd.write(stext + "\n\n")
            fd.write("gpu:\n")
            fd.write("range:\n")
            fd.write("kernel:\n")
            fd.close()

            i += 1

        # Remove old pmc_perf.txt input from perfmon dir
        os.remove(workload_perfmon_dir + "/pmc_perf.txt")

    # joins disparate runs less dumbly than rocprof
    @demarcate
    def join_prof(self, out=None):
        """Manually join separated rocprof runs
        """
        # Set default output directory if not specified
        if type(self.__args.path) == str:
            if out is None:
                out = self.__args.path + "/pmc_perf.csv"
            files = glob.glob(self.__args.path + "/" + "pmc_perf_*.csv")
        elif type(self.__args.path) == list:
            files = self.__args.path
        else:
            logging.error("ERROR: Invalid workload_dir")
            sys.exit(1)

        df = None
        for i, file in enumerate(files):
            _df = pd.read_csv(file) if type(self.__args.path) == str else file
            if self.__args.join_type == "kernel":
                key = _df.groupby("Kernel_Name").cumcount()
                _df["key"] = _df.Kernel_Name + " - " + key.astype(str)
            elif self.__args.join_type == "grid":
                key = _df.groupby(["Kernel_Name", "Grid_Size"]).cumcount()
                _df["key"] = (
                    _df["Kernel_Name"] + " - " + _df["Grid_Size"].astype(str) + " - " + key.astype(str)
                )
            else:
                print("ERROR: Unrecognized --join-type")
                sys.exit(1)

            if df is None:
                df = _df
            else:
                # join by unique index of kernel
                df = pd.merge(df, _df, how="inner", on="key", suffixes=("", f"_{i}"))

        # TODO: check for any mismatch in joins
        duplicate_cols = {
            "GPU_ID": [col for col in df.columns if "GPU_ID" in col],
            "Grid_Size": [col for col in df.columns if "Grid_Size" in col],
            "Workgroup_Size": [col for col in df.columns if "Workgroup_Size" in col],
            "LDS_Per_Workgroup": [col for col in df.columns if "LDS_Per_Workgroup" in col],
            "Scratch_Per_Workitem": [col for col in df.columns if "Scratch_Per_Workitem" in col],
            "SGPR": [col for col in df.columns if "SGPR" in col],
        }
        # Check for vgpr counter in ROCm < 5.3
        if "vgpr" in df.columns:
            duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col]
        # Check for vgpr counter in ROCm >= 5.3
        else:
            duplicate_cols["Arch_VGPR"] = [col for col in df.columns if "Arch_VGPR" in col]
            duplicate_cols["Accum_VGPR"] = [col for col in df.columns if "Accum_VGPR" in col]
        for key, cols in duplicate_cols.items():
            _df = df[cols]
            if not test_df_column_equality(_df):
                msg = (
                    "WARNING: Detected differing {} values while joining pmc_perf.csv".format(
                        key
                    )
                )
                logging.warning(msg + "\n")
            else:
                msg = "Successfully joined {} in pmc_perf.csv".format(key)
                logging.debug(msg + "\n")
            if test_df_column_equality(_df) and self.__args.verbose:
                logging.info(msg)

        # now, we can:
        #   A) throw away any of the "boring" duplicats
        df = df[
            [
                k
                for k in df.keys()
                if not any(
                    check in k
                    for check in [
                       # rocprofv2 headers
                        "GPU_ID_",
                        "Grid_Size_",
                        "Workgroup_Size_",
                        "LDS_Per_Workgroup_",
                        "Scratch_Per_Workitem_",
                        "vgpr_",
                        "Arch_VGPR_",
                        "Accum_VGPR_",
                        "SGPR_",
                        "Dispatch_ID_",
                        "Queue_ID",
                        "Queue_Index",
                        "PID",
                        "TID",
                        "SIG",
                        "OBJ",
                        # rocscope specific merged counters, keep original
                        "dispatch_",
                        # extras
                        "sig",
                        "queue-id",
                        "queue-index",
                        "pid",
                        "tid",
                        "fbar",
                    ]
                )
            ]
        ]
        #   B) any timestamps that are _not_ the duration, which is the one we care about
        df = df[
            [
                k
                for k in df.keys()
                if not any(
                    check in k
                    for check in [
                        "DispatchNs",
                        "CompleteNs",
                        # rocscope specific timestamp
                        "HostDuration",
                    ]
                )
            ]
        ]
        #   C) sanity check the name and key
        namekeys = [k for k in df.keys() if "Kernel_Name" in k]
        assert len(namekeys)
        for k in namekeys[1:]:
            assert (df[namekeys[0]] == df[k]).all()
        df = df.drop(columns=namekeys[1:])
        # now take the median of the durations
        bkeys = []
        ekeys = []
        for k in df.keys():
            if "Start_Timestamp" in k:
                bkeys.append(k)
            if "End_Timestamp" in k:
                ekeys.append(k)
        # compute mean begin and end timestamps
        endNs = df[ekeys].mean(axis=1)
        beginNs = df[bkeys].mean(axis=1)
        # and replace
        df = df.drop(columns=bkeys)
        df = df.drop(columns=ekeys)
        df["Start_Timestamp"] = beginNs
        df["End_Timestamp"] = endNs
        # finally, join the drop key
        df = df.drop(columns=["key"])
        # save to file and delete old file(s), skip if we're being called outside of Omniperf
        if type(self.__args.path) == str:
            df.to_csv(out, index=False)
            if not self.__args.verbose:
                for file in files:
                    os.remove(file)
        else:
            return df

    #----------------------------------------------------
    # Required methods to be implemented by child classes
    #----------------------------------------------------
    @abstractmethod
    def pre_processing(self):
        """Perform any pre-processing steps prior to profiling.
        """
        logging.debug("[profiling] pre-processing using %s profiler" % self.__profiler)
        
        # verify soc compatibility
        if self.__profiler not in self._soc.get_compatible_profilers():
            error("%s is not enabled in %s. Available profilers include: %s" % (self._soc.get_soc_name(), self.__profiler, self._soc.get_compatible_profilers()))
        # verify not accessing parent directories
        if ".." in str(self.__args.path):
            error("Access denied. Cannot access parent directories in path (i.e. ../)")
        
        # verify correct formatting for application binary
        self.__args.remaining = self.__args.remaining[1:]
        if self.__args.remaining:
            if not os.path.isfile(self.__args.remaining[0]):
                error("Your command %s doesn't point to a executable. Please verify." % self.__args.remaining[0])
            self.__args.remaining = " ".join(self.__args.remaining)
        else:
            error("Profiling command required. Pass application executable after -- at the end of options.\n\t\ti.e. omniperf profile -n vcopy -- ./vcopy 1048576 256")
        
        # verify name meets MongoDB length requirements and no illegal chars
        if len(self.__args.name) > 35:
            error("-n/--name exceeds 35 character limit. Try again.")
        if self.__args.name.find(".") != -1 or self.__args.name.find("-") != -1:
            error("'-' and '.' are not permitted in -n/--name")

    @abstractmethod
    def run_profiling(self, version:str, prog:str):
        """Run profiling.
        """
        logging.debug("[profiling] performing profiling using %s profiler" % self.__profiler)
        
        # log basic info
        logging.info(str(prog) + " ver: " + str(version))
        logging.info("Path: " + str(os.path.abspath(self.__args.path)))
        logging.info("Target: " + str(self.__args.target))
        logging.info("Command: " + str(self.__args.remaining))
        logging.info("Kernel Selection: " + str(self.__args.kernel))
        logging.info("Dispatch Selection: " + str(self.__args.dispatch))
        if self.__args.ipblocks == None:
            logging.info("IP Blocks: All")
        else:
            logging.info("IP Blocks: "+ str(self.__args.ipblocks))
        if self.__args.kernel_verbose > 5:
            logging.info("KernelName verbose: DISABLED")
        else:
            logging.info("KernelName verbose: " + str(self.__args.kernel_verbose))

        # Run profiling on each input file
        for fname in glob.glob(self.get_args().path + "/perfmon/*.txt"):
            # Kernel filtering (in-place replacement)
            if not self.__args.kernel == None:
                success, output = capture_subprocess_output(
                    [
                        "sed",
                        "-i",
                        "-r",
                        "s%^(kernel:).*%" + "kernel: " + ",".join(self.__args.kernel) + "%g",
                        fname,
                    ]
                )
                # log output from profile filtering
                if not success:
                    error(output)
                else:
                    logging.debug(output)

            # Dispatch filtering (inplace replacement)
            if not self.__args.dispatch == None:
                success, output = capture_subprocess_output(
                    [
                        "sed",
                        "-i",
                        "-r",
                        "s%^(range:).*%" + "range: " + " ".join(self.__args.dispatch) + "%g",
                        fname,
                    ]
                )
                # log output from profile filtering
                if not success:
                    error(output)
                else:
                    logging.debug(output)
            logging.info("\nCurrent input file: %s" % fname)
            
            # Fetch any SoC/profiler specific profiling options
            options = self._soc.get_profiler_options()
            options += self.get_profiler_options(fname)

            if self.__profiler == "rocprofv1" or self.__profiler == "rocprofv2":
                run_prof(
                    fname=fname, 
                    # workload_dir=self.get_args().path, 
                    # perfmon_dir=self.__perfmon_dir, 
                    # cmd=self.__args.remaining,
                    # target=self.__args.target,
                    profiler_options=options,
                    target=self.__args.target,
                    workload_dir=self.get_args().path
                )

            elif self.__profiler == "rocscope":
                run_rocscope(self.__args, fname)
            else:
                #TODO: Finish logic
                error("profiler not supported")

    @abstractmethod
    def post_processing(self):
        """Perform any post-processing steps prior to profiling.
        """
        logging.debug("[profiling] performing post-processing using %s profiler" % self.__profiler)

        gen_sysinfo(
            workload_name=self.__args.name, 
            workload_dir=self.get_args().path, 
            ip_blocks=self.__args.ipblocks, 
            app_cmd=self.__args.remaining, 
            skip_roof=self.__args.no_roof, 
            roof_only=self.__args.roof_only,
        )

def test_df_column_equality(df):
    return df.eq(df.iloc[:, 0], axis=0).all(1).all()