diff --git a/projects/roctracer/bin/dform.py b/projects/roctracer/bin/dform.py new file mode 100644 index 0000000000..ff688f3ebc --- /dev/null +++ b/projects/roctracer/bin/dform.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +from sqlitedb import SQLiteDB + +def post_process_data(db, table_name, outfile = ''): +# db.add_data_column('A', 'DispDurNs', 'INTEGER', 'BeginNs - DispatchNs') +# db.add_data_column('A', 'ComplDurNs', 'INTEGER', 'CompleteNs - EndNs') +# db.add_data_column('A', 'TotalDurNs', 'INTEGER', 'CompleteNs - DispatchNs') +# db.add_data_column('A', 'TimeNs', 'INTEGER', 'BeginNs - (select BeginNs from A limit 1)') + db.add_data_column(table_name, 'DurationNs', 'INTEGER', 'EndNs - BeginNs') + if outfile != '': db.dump_csv(table_name, outfile) + +def gen_data_bins(db, outfile): + db.execute('create view C as select Name, Calls, TotalDurationNs, TotalDurationNs/Calls as AverageNs, TotalDurationNs*100.0/(select sum(TotalDurationNs) from %s) as Percentage from %s order by TotalDurationNs desc;' % ('B', 'B')); + db.dump_csv('C', outfile) + db.execute('DROP VIEW C') + +def gen_table_bins(db, table, outfile, name_var, dur_ns_var): + db.execute('create view B as select (%s) as Name, count(%s) as Calls, sum(%s) as TotalDurationNs from %s group by %s' % (name_var, name_var, dur_ns_var, table, name_var)) + gen_data_bins(db, outfile) + db.execute('DROP VIEW B') + +def gen_api_json_trace(db, table, outfile): + db.execute('create view B as select Name as name, (BeginNs/1000) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % table); + db.dump_json('B', table, outfile) + db.execute('DROP VIEW B') + +def gen_kernel_json_trace(db, table, outfile): + db.execute('create view B as select (KernelName) as name, ("gpu-id") as pid, (0) as tid, (BeginNs/1000) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % table); + db.dump_json('B', table, outfile) + db.execute('DROP VIEW B') +############################################################################################## diff --git a/projects/roctracer/bin/rpl_run.sh b/projects/roctracer/bin/rpl_run.sh new file mode 100755 index 0000000000..ef26472ed6 --- /dev/null +++ b/projects/roctracer/bin/rpl_run.sh @@ -0,0 +1,379 @@ +#!/bin/sh + +################################################################################ +# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +time_stamp=`date +%y%m%d_%H%M%S` +BIN_DIR=`dirname $0` +BIN_DIR=`cd $BIN_DIR; pwd` +RUN_DIR=`pwd` +TMP_DIR="/tmp" +DATA_DIR="rpl_data_${time_stamp}_$$" + +PKG_DIR=`echo $BIN_DIR | sed "s/\/bin\/*//"` +BIN_DIR=$PKG_DIR/bin + +# PATH to custom HSA and OpenCl runtimes +HSA_PATH=$PKG_DIR/lib/hsa + +# runtime API trace +HSA_TRACE=0 +HIP_TRACE=0 + +# Generate stats +GEN_STATS=0 + +export LD_LIBRARY_PATH=$PKG_DIR/lib:$PKG_DIR/tool:$PKG_DIR/roctracer/lib:$PKG_DIR/roctracer/tool:$HSA_PATH +export PATH=.:$PATH + +# enable error logging +export HSA_TOOLS_REPORT_LOAD_FAILURE=1 +export HSA_VEN_AMD_AQLPROFILE_LOG=1 +export ROCPROFILER_LOG=1 +unset ROCPROFILER_SESS + +# ROC Profiler environment +# Loading of ROC Profiler by HSA runtime +export HSA_TOOLS_LIB=librocprofiler64.so +# Loading of the test tool by ROC Profiler +export ROCP_TOOL_LIB=libtool.so +# Enabling HSA dispatches intercepting by ROC PRofiler +export ROCP_HSA_INTERCEPT=1 +# Disabling internal ROC Profiler proxy queue (simple version supported for testing purposes) +unset ROCP_PROXY_QUEUE +# ROC Profiler metrics definition +export ROCP_METRICS=$PKG_DIR/lib/metrics.xml +# Disable AQL-profile read API +export AQLPROFILE_READ_API=0 +# ROC Profiler package path +export ROCP_PACKAGE_DIR=$PKG_DIR + +# error handling +fatal() { + echo "$0: Error: $1" + echo "" + usage +} + +error() { + echo "$0: Error: $1" + echo "" + exit 1 +} + +# usage method +usage() { + bin_name=`basename $0` + echo "ROCm Profiling Library (RPL) run script, a part of ROCprofiler library package." + echo "Full path: $BIN_DIR/$bin_name" + echo "Metrics definition: $PKG_DIR/lib/metrics.xml" + echo "" + echo "Usage:" + echo " $bin_name [-h] [--list-basic] [--list-derived] [-i ] [-o ] " + echo "" + echo "Options:" + echo " -h - this help" + echo " --verbose - verbose mode, dumping all base counters used in the input metrics" + echo " --list-basic - to print the list of basic HW counters" + echo " --list-derived - to print the list of derived metrics with formulas" + echo "" + echo " -i <.txt|.xml file> - input file" + echo " Input file .txt format, automatically rerun application for every pmc line:" + echo "" + echo " # Perf counters group 1" + echo " pmc : Wavefronts VALUInsts SALUInsts SFetchInsts FlatVMemInsts LDSInsts FlatLDSInsts GDSInsts VALUUtilization FetchSize" + echo " # Perf counters group 2" + echo " pmc : WriteSize L2CacheHit" + echo " # Filter by dispatches range, GPU index and kernel names" + echo " # supported range formats: \"3:9\", \"3:\", \"3\"" + echo " range: 1 : 4" + echo " gpu: 0 1 2 3" + echo " kernel: simple Pass1 simpleConvolutionPass2" + echo "" + echo " Input file .xml format, for single profiling run:" + echo "" + echo " # Metrics list definition, also the form \":\" can be used" + echo " # All defined metrics can be found in the 'metrics.xml'" + echo " # There are basic metrics for raw HW counters and high-level metrics for derived counters" + echo " " + echo "" + echo " # Filter by dispatches range, GPU index and kernel names" + echo " " + echo "" + echo " -o - output CSV file [.csv]" + echo " -d - directory where profiler store profiling data including thread treaces [/tmp]" + echo " The data directory is renoving autonatically if the directory is matching the temporary one, which is the default." + echo " -t - to change the temporary directory [/tmp]" + echo " By changing the temporary directory you can prevent removing the profiling data from /tmp or enable removing from not '/tmp' directory." + echo "" + echo " --basenames - to turn on/off truncating of the kernel full function names till the base ones [off]" + echo " --timestamp - to turn on/off the kernel disoatches timestamps, dispatch/begin/end/complete [off]" + echo " --ctx-wait - to wait for outstanding contexts on profiler exit [on]" + echo " --ctx-limit - maximum number of outstanding contexts [0 - unlimited]" + echo " --heartbeat - to print progress heartbeats [0 - disabled]" + echo "" + echo " --stats - generating stats and json trace output" + echo " --hsa-trace - to trace HSA" + echo " --hip-trace - to trace HIP" + echo "" + echo "Configuration file:" + echo " You can set your parameters defaults preferences in the configuration file 'rpl_rc.xml'. The search path sequence: .:${HOME}:" + echo " First the configuration file is looking in the current directory, then in your home, and then in the package directory." + echo " Configurable options: 'basenames', 'timestamp', 'ctx-limit', 'heartbeat'." + echo " An example of 'rpl_rc.xml':" + echo " " + echo "" + exit 1 +} + +# profiling run method +OUTPUT_LIST="" +run() { + export ROCP_INPUT="$1" + OUTPUT_DIR="$2" + shift + shift + APP_CMD=$* + + if [ "$OUTPUT_DIR" = "-" ] ; then + input_tag=`echo $ROCP_INPUT | sed "s/\.xml//"` + export ROCP_OUTPUT_DIR=${input_tag}_results_${time_stamp} + elif [ "$OUTPUT_DIR" = "--" ] ; then + unset ROCP_OUTPUT_DIR + else + export ROCP_OUTPUT_DIR=$OUTPUT_DIR + fi + echo "RPL: result dir '$ROCP_OUTPUT_DIR'" + + if [ ! -e "$ROCP_INPUT" ] ; then + error "Input file '$ROCP_INPUT' not found" + fi + + if [ -n "$ROCP_OUTPUT_DIR" ] ; then + if [ "$OUTPUT_DIR" = "-" ] ; then + if [ -e "$ROCP_OUTPUT_DIR" ] ; then + error "generated dir '$ROCP_OUTPUT_DIR' exists" + fi + fi + mkdir -p "$ROCP_OUTPUT_DIR" + fi + + if [ "$HSA_TRACE" = 1 ] ; then + export ROCTRACER_DOMAIN="hsa" + export HSA_TOOLS_LIB="libtracer_tool.so libroctracer64.so $HSA_TOOLS_LIB" + fi + + redirection_cmd="" + if [ -n "$ROCP_OUTPUT_DIR" ] ; then + OUTPUT_LIST="$OUTPUT_LIST $ROCP_OUTPUT_DIR/results.txt" + redirection_cmd="2>&1 | tee $ROCP_OUTPUT_DIR/log.txt" + fi + + #unset ROCP_OUTPUT_DIR + eval "LD_PRELOAD='$HSA_TOOLS_LIB' $APP_CMD $redirection_cmd" +} + +# main +echo "RPL: on '$time_stamp' from '$PKG_DIR' in '$RUN_DIR'" +# Parsing arguments +if [ -z "$1" ] ; then + usage +fi + +INPUT_FILE="" +DATA_PATH="-" +OUTPUT_DIR="-" +output="" +csv_output="" + +ARG_IN="" +while [ 1 ] ; do + ARG_IN=$1 + ARG_VAL=1 + if [ "$1" = "-h" ] ; then + usage + elif [ "$1" = "-i" ] ; then + INPUT_FILE="$2" + elif [ "$1" = "-o" ] ; then + output="$2" + elif [ "$1" = "-d" ] ; then + DATA_PATH=$2 + elif [ "$1" = "-t" ] ; then + TMP_DIR="$2" + if [ "$OUTPUT_DIR" = "-" ] ; then + DATA_PATH=$TMP_DIR + fi + elif [ "$1" = "--list-basic" ] ; then + export ROCP_INFO=b + eval "$PKG_DIR/tool/ctrl" + exit 1 + elif [ "$1" = "--list-derived" ] ; then + export ROCP_INFO=d + eval "$PKG_DIR/tool/ctrl" + exit 1 + elif [ "$1" = "--basenames" ] ; then + if [ "$2" = "on" ] ; then + export ROCP_TRUNCATE_NAMES=1 + else + export ROCP_TRUNCATE_NAMES=0 + fi + elif [ "$1" = "--timestamp" ] ; then + if [ "$2" = "on" ] ; then + export ROCP_TIMESTAMP_ON=1 + else + export ROCP_TIMESTAMP_ON=0 + fi + elif [ "$1" = "--ctx-wait" ] ; then + if [ "$2" = "on" ] ; then + export ROCP_OUTSTANDING_WAIT=1 + else + export ROCP_OUTSTANDING_WAIT=0 + fi + elif [ "$1" = "--ctx-limit" ] ; then + export ROCP_OUTSTANDING_MAX="$2" + elif [ "$1" = "--heartbeat" ] ; then + export ROCP_OUTSTANDING_MON="$2" + elif [ "$1" = "--hsa-trace" ] ; then + ARG_VAL=0 + HSA_TRACE=1 + elif [ "$1" = "--hip-trace" ] ; then + ARG_VAL=0 + HIP_TRACE=1 + elif [ "$1" = "--verbose" ] ; then + ARG_VAL=0 + export ROCP_VERBOSE_MODE=1 + elif [ "$1" = "--stats" ] ; then + ARG_VAL=0 + GEN_STATS=1 + else + break + fi + shift + if [ "$ARG_VAL" = 1 ] ; then shift; fi +done + +ARG_CK=`echo $ARG_IN | sed "s/^-.*$/-/"` +if [ "$ARG_CK" = "-" ] ; then + fatal "Wrong option '$ARG_IN'" +fi + +if [ -z "$INPUT_FILE" ] ; then + input_base="results" + input_type="none" +else + input_base=`echo "$INPUT_FILE" | sed "s/^\(.*\)\.\([^\.]*\)$/\1/"` + input_type=`echo "$INPUT_FILE" | sed "s/^\(.*\)\.\([^\.]*\)$/\2/"` + if [ -z "${input_base}" -o -z "${input_type}" ] ; then + fatal "Bad input file '$INPUT_FILE'" + fi + input_base=`basename $input_base` +fi + +if [ "$DATA_PATH" = "-" ] ; then + DATA_PATH=$TMP_DIR +fi + +if [ -n "$output" ] ; then + if [ "$output" = "--" ] ; then + OUTPUT_DIR="--" + else + csv_output=$output + fi +else + csv_output=$RUN_DIR/${input_base}.csv +fi + +APP_CMD=$* + +echo "RPL: profiling '$APP_CMD'" +echo "RPL: input file '$INPUT_FILE'" + +input_list="" +RES_DIR="" +if [ "$input_type" = "xml" ] ; then + OUTPUT_DIR=$DATA_PATH + input_list=$INPUT_FILE +elif [ "$input_type" = "txt" -o "$input_type" = "none" ] ; then + RES_DIR=$DATA_PATH/$DATA_DIR + if [ -e $RES_DIR ] ; then + error "Rundir '$RES_DIR' exists" + fi + mkdir -p $RES_DIR + echo "RPL: output dir '$RES_DIR'" + if [ "$input_type" = "txt" ] ; then + $BIN_DIR/txt2xml.sh $INPUT_FILE $RES_DIR + else + echo "" > $RES_DIR/input.xml + fi + input_list=`/bin/ls $RES_DIR/input*.xml` + export ROCPROFILER_SESS=$RES_DIR +else + fatal "Bad input file type '$INPUT_FILE'" +fi + +if [ -n "$csv_output" ] ; then + rm -f $csv_output +fi + +for name in $input_list; do + run $name $OUTPUT_DIR $APP_CMD + if [ -n "$ROCPROFILER_SESS" -a -e "$ROCPROFILER_SESS/error" ] ; then + echo "Error found, profiling aborted." + csv_output="" + break + fi +done + +if [ -n "$csv_output" ] ; then + if [ "$GEN_STATS" = "1" ] ; then + db_output=$(echo $csv_output | sed "s/\.csv/.db/") + python $BIN_DIR/tblextr.py $db_output $OUTPUT_LIST + else + python $BIN_DIR/tblextr.py $csv_output $OUTPUT_LIST + fi + if [ "$?" -eq 0 ] ; then + echo "RPL: '$csv_output' is generated" + else + echo "Data extracting error: $OUTPUT_LIST'" + fi +fi + +if [ "$DATA_PATH" = "$TMP_DIR" ] ; then + if [ -e "$RES_DIR" ] ; then + rm -rf $RES_DIR + fi +fi + +exit 0 diff --git a/projects/roctracer/bin/sqlitedb.py b/projects/roctracer/bin/sqlitedb.py new file mode 100644 index 0000000000..789ac460cd --- /dev/null +++ b/projects/roctracer/bin/sqlitedb.py @@ -0,0 +1,196 @@ +import csv, sqlite3, re + +# SQLite Database class +class SQLiteDB: + def __init__(self, file_name): + self.connection = sqlite3.connect(file_name) + self.tables = {} + + def __del__(self): + self.connection.close() + + # add DB table + def add_table(self, name, descr, extra = ()): + (field_list, field_dict) = descr + if name in self.tables: raise Exception('table is already added: "' + name + '"') + + # create DB table + table_descr = [] + for field in field_list: table_descr.append('"%s" %s' % (field, field_dict[field])) + for item in extra: table_descr.append('"%s" %s' % (item[0], item[1])) + stm = 'CREATE TABLE ' + name + ' (%s)' % ', '.join(table_descr) + cursor = self.connection.cursor() + cursor.execute(stm) + self.connection.commit() + + # register table + fields_str = ','.join(map(lambda x: '"' + x + '"', field_list)) + templ_str = ','.join('?' * len(field_list)) + stm = 'INSERT INTO ' + name + '(' + fields_str + ') VALUES(' + templ_str + ');' + self.tables[name] = stm + + return (cursor, stm); + + # add columns to table + def add_columns(self, name, columns): + cursor = self.connection.cursor() + for item in columns: + stm = 'ALTER TABLE ' + name + ' ADD COLUMN "%s" %s' % (item[0], item[1]) + cursor.execute(stm) + self.connection.commit() + + # add columns with expression + def add_data_column(self, table_name, data_label, data_type, data_expr): + cursor = self.connection.cursor() + cursor.execute('ALTER TABLE %s ADD COLUMN "%s" %s' % (table_name, data_label, data_type)) + cursor.execute('UPDATE %s SET %s = (%s);' % (table_name, data_label, data_expr)) + + # populate DB table entry + def insert_entry(self, table, val_list): + (cursor, stm) = table + cursor.execute(stm, val_list) + + # populate DB table entry + def commit_entry(self, table, val_list): + self.insert_entry(table, val_list) + self.connection.commit() + + # populate DB table data + def insert_table(self, table, reader): + for val_list in reader: + if not val_list[-1]: val_list.pop() + self.insert_entry(table, val_list) + self.connection.commit() + + # return table fields list + def _get_fields(self, table_name): + cursor = self.connection.execute('SELECT * FROM ' + table_name) + return list(map(lambda x: '"%s"' % (x[0]), cursor.description)) + + # return table raws list + def _get_raws(self, table_name): + cursor = self.connection.execute('SELECT * FROM ' + table_name) + return cursor.fetchall() + + # dump CSV table + def dump_csv(self, table_name, file_name): + if not re.search(r'\.csv$', file_name): + raise Exception('wrong output file type: "' + file_name + '"' ) + + fields = self._get_fields(table_name) + stm = 'select ' + ','.join(fields) + ' from ' + table_name + + with open(file_name, mode='w') as fd: + fd.write(','.join(fields) + '\n') + for values in self.connection.execute(stm): + fd.write(reduce(lambda a, b: str(a) + ',' + str(b), values) + '\n') + + # dump JSON trace + def open_json(self, file_name): + if not re.search(r'\.json$', file_name): + raise Exception('wrong output file type: "' + file_name + '"' ) + with open(file_name, mode='w') as fd: + fd.write('{ "traceEvents":[{}\n'); + def close_json(self, file_name): + if not re.search(r'\.json$', file_name): + raise Exception('wrong output file type: "' + file_name + '"' ) + with open(file_name, mode='a') as fd: + fd.write(']}\n'); + + def label_json(self, pid, label, file_name): + if not re.search(r'\.json$', file_name): + raise Exception('wrong output file type: "' + file_name + '"' ) + with open(file_name, mode='a') as fd: + fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name"}\n' %(label, pid)); + + def dump_json(self, table_name, data_name, file_name): + if not re.search(r'\.json$', file_name): + raise Exception('wrong output file type: "' + file_name + '"' ) + + sub_ptrn = re.compile(r'(^"|"$)') + name_ptrn = re.compile(r'(name|Name)') + + table_fields = self._get_fields(table_name) + table_raws = self._get_raws(table_name) + data_fields = self._get_fields(data_name) + data_raws = self._get_raws(data_name) + + with open(file_name, mode='a') as fd: + for raw_index in range(len(table_raws)): + values = list(table_raws[raw_index]) + data = list(data_raws[raw_index]) + + vals_list = [] + for value_index in range(len(values)): + label = table_fields[value_index] + value = values[value_index] + if name_ptrn.search(label): value = sub_ptrn.sub(r'', value) + vals_list.append('%s:"%s"' % (label, value)) + + args_list = [] + for value_index in range(len(data)): + label = data_fields[value_index] + value = data[value_index] + if name_ptrn.search(label): value = sub_ptrn.sub(r'', value) + args_list.append('%s:"%s"' % (label, value)) + + fd.write(',{"ph":"%s",%s,\n "args":{\n %s\n }\n}\n' % ('X', ','.join(vals_list), ',\n '.join(args_list))) + + # execute query on DB + def execute(self, cmd): + cursor = self.connection.cursor() + cursor.execute(cmd) + + # commit DB + def commit(self): + self.connection.commit() + + # close DB + def close(self): + self.connection.close() + + # access DB + def get_raws(self, table_name): + cur = self.connection.cursor() + cur.execute("SELECT * FROM %s" % table_name) + return cur.fetchall() + + # return CSV descriptor + # list of fields and dictionaly for the fields types + def _get_csv_descr(self, table_name, fd): + reader = csv.DictReader(fd) + field_names = reader.fieldnames + if not field_names[-1]: field_names.pop() + field_types = {} + + for entry in reader: + fields_left = [f for f in field_names if f not in field_types.keys()] + # all fields processed + if not fields_left: break + + for field in fields_left: + data = entry[field] + # need data for the field to be processed + if len(data) == 0: continue + + if data.isdigit(): + field_types[field] = "INTEGER" + else: + field_types[field] = "TEXT" + + if len(fields_left) > 0: raise Exception('types not found for fields: ', fields_left) + return (field_names, field_types) + + # add CSV table + def add_csv_table(self, table_name, file_name, extra = ()): + with open(file_name, mode='r') as fd: + # get CSV table descriptor + descr = self._get_csv_descr(table_name, fd) + # reader to populate the table + fd.seek(0) + reader = csv.reader(fd) + reader.next() + table = self.add_table(table_name, descr, extra) + self.insert_table(table, reader) + +############################################################################################## diff --git a/projects/roctracer/bin/tblextr.py b/projects/roctracer/bin/tblextr.py new file mode 100644 index 0000000000..740a0dd1f1 --- /dev/null +++ b/projects/roctracer/bin/tblextr.py @@ -0,0 +1,259 @@ +#!/usr/bin/python + +################################################################################ +# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +import os, sys, re +from sqlitedb import SQLiteDB +import dform + +# Parsing results in the format: +#dispatch[0], queue_index(0), kernel_name("SimpleConvolution"), time(1048928000311041,1048928006154674,1048928006168274,1048928006170503): +# GRBM_GUI_ACTIVE (74332) +# SQ_WAVES (4096) +# SQ_INSTS_VMEM_RD (36864) + +COPY_PID = 0 +HSA_PID = 1 +GPU_BASE_PID = 2 +max_gpu_id = 0 + +# global vars +table_descr = [ + ['Index', 'KernelName'], + {'Index': 'INTEGER', 'KernelName': 'TEXT'} +] +var_list = table_descr[0] +var_table = {} +############################################################# + +def fatal(msg): + sys.stderr.write(sys.argv[0] + ": " + msg + "\n"); + sys.exit(1) +############################################################# + +# parse results method +def parse_res(infile): + global max_gpu_id + if not os.path.isfile(infile): fatal("Error: input file '" + infile + "' not found") + inp = open(infile, 'r') + + beg_pattern = re.compile("^dispatch\[(\d*)\], (.*) kernel-name\(\"([^\"]*)\"\)") + prop_pattern = re.compile("([\w-]+)\((\w+)\)"); + ts_pattern = re.compile(", time\((\d*),(\d*),(\d*),(\d*)\)") + var_pattern = re.compile("^\s*([^\s]*)\s+\((\d*)\)") + + dispatch_number = 0 + for line in inp.readlines(): + record = line[:-1] + + m = var_pattern.match(record) + if m: + if not dispatch_number in var_table: fatal("Error: dispatch number not found '" + str(dispatch_number) + "'") + var = m.group(1) + val = m.group(2) + var_table[dispatch_number][m.group(1)] = m.group(2) + if not var in var_list: var_list.append(var) + + m = beg_pattern.match(record) + if m: + dispatch_number = m.group(1) + if not dispatch_number in var_table: + var_table[dispatch_number] = { + 'Index': dispatch_number, + 'KernelName': "\"" + m.group(3) + "\"" + } + kernel_properties = m.group(2) + for prop in kernel_properties.split(', '): + m = prop_pattern.match(prop) + if m: + var = m.group(1) + val = m.group(2) + var_table[dispatch_number][var] = val + if not var in var_list: var_list.append(var); + if re.search(r'gpu-id', var): + if (val > max_gpu_id): max_gpu_id = val + else: fatal('wrong kernel property "' + prop + '" in "'+ kernel_properties + '"') + m = ts_pattern.search(record) + if m: + var_table[dispatch_number]['DispatchNs'] = m.group(1) + var_table[dispatch_number]['BeginNs'] = m.group(2) + var_table[dispatch_number]['EndNs'] = m.group(3) + var_table[dispatch_number]['CompleteNs'] = m.group(4) + + inp.close() +############################################################# + +# merge results table +def merge_table(): + global var_list + keys = sorted(var_table.keys(), key=int) + + fields = set(var_table[keys[0]]) + if 'DispatchNs' in fields: + var_list.append('DispatchNs') + var_list.append('BeginNs') + var_list.append('EndNs') + var_list.append('CompleteNs') + var_list = [x for x in var_list if x in fields] +############################################################# + +# dump CSV results +def dump_csv(file_name): + global var_list + keys = sorted(var_table.keys(), key=int) + + with open(file_name, mode='w') as fd: + fd.write(','.join(var_list) + '\n'); + for ind in keys: + entry = var_table[ind] + dispatch_number = entry['Index'] + if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n") + val_list = [entry[var] for var in var_list] + fd.write(','.join(val_list) + '\n'); +############################################################# + +# fill kernels DB +def fill_kernel_db(table_name, db): + global var_list + keys = sorted(var_table.keys(), key=int) + + for var in set(var_list).difference(set(table_descr[1])): + table_descr[1][var] = 'INTEGER' + table_descr[0] = var_list; + + table_handle = db.add_table(table_name, table_descr) + + for ind in keys: + entry = var_table[ind] + dispatch_number = entry['Index'] + if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n") + val_list = [entry[var] for var in var_list] + db.insert_entry(table_handle, val_list) +############################################################# + +# fill HSA DB +hsa_table_descr = [ + ['BeginNs', 'EndNs', 'pid', 'tid', 'Name', 'args'], + {'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'} +] +def fill_hsa_db(table_name, db, indir): + file_name = indir + '/' + 'hsa_api_trace.txt' + ptrn = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$') + + table_handle = db.add_table(table_name, hsa_table_descr) + with open(file_name, mode='r') as fd: + for line in fd.readlines(): + record = line[:-1] + m = ptrn.match(record) + if m: + rec_vals = [] + for ind in range(1,7): + if hsa_table_descr[0][ind - 1] == 'pid': + rec_vals.append(HSA_PID) + else: + rec_vals.append(m.group(ind)) + db.insert_entry(table_handle, rec_vals) +############################################################# + +# fill COPY DB +copy_table_descr = [ + ['BeginNs', 'EndNs', 'Name', 'pid', 'tid'], + {'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'} +] +def fill_copy_db(table_name, db, indir): + file_name = indir + '/' + 'async_copy_trace.txt' + ptrn = re.compile(r'(\d+):(\d+) (.*)$') + + table_handle = db.add_table(table_name, copy_table_descr) + with open(file_name, mode='r') as fd: + for line in fd.readlines(): + record = line[:-1] + m = ptrn.match(record) + if m: + rec_vals = [] + for ind in range(1,4): rec_vals.append(m.group(ind)) + rec_vals.append(COPY_PID) + rec_vals.append(0) + db.insert_entry(table_handle, rec_vals) +############################################################# +# main +if (len(sys.argv) < 3): fatal("Usage: " + sys.argv[0] + " ") + +outfile = sys.argv[1] +infiles = sys.argv[2:] +indir = re.sub(r'\/[^\/]*$', r'', infiles[0]) +print "indir: '" + indir + "'" + +dbfile = '' +csvfile = '' + +if re.search(r'\.csv$', outfile): + csvfile = outfile +elif re.search(r'\.db$', outfile): + dbfile = outfile + csvfile = re.sub(r'\.db$', '.csv', outfile) +else: + fatal("Bad output file '" + outfile + "'") + +for f in infiles: parse_res(f) +if len(var_table) == 0: sys.exit(1) +merge_table() + +if dbfile == '': + dump_csv(csvfile) +else: + statfile = re.sub(r'\.csv$', '.stats.csv', csvfile) + jsonfile = re.sub(r'\.csv$', '.json', csvfile) + + with open(dbfile, mode='w') as fd: fd.truncate() + db = SQLiteDB(dbfile) + + fill_hsa_db('HSA', db, indir) + fill_copy_db('COPY', db, indir) + fill_kernel_db('A', db) + + db.open_json(jsonfile); + db.label_json(HSA_PID, "CPU", jsonfile) + db.label_json(COPY_PID, "COPY", jsonfile) + for ind in range(0, int(max_gpu_id) + 1): db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile) + + if 'BeginNs' in var_list: + dform.post_process_data(db, 'A', csvfile) + dform.gen_table_bins(db, 'A', statfile, 'KernelName', 'DurationNs') + dform.gen_kernel_json_trace(db, 'A', jsonfile) + else: + db.dump_csv('A', csvfile) + + statfile = re.sub(r'stats', r'hsa_stats', statfile) + dform.post_process_data(db, 'HSA') + dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs') + dform.gen_api_json_trace(db, 'HSA', jsonfile) + + dform.post_process_data(db, 'COPY') + dform.gen_api_json_trace(db, 'COPY', jsonfile) + + db.close_json(jsonfile); + db.close() + +sys.exit(0) +############################################################# diff --git a/projects/roctracer/bin/txt2xml.sh b/projects/roctracer/bin/txt2xml.sh new file mode 100755 index 0000000000..27bbe8c474 --- /dev/null +++ b/projects/roctracer/bin/txt2xml.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +################################################################################ +# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ + +timestamp=`date +%y%m%d_%H%M%S` + +if [ $# = 0 ] ; then + echo "Usage: $0 [output dir]" + exit -1 +fi + +input=$1 +outdir=$2 +if [ -z "$outdir" ] ; then + outdir="." +fi + +range="" +kernel="" +gpu_index="" + +parse() { + scan="$1" + index=0 + while read -r line || [[ -n "$line" ]] ; do + line=`echo $line | sed "s/\s*#.*$//"` + if [ -z "$line" ] ; then + continue + fi + + feature=`echo $line | sed -n "s/^\s*\([a-z]*\)\s*:.*$/\1/p"` + line=`echo $line | sed "s/^[^:]*:\s*//"` + line=`echo "$line" | sed -e "s/\s*=\s*/=/g" -e "s/\s*:\s*/:/g" -e "s/,\{1,\}/ /g" -e "s/\s\{1,\}/ /g" -e "s/\s*$//"` + + if [ "$scan" = 0 ] ; then + line=`echo "$line" | sed -e "s/ /,/g"` + if [ "$feature" == "range" ] ; then + range=$line + fi + if [ "$feature" == "kernel" ] ; then + kernel=$line + fi + if [ "$feature" == "gpu" ] ; then + gpu_index=$line + fi + else + output=$outdir/input${index}.xml + header="# $timestamp '$output' generated with '$0 $*'" + echo $header > $output + + if [ "$feature" == "pmc" ] ; then + line=`echo "$line" | sed -e "s/ /,/g"` + cat >> $output < + +EOF + fi + + if [ "$feature" == "sqtt" ] ; then + cat >> $output < + +EOF + fi + + if [ "$feature" == "hsa" ] ; then + cat >> $output < +EOF + fi + fi + + index=$((index + 1)) + done < $input +} + +parse 0 +parse 1 + +exit 0