diff --git a/bin/dform.py b/bin/dform.py deleted file mode 100644 index 107188fa0a..0000000000 --- a/bin/dform.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/python -from sqlitedb import SQLiteDB - -def post_process_data(db, table_name, outfile = ''): -# db.add_data_column('A', 'DispDurNs', 'INTEGER', 'BeginNs - DispatchNs') -# db.add_data_column('A', 'ComplDurNs', 'INTEGER', 'CompleteNs - EndNs') -# db.add_data_column('A', 'TotalDurNs', 'INTEGER', 'CompleteNs - DispatchNs') -# db.add_data_column(table_name, 'TimeNs', 'INTEGER', 'BeginNs - %d' % start_ns) - db.add_data_column(table_name, 'DurationNs', 'INTEGER', 'EndNs - BeginNs') - if outfile != '': db.dump_csv(table_name, outfile) - -def gen_data_bins(db, outfile): - db.execute('create view C as select Name, Calls, TotalDurationNs, TotalDurationNs/Calls as AverageNs, TotalDurationNs*100.0/(select sum(TotalDurationNs) from %s) as Percentage from %s order by TotalDurationNs desc;' % ('B', 'B')); - db.dump_csv('C', outfile) - db.execute('DROP VIEW C') - -def gen_table_bins(db, table, outfile, name_var, dur_ns_var): - db.execute('create view B as select (%s) as Name, count(%s) as Calls, sum(%s) as TotalDurationNs from %s group by %s' % (name_var, name_var, dur_ns_var, table, name_var)) - gen_data_bins(db, outfile) - db.execute('DROP VIEW B') - -def gen_api_json_trace(db, table, start_us, outfile): - db.execute('create view B as select "Index", Name as name, pid, tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % (start_us, table)); - db.dump_json('B', table, outfile) - db.execute('DROP VIEW B') - -def gen_kernel_json_trace(db, table, base_pid, start_us, outfile): - db.execute('create view B as select "Index", KernelName as name, ("gpu-id" + %d) as pid, (0) as tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % (base_pid, start_us, table)); - db.dump_json('B', table, outfile) - db.execute('DROP VIEW B') -############################################################################################## diff --git a/bin/rpl_run.sh b/bin/rpl_run.sh deleted file mode 100755 index efedfbdba9..0000000000 --- a/bin/rpl_run.sh +++ /dev/null @@ -1,377 +0,0 @@ -#!/bin/sh - -################################################################################ -# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -################################################################################ - -time_stamp=`date +%y%m%d_%H%M%S` -BIN_DIR=$(dirname $(realpath $0)) -PKG_DIR=$(dirname $BIN_DIR) -ROOT_DIR=$(dirname $PKG_DIR) -RUN_DIR=`pwd` -TMP_DIR="/tmp" -DATA_DIR="rpl_data_${time_stamp}_$$" - -# roctracer path -if [ -z "$ROCTRACER_PATH" ] ; then ROCTRACER_PATH=$ROOT_DIR/roctracer; fi - -# runtime API trace -HSA_TRACE=0 -HIP_TRACE=0 - -# Generate stats -GEN_STATS=0 - -export LD_LIBRARY_PATH=$PKG_DIR/lib:$PKG_DIR/lib/hsa:$PKG_DIR/tool:$ROCTRACER_PATH/lib:$ROCTRACER_PATH/tool -export PATH=.:$PATH - -# enable error logging -export HSA_TOOLS_REPORT_LOAD_FAILURE=1 -export HSA_VEN_AMD_AQLPROFILE_LOG=1 -export ROCPROFILER_LOG=1 -unset ROCPROFILER_SESS - -# ROC Profiler environment -# Loading of ROC Profiler by HSA runtime -export HSA_TOOLS_LIB=librocprofiler64.so -# Loading of the test tool by ROC Profiler -export ROCP_TOOL_LIB=libtool.so -# Enabling HSA dispatches intercepting by ROC PRofiler -export ROCP_HSA_INTERCEPT=1 -# Disabling internal ROC Profiler proxy queue (simple version supported for testing purposes) -unset ROCP_PROXY_QUEUE -# ROC Profiler metrics definition -export ROCP_METRICS=$PKG_DIR/lib/metrics.xml -# Disable AQL-profile read API -export AQLPROFILE_READ_API=0 -# ROC Profiler package path -export ROCP_PACKAGE_DIR=$PKG_DIR - -# error handling -fatal() { - echo "$0: Error: $1" - echo "" - usage -} - -error() { - echo "$0: Error: $1" - echo "" - exit 1 -} - -# usage method -usage() { - bin_name=`basename $0` - echo "ROCm Profiling Library (RPL) run script, a part of ROCprofiler library package." - echo "Full path: $BIN_DIR/$bin_name" - echo "Metrics definition: $PKG_DIR/lib/metrics.xml" - echo "" - echo "Usage:" - echo " $bin_name [-h] [--list-basic] [--list-derived] [-i ] [-o ] " - echo "" - echo "Options:" - echo " -h - this help" - echo " --verbose - verbose mode, dumping all base counters used in the input metrics" - echo " --list-basic - to print the list of basic HW counters" - echo " --list-derived - to print the list of derived metrics with formulas" - echo "" - echo " -i <.txt|.xml file> - input file" - echo " Input file .txt format, automatically rerun application for every pmc line:" - echo "" - echo " # Perf counters group 1" - echo " pmc : Wavefronts VALUInsts SALUInsts SFetchInsts FlatVMemInsts LDSInsts FlatLDSInsts GDSInsts VALUUtilization FetchSize" - echo " # Perf counters group 2" - echo " pmc : WriteSize L2CacheHit" - echo " # Filter by dispatches range, GPU index and kernel names" - echo " # supported range formats: \"3:9\", \"3:\", \"3\"" - echo " range: 1 : 4" - echo " gpu: 0 1 2 3" - echo " kernel: simple Pass1 simpleConvolutionPass2" - echo "" - echo " Input file .xml format, for single profiling run:" - echo "" - echo " # Metrics list definition, also the form \":\" can be used" - echo " # All defined metrics can be found in the 'metrics.xml'" - echo " # There are basic metrics for raw HW counters and high-level metrics for derived counters" - echo " " - echo "" - echo " # Filter by dispatches range, GPU index and kernel names" - echo " " - echo "" - echo " -o - output CSV file [.csv]" - echo " -d - directory where profiler store profiling data including thread treaces [/tmp]" - echo " The data directory is renoving autonatically if the directory is matching the temporary one, which is the default." - echo " -t - to change the temporary directory [/tmp]" - echo " By changing the temporary directory you can prevent removing the profiling data from /tmp or enable removing from not '/tmp' directory." - echo "" - echo " --basenames - to turn on/off truncating of the kernel full function names till the base ones [off]" - echo " --timestamp - to turn on/off the kernel disoatches timestamps, dispatch/begin/end/complete [off]" - echo " --ctx-wait - to wait for outstanding contexts on profiler exit [on]" - echo " --ctx-limit - maximum number of outstanding contexts [0 - unlimited]" - echo " --heartbeat - to print progress heartbeats [0 - disabled]" - echo "" - echo " --stats - generating stats and json trace output" - echo " --hsa-trace - to trace HSA" - echo " --hip-trace - to trace HIP" - echo "" - echo "Configuration file:" - echo " You can set your parameters defaults preferences in the configuration file 'rpl_rc.xml'. The search path sequence: .:${HOME}:" - echo " First the configuration file is looking in the current directory, then in your home, and then in the package directory." - echo " Configurable options: 'basenames', 'timestamp', 'ctx-limit', 'heartbeat'." - echo " An example of 'rpl_rc.xml':" - echo " " - echo "" - exit 1 -} - -# profiling run method -OUTPUT_LIST="" -run() { - export ROCP_INPUT="$1" - OUTPUT_DIR="$2" - shift - shift - APP_CMD=$* - - if [ "$OUTPUT_DIR" = "-" ] ; then - input_tag=`echo $ROCP_INPUT | sed "s/\.xml//"` - export ROCP_OUTPUT_DIR=${input_tag}_results_${time_stamp} - elif [ "$OUTPUT_DIR" = "--" ] ; then - unset ROCP_OUTPUT_DIR - else - export ROCP_OUTPUT_DIR=$OUTPUT_DIR - fi - echo "RPL: result dir '$ROCP_OUTPUT_DIR'" - - if [ ! -e "$ROCP_INPUT" ] ; then - error "Input file '$ROCP_INPUT' not found" - fi - - if [ -n "$ROCP_OUTPUT_DIR" ] ; then - if [ "$OUTPUT_DIR" = "-" ] ; then - if [ -e "$ROCP_OUTPUT_DIR" ] ; then - error "generated dir '$ROCP_OUTPUT_DIR' exists" - fi - fi - mkdir -p "$ROCP_OUTPUT_DIR" - fi - - if [ "$HSA_TRACE" = 1 ] ; then - export ROCTRACER_DOMAIN="hsa" - export HSA_TOOLS_LIB="libtracer_tool.so libroctracer64.so $HSA_TOOLS_LIB" - fi - - redirection_cmd="" - if [ -n "$ROCP_OUTPUT_DIR" ] ; then - OUTPUT_LIST="$OUTPUT_LIST $ROCP_OUTPUT_DIR/results.txt" - redirection_cmd="2>&1 | tee $ROCP_OUTPUT_DIR/log.txt" - fi - - #unset ROCP_OUTPUT_DIR - eval "LD_PRELOAD='$HSA_TOOLS_LIB' $APP_CMD $redirection_cmd" -} - -# main -echo "RPL: on '$time_stamp' from '$PKG_DIR' in '$RUN_DIR'" -# Parsing arguments -if [ -z "$1" ] ; then - usage -fi - -INPUT_FILE="" -DATA_PATH="-" -OUTPUT_DIR="-" -output="" -csv_output="" - -ARG_IN="" -while [ 1 ] ; do - ARG_IN=$1 - ARG_VAL=1 - if [ "$1" = "-h" ] ; then - usage - elif [ "$1" = "-i" ] ; then - INPUT_FILE="$2" - elif [ "$1" = "-o" ] ; then - output="$2" - elif [ "$1" = "-d" ] ; then - DATA_PATH=$2 - elif [ "$1" = "-t" ] ; then - TMP_DIR="$2" - if [ "$OUTPUT_DIR" = "-" ] ; then - DATA_PATH=$TMP_DIR - fi - elif [ "$1" = "--list-basic" ] ; then - export ROCP_INFO=b - eval "$PKG_DIR/tool/ctrl" - exit 1 - elif [ "$1" = "--list-derived" ] ; then - export ROCP_INFO=d - eval "$PKG_DIR/tool/ctrl" - exit 1 - elif [ "$1" = "--basenames" ] ; then - if [ "$2" = "on" ] ; then - export ROCP_TRUNCATE_NAMES=1 - else - export ROCP_TRUNCATE_NAMES=0 - fi - elif [ "$1" = "--timestamp" ] ; then - if [ "$2" = "on" ] ; then - export ROCP_TIMESTAMP_ON=1 - else - export ROCP_TIMESTAMP_ON=0 - fi - elif [ "$1" = "--ctx-wait" ] ; then - if [ "$2" = "on" ] ; then - export ROCP_OUTSTANDING_WAIT=1 - else - export ROCP_OUTSTANDING_WAIT=0 - fi - elif [ "$1" = "--ctx-limit" ] ; then - export ROCP_OUTSTANDING_MAX="$2" - elif [ "$1" = "--heartbeat" ] ; then - export ROCP_OUTSTANDING_MON="$2" - elif [ "$1" = "--hsa-trace" ] ; then - ARG_VAL=0 - HSA_TRACE=1 - elif [ "$1" = "--hip-trace" ] ; then - ARG_VAL=0 - HIP_TRACE=1 - elif [ "$1" = "--verbose" ] ; then - ARG_VAL=0 - export ROCP_VERBOSE_MODE=1 - elif [ "$1" = "--stats" ] ; then - ARG_VAL=0 - GEN_STATS=1 - else - break - fi - shift - if [ "$ARG_VAL" = 1 ] ; then shift; fi -done - -ARG_CK=`echo $ARG_IN | sed "s/^-.*$/-/"` -if [ "$ARG_CK" = "-" ] ; then - fatal "Wrong option '$ARG_IN'" -fi - -if [ -z "$INPUT_FILE" ] ; then - input_base="results" - input_type="none" -else - input_base=`echo "$INPUT_FILE" | sed "s/^\(.*\)\.\([^\.]*\)$/\1/"` - input_type=`echo "$INPUT_FILE" | sed "s/^\(.*\)\.\([^\.]*\)$/\2/"` - if [ -z "${input_base}" -o -z "${input_type}" ] ; then - fatal "Bad input file '$INPUT_FILE'" - fi - input_base=`basename $input_base` -fi - -if [ "$DATA_PATH" = "-" ] ; then - DATA_PATH=$TMP_DIR -fi - -if [ -n "$output" ] ; then - if [ "$output" = "--" ] ; then - OUTPUT_DIR="--" - else - csv_output=$output - fi -else - csv_output=$RUN_DIR/${input_base}.csv -fi - -APP_CMD=$* - -echo "RPL: profiling '$APP_CMD'" -echo "RPL: input file '$INPUT_FILE'" - -input_list="" -RES_DIR="" -if [ "$input_type" = "xml" ] ; then - OUTPUT_DIR=$DATA_PATH - input_list=$INPUT_FILE -elif [ "$input_type" = "txt" -o "$input_type" = "none" ] ; then - RES_DIR=$DATA_PATH/$DATA_DIR - if [ -e $RES_DIR ] ; then - error "Rundir '$RES_DIR' exists" - fi - mkdir -p $RES_DIR - echo "RPL: output dir '$RES_DIR'" - if [ "$input_type" = "txt" ] ; then - $BIN_DIR/txt2xml.sh $INPUT_FILE $RES_DIR - else - echo "" > $RES_DIR/input.xml - fi - input_list=`/bin/ls $RES_DIR/input*.xml` - export ROCPROFILER_SESS=$RES_DIR -else - fatal "Bad input file type '$INPUT_FILE'" -fi - -if [ -n "$csv_output" ] ; then - rm -f $csv_output -fi - -for name in $input_list; do - run $name $OUTPUT_DIR $APP_CMD - if [ -n "$ROCPROFILER_SESS" -a -e "$ROCPROFILER_SESS/error" ] ; then - echo "Error found, profiling aborted." - csv_output="" - break - fi -done - -if [ -n "$csv_output" ] ; then - if [ "$GEN_STATS" = "1" ] ; then - db_output=$(echo $csv_output | sed "s/\.csv/.db/") - python $BIN_DIR/tblextr.py $db_output $OUTPUT_LIST - else - python $BIN_DIR/tblextr.py $csv_output $OUTPUT_LIST - fi - if [ "$?" -eq 0 ] ; then - echo "RPL: '$csv_output' is generated" - else - echo "Data extracting error: $OUTPUT_LIST'" - fi -fi - -if [ "$DATA_PATH" = "$TMP_DIR" ] ; then - if [ -e "$RES_DIR" ] ; then - rm -rf $RES_DIR - fi -fi - -exit 0 diff --git a/bin/sqlitedb.py b/bin/sqlitedb.py deleted file mode 100644 index bfcf12fae9..0000000000 --- a/bin/sqlitedb.py +++ /dev/null @@ -1,224 +0,0 @@ -import csv, sqlite3, re, sys - -# SQLite Database class -class SQLiteDB: - def __init__(self, file_name): - self.connection = sqlite3.connect(file_name) - self.tables = {} - self.json_arg_list_enabled = 0 - - def __del__(self): - self.connection.close() - - # add DB table - def add_table(self, name, descr, extra = ()): - (field_list, field_dict) = descr - if name in self.tables: raise Exception('table is already added: "' + name + '"') - - # create DB table - table_descr = [] - for field in field_list: table_descr.append('"%s" %s' % (field, field_dict[field])) - for item in extra: table_descr.append('"%s" %s' % (item[0], item[1])) - stm = 'CREATE TABLE ' + name + ' (%s)' % ', '.join(table_descr) - cursor = self.connection.cursor() - cursor.execute(stm) - self.connection.commit() - - # register table - fields_str = ','.join(map(lambda x: '"' + x + '"', field_list)) - templ_str = ','.join('?' * len(field_list)) - stm = 'INSERT INTO ' + name + '(' + fields_str + ') VALUES(' + templ_str + ');' - self.tables[name] = stm - - return (cursor, stm); - - # add columns to table - def add_columns(self, name, columns): - cursor = self.connection.cursor() - for item in columns: - stm = 'ALTER TABLE ' + name + ' ADD COLUMN "%s" %s' % (item[0], item[1]) - cursor.execute(stm) - self.connection.commit() - - # add columns with expression - def add_data_column(self, table_name, data_label, data_type, data_expr): - cursor = self.connection.cursor() - cursor.execute('ALTER TABLE %s ADD COLUMN "%s" %s' % (table_name, data_label, data_type)) - cursor.execute('UPDATE %s SET %s = (%s);' % (table_name, data_label, data_expr)) - - # populate DB table entry - def insert_entry(self, table, val_list): - (cursor, stm) = table - cursor.execute(stm, val_list) - - # populate DB table entry - def commit_entry(self, table, val_list): - self.insert_entry(table, val_list) - self.connection.commit() - - # populate DB table data - def insert_table(self, table, reader): - for val_list in reader: - if not val_list[-1]: val_list.pop() - self.insert_entry(table, val_list) - self.connection.commit() - - # return table fields list - def _get_fields(self, table_name): - cursor = self.connection.execute('SELECT * FROM ' + table_name) - return list(map(lambda x: '"%s"' % (x[0]), cursor.description)) - - # return table raws list - def _get_raws(self, table_name): - cursor = self.connection.execute('SELECT * FROM ' + table_name) - return cursor.fetchall() - def _get_raws_indexed(self, table_name): - cursor = self.connection.execute('SELECT * FROM ' + table_name + ' order by "Index" asc;') - return cursor.fetchall() - def _get_raw_by_id(self, table_name, req_id): - cursor = self.connection.execute('SELECT * FROM ' + table_name + ' WHERE "Index"=?', (req_id,)) - raws = cursor.fetchall() - if len(raws) != 1: - raise Exception('Index is not unique, table "' + table_name + '"') - return list(raws[0]) - - # dump CSV table - def dump_csv(self, table_name, file_name): - if not re.search(r'\.csv$', file_name): - raise Exception('wrong output file type: "' + file_name + '"' ) - - fields = self._get_fields(table_name) - with open(file_name, mode='w') as fd: - fd.write(','.join(fields) + '\n') - for raw in self._get_raws(table_name): - fd.write(reduce(lambda a, b: str(a) + ',' + str(b), raw) + '\n') - - # dump JSON trace - def open_json(self, file_name): - if not re.search(r'\.json$', file_name): - raise Exception('wrong output file type: "' + file_name + '"' ) - with open(file_name, mode='w') as fd: - fd.write('{ "traceEvents":[{}\n'); - - def close_json(self, file_name): - if not re.search(r'\.json$', file_name): - raise Exception('wrong output file type: "' + file_name + '"' ) - with open(file_name, mode='a') as fd: - fd.write(']}\n'); - - def label_json(self, pid, label, file_name): - if not re.search(r'\.json$', file_name): - raise Exception('wrong output file type: "' + file_name + '"' ) - with open(file_name, mode='a') as fd: - fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name"}\n' %(label, pid)); - - def flow_json(self, base_id, from_pid, from_tid, from_us_list, to_pid, to_us_dict, start_us, file_name): - if not re.search(r'\.json$', file_name): - raise Exception('wrong output file type: "' + file_name + '"' ) - with open(file_name, mode='a') as fd: - dep_id = base_id - for ind in range(len(from_tid)): - from_ts = from_us_list[ind] - start_us - to_ts = to_us_dict[ind] - start_us - fd.write(',{"ts":%d,"ph":"s","cat":"DataFlow","id":%d,"pid":%s,"tid":%d,"name":"dep"}\n' % (from_ts, dep_id, str(from_pid), from_tid[ind])) - fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%s,"tid":0,"name":"dep"}\n' % (to_ts, dep_id, str(to_pid))) - dep_id += 1 - - def dump_json(self, table_name, data_name, file_name): - if not re.search(r'\.json$', file_name): - raise Exception('wrong output file type: "' + file_name + '"' ) - - sub_ptrn = re.compile(r'(^"|"$)') - name_ptrn = re.compile(r'(name|Name)') - - table_fields = self._get_fields(table_name) - table_raws = self._get_raws_indexed(table_name) - data_fields = self._get_fields(data_name) - data_raws = self._get_raws_indexed(data_name) - - with open(file_name, mode='a') as fd: - table_raws_len = len(table_raws) - for raw_index in range(table_raws_len): - if (raw_index == table_raws_len - 1) or (raw_index % 1000 == 0): - sys.stdout.write( \ - "\rdump json " + str(raw_index) + ":" + str(len(table_raws)) + " "*100 \ - ) - - vals_list = [] - values = list(table_raws[raw_index]) - for value_index in range(len(values)): - label = table_fields[value_index] - value = values[value_index] - if name_ptrn.search(label): value = sub_ptrn.sub(r'', value) - if label != '"Index"': vals_list.append('%s:"%s"' % (label, value)) - - args_list = [] - data = list(data_raws[raw_index]) - for value_index in range(len(data)): - label = data_fields[value_index] - value = data[value_index] - if name_ptrn.search(label): value = sub_ptrn.sub(r'', value) - if label != '"Index"': args_list.append('%s:"%s"' % (label, value)) - - fd.write(',{"ph":"%s",%s,\n "args":{\n %s\n }\n}\n' % ('X', ','.join(vals_list), ',\n '.join(args_list))) - - sys.stdout.write('\n') - - # execute query on DB - def execute(self, cmd): - cursor = self.connection.cursor() - cursor.execute(cmd) - - # commit DB - def commit(self): - self.connection.commit() - - # close DB - def close(self): - self.connection.close() - - # access DB - def get_raws(self, table_name): - cur = self.connection.cursor() - cur.execute("SELECT * FROM %s" % table_name) - return cur.fetchall() - - # return CSV descriptor - # list of fields and dictionaly for the fields types - def _get_csv_descr(self, table_name, fd): - reader = csv.DictReader(fd) - field_names = reader.fieldnames - if not field_names[-1]: field_names.pop() - field_types = {} - - for entry in reader: - fields_left = [f for f in field_names if f not in field_types.keys()] - # all fields processed - if not fields_left: break - - for field in fields_left: - data = entry[field] - # need data for the field to be processed - if len(data) == 0: continue - - if data.isdigit(): - field_types[field] = "INTEGER" - else: - field_types[field] = "TEXT" - - if len(fields_left) > 0: raise Exception('types not found for fields: ', fields_left) - return (field_names, field_types) - - # add CSV table - def add_csv_table(self, table_name, file_name, extra = ()): - with open(file_name, mode='r') as fd: - # get CSV table descriptor - descr = self._get_csv_descr(table_name, fd) - # reader to populate the table - fd.seek(0) - reader = csv.reader(fd) - reader.next() - table = self.add_table(table_name, descr, extra) - self.insert_table(table, reader) - -############################################################################################## diff --git a/bin/tblextr.py b/bin/tblextr.py deleted file mode 100644 index aff6d6a880..0000000000 --- a/bin/tblextr.py +++ /dev/null @@ -1,339 +0,0 @@ -#!/usr/bin/python - -################################################################################ -# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -################################################################################ - -import os, sys, re -from sqlitedb import SQLiteDB -import dform - -# Parsing results in the format: -#dispatch[0], queue_index(0), kernel_name("SimpleConvolution"), time(1048928000311041,1048928006154674,1048928006168274,1048928006170503): -# GRBM_GUI_ACTIVE (74332) -# SQ_WAVES (4096) -# SQ_INSTS_VMEM_RD (36864) - -COPY_PID = 0 -HSA_PID = 1 -GPU_BASE_PID = 2 -max_gpu_id = 0 -START_US = 0 - -# dependencies dictionary -dep_dict = {} -kern_dep_list = [] - -# global vars -table_descr = [ - ['Index', 'KernelName'], - {'Index': 'INTEGER', 'KernelName': 'TEXT'} -] -var_list = table_descr[0] -var_table = {} -############################################################# - -def fatal(msg): - sys.stderr.write(sys.argv[0] + ": " + msg + "\n"); - sys.exit(1) -############################################################# - -# parse results method -def parse_res(infile): - global max_gpu_id - if not os.path.isfile(infile): fatal("Error: input file '" + infile + "' not found") - inp = open(infile, 'r') - - beg_pattern = re.compile("^dispatch\[(\d*)\], (.*) kernel-name\(\"([^\"]*)\"\)") - prop_pattern = re.compile("([\w-]+)\((\w+)\)"); - ts_pattern = re.compile(", time\((\d*),(\d*),(\d*),(\d*)\)") - var_pattern = re.compile("^\s*([^\s]*)\s+\((\d*)\)") - - dispatch_number = 0 - for line in inp.readlines(): - record = line[:-1] - - m = var_pattern.match(record) - if m: - if not dispatch_number in var_table: fatal("Error: dispatch number not found '" + str(dispatch_number) + "'") - var = m.group(1) - val = m.group(2) - var_table[dispatch_number][m.group(1)] = m.group(2) - if not var in var_list: var_list.append(var) - - m = beg_pattern.match(record) - if m: - dispatch_number = m.group(1) - if not dispatch_number in var_table: - var_table[dispatch_number] = { - 'Index': dispatch_number, - 'KernelName': "\"" + m.group(3) + "\"" - } - - gpu_id = 0 - disp_tid = 0 - - kernel_properties = m.group(2) - for prop in kernel_properties.split(', '): - m = prop_pattern.match(prop) - if m: - var = m.group(1) - val = m.group(2) - var_table[dispatch_number][var] = val - if not var in var_list: var_list.append(var); - if var == 'gpu-id': - if (val > max_gpu_id): max_gpu_id = val - gpu_id = val - if var == 'tid': disp_tid = int(val) - else: fatal('wrong kernel property "' + prop + '" in "'+ kernel_properties + '"') - m = ts_pattern.search(record) - if m: - var_table[dispatch_number]['DispatchNs'] = m.group(1) - var_table[dispatch_number]['BeginNs'] = m.group(2) - var_table[dispatch_number]['EndNs'] = m.group(3) - var_table[dispatch_number]['CompleteNs'] = m.group(4) - - gpu_pid = GPU_BASE_PID + int(gpu_id) - if not gpu_pid in dep_dict: dep_dict[gpu_pid] = {} - dep_str = dep_dict[gpu_pid] - if not 'tid' in dep_str: dep_str['tid'] = [] - if not 'from' in dep_str: dep_str['from'] = [] - if not 'to' in dep_str: dep_str['to'] = {} - to_id = len(dep_str['tid']) - from_us = int(m.group(1)) / 1000 - to_us = int(m.group(2)) / 1000 - dep_str['to'][to_id] = to_us - dep_str['from'].append(from_us) - dep_str['tid'].append(disp_tid) - kern_dep_list.append((disp_tid, m.group(1))) - - inp.close() -############################################################# - -# merge results table -def merge_table(): - global var_list - keys = sorted(var_table.keys(), key=int) - - fields = set(var_table[keys[0]]) - if 'DispatchNs' in fields: - var_list.append('DispatchNs') - var_list.append('BeginNs') - var_list.append('EndNs') - var_list.append('CompleteNs') - var_list = [x for x in var_list if x in fields] -############################################################# - -# dump CSV results -def dump_csv(file_name): - global var_list - keys = sorted(var_table.keys(), key=int) - - with open(file_name, mode='w') as fd: - fd.write(','.join(var_list) + '\n'); - for ind in keys: - entry = var_table[ind] - dispatch_number = entry['Index'] - if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n") - val_list = [entry[var] for var in var_list] - fd.write(','.join(val_list) + '\n'); -############################################################# - -# fill kernels DB -def fill_kernel_db(table_name, db): - global var_list - keys = sorted(var_table.keys(), key=int) - - for var in set(var_list).difference(set(table_descr[1])): - table_descr[1][var] = 'INTEGER' - table_descr[0] = var_list; - - table_handle = db.add_table(table_name, table_descr) - - for ind in keys: - entry = var_table[ind] - dispatch_number = entry['Index'] - if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n") - val_list = [entry[var] for var in var_list] - db.insert_entry(table_handle, val_list) -############################################################# - -# fill HSA DB -hsa_table_descr = [ - ['BeginNs', 'EndNs', 'pid', 'tid', 'Name', 'args', 'Index'], - {'Index':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'} -] -def fill_hsa_db(table_name, db, indir): - file_name = indir + '/' + 'hsa_api_trace.txt' - ptrn_val = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$') - ptrn_ac = re.compile(r'hsa_amd_memory_async_copy') - - if not os.path.isfile(file_name): return 0 - - if not COPY_PID in dep_dict: dep_dict[COPY_PID] = {} - dep_tid_list = [] - dep_from_us_list = [] - - global START_US - with open(file_name, mode='r') as fd: - line = fd.readline() - record = line[:-1] - m = ptrn_val.match(record) - if m: START_US = int(m.group(1)) / 1000 - START_US = 0 - - record_id = 0 - table_handle = db.add_table(table_name, hsa_table_descr) - with open(file_name, mode='r') as fd: - for line in fd.readlines(): - record = line[:-1] - m = ptrn_val.match(record) - if m: - rec_vals = [] - for ind in range(1,7): - rec_vals.append(m.group(ind)) - rec_vals[2] = HSA_PID - rec_vals.append(record_id) - db.insert_entry(table_handle, rec_vals) - if ptrn_ac.search(rec_vals[4]): - beg_ns = int(rec_vals[0]) - end_ns = int(rec_vals[1]) - from_us = (beg_ns / 1000) + ((end_ns - beg_ns) / 1000) - dep_from_us_list.append(from_us) - dep_tid_list.append(int(rec_vals[3])) - record_id += 1 - else: fatal("hsa bad record") - - for (tid, from_ns) in kern_dep_list: - db.insert_entry(table_handle, [from_ns, from_ns, HSA_PID, tid, 'hsa_dispatch', '', record_id]) - record_id += 1 - - dep_dict[COPY_PID]['tid'] = dep_tid_list - dep_dict[COPY_PID]['from'] = dep_from_us_list - - return 1 -############################################################# - -# fill COPY DB -copy_table_descr = [ - ['BeginNs', 'EndNs', 'Name', 'pid', 'tid', 'Index'], - {'Index':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'} -] -def fill_copy_db(table_name, db, indir): - file_name = indir + '/' + 'async_copy_trace.txt' - ptrn_val = re.compile(r'(\d+):(\d+) (.*)$') - ptrn_id = re.compile(r'^async-copy(\d+)$') - - if not COPY_PID in dep_dict: dep_dict[COPY_PID] = {} - dep_to_us_dict = {} - - table_handle = db.add_table(table_name, copy_table_descr) - with open(file_name, mode='r') as fd: - for line in fd.readlines(): - record = line[:-1] - m = ptrn_val.match(record) - if m: - rec_vals = [] - for ind in range(1,4): rec_vals.append(m.group(ind)) - rec_vals.append(COPY_PID) - rec_vals.append(0) - m = ptrn_id.match(rec_vals[2]) - if m: dep_to_us_dict[int(m.group(1))] = int(rec_vals[0]) / 1000 - else: fatal("async-copy bad name") - rec_vals.append(m.group(1)) - db.insert_entry(table_handle, rec_vals) - else: fatal("async-copy bad record") - - dep_dict[COPY_PID]['to'] = dep_to_us_dict -############################################################# -# main -if (len(sys.argv) < 3): fatal("Usage: " + sys.argv[0] + " ") - -outfile = sys.argv[1] -infiles = sys.argv[2:] -indir = re.sub(r'\/[^\/]*$', r'', infiles[0]) - -dbfile = '' -csvfile = '' - -if re.search(r'\.csv$', outfile): - csvfile = outfile -elif re.search(r'\.db$', outfile): - dbfile = outfile - csvfile = re.sub(r'\.db$', '.csv', outfile) -else: - fatal("Bad output file '" + outfile + "'") - -for f in infiles: parse_res(f) -if len(var_table) == 0: sys.exit(1) -merge_table() - -if dbfile == '': - dump_csv(csvfile) -else: - statfile = re.sub(r'\.csv$', '.stats.csv', csvfile) - jsonfile = re.sub(r'\.csv$', '.json', csvfile) - - with open(dbfile, mode='w') as fd: fd.truncate() - db = SQLiteDB(dbfile) - db.open_json(jsonfile); - - hsa_trace_found = fill_hsa_db('HSA', db, indir) - if hsa_trace_found: - fill_copy_db('COPY', db, indir) - fill_kernel_db('A', db) - - if hsa_trace_found: - db.label_json(HSA_PID, "CPU", jsonfile) - db.label_json(COPY_PID, "COPY", jsonfile) - - for ind in range(0, int(max_gpu_id) + 1): - db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile) - - if 'BeginNs' in var_list: - dform.post_process_data(db, 'A', csvfile) - dform.gen_table_bins(db, 'A', statfile, 'KernelName', 'DurationNs') - dform.gen_kernel_json_trace(db, 'A', GPU_BASE_PID, START_US, jsonfile) - else: - db.dump_csv('A', csvfile) - - if hsa_trace_found: - statfile = re.sub(r'stats', r'hsa_stats', statfile) - dform.post_process_data(db, 'HSA') - dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs') - dform.gen_api_json_trace(db, 'HSA', START_US, jsonfile) - - dform.post_process_data(db, 'COPY') - dform.gen_api_json_trace(db, 'COPY', START_US, jsonfile) - - dep_id = 0 - for (to_pid, dep_str) in dep_dict.items(): - tid_list = dep_str['tid'] - from_us_list = dep_str['from'] - to_us_dict = dep_str['to'] - db.flow_json(dep_id, HSA_PID, tid_list, from_us_list, to_pid, to_us_dict, START_US, jsonfile) - dep_id += len(tid_list) - - db.close_json(jsonfile); - db.close() - -sys.exit(0) -############################################################# diff --git a/bin/txt2xml.sh b/bin/txt2xml.sh deleted file mode 100755 index 27bbe8c474..0000000000 --- a/bin/txt2xml.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -################################################################################ -# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -################################################################################ - -timestamp=`date +%y%m%d_%H%M%S` - -if [ $# = 0 ] ; then - echo "Usage: $0 [output dir]" - exit -1 -fi - -input=$1 -outdir=$2 -if [ -z "$outdir" ] ; then - outdir="." -fi - -range="" -kernel="" -gpu_index="" - -parse() { - scan="$1" - index=0 - while read -r line || [[ -n "$line" ]] ; do - line=`echo $line | sed "s/\s*#.*$//"` - if [ -z "$line" ] ; then - continue - fi - - feature=`echo $line | sed -n "s/^\s*\([a-z]*\)\s*:.*$/\1/p"` - line=`echo $line | sed "s/^[^:]*:\s*//"` - line=`echo "$line" | sed -e "s/\s*=\s*/=/g" -e "s/\s*:\s*/:/g" -e "s/,\{1,\}/ /g" -e "s/\s\{1,\}/ /g" -e "s/\s*$//"` - - if [ "$scan" = 0 ] ; then - line=`echo "$line" | sed -e "s/ /,/g"` - if [ "$feature" == "range" ] ; then - range=$line - fi - if [ "$feature" == "kernel" ] ; then - kernel=$line - fi - if [ "$feature" == "gpu" ] ; then - gpu_index=$line - fi - else - output=$outdir/input${index}.xml - header="# $timestamp '$output' generated with '$0 $*'" - echo $header > $output - - if [ "$feature" == "pmc" ] ; then - line=`echo "$line" | sed -e "s/ /,/g"` - cat >> $output < - -EOF - fi - - if [ "$feature" == "sqtt" ] ; then - cat >> $output < - -EOF - fi - - if [ "$feature" == "hsa" ] ; then - cat >> $output < -EOF - fi - fi - - index=$((index + 1)) - done < $input -} - -parse 0 -parse 1 - -exit 0