tool rim scripts

[ROCm/roctracer commit: fda5f8b14e]

projects/roctracer/bin/dform.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python
+from sqlitedb import SQLiteDB
+
+def post_process_data(db, table_name, outfile = ''): 
+#  db.add_data_column('A', 'DispDurNs', 'INTEGER', 'BeginNs - DispatchNs')
+#  db.add_data_column('A', 'ComplDurNs', 'INTEGER', 'CompleteNs - EndNs')
+#  db.add_data_column('A', 'TotalDurNs', 'INTEGER', 'CompleteNs - DispatchNs')
+#  db.add_data_column('A', 'TimeNs', 'INTEGER', 'BeginNs - (select BeginNs from A limit 1)')
+  db.add_data_column(table_name, 'DurationNs', 'INTEGER', 'EndNs - BeginNs')
+  if outfile != '': db.dump_csv(table_name, outfile)
+
+def gen_data_bins(db, outfile):
+  db.execute('create view C as select Name, Calls, TotalDurationNs, TotalDurationNs/Calls as AverageNs, TotalDurationNs*100.0/(select sum(TotalDurationNs) from %s) as Percentage from %s order by TotalDurationNs desc;' % ('B', 'B'));
+  db.dump_csv('C', outfile)
+  db.execute('DROP VIEW C')
+
+def gen_table_bins(db, table, outfile, name_var, dur_ns_var):
+  db.execute('create view B as select (%s) as Name, count(%s) as Calls, sum(%s) as TotalDurationNs from %s group by %s' % (name_var, name_var, dur_ns_var, table, name_var))
+  gen_data_bins(db, outfile)
+  db.execute('DROP VIEW B')
+
+def gen_api_json_trace(db, table, outfile):
+  db.execute('create view B as select Name as name, (BeginNs/1000) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % table);
+  db.dump_json('B', table, outfile)
+  db.execute('DROP VIEW B')
+
+def gen_kernel_json_trace(db, table, outfile):
+  db.execute('create view B as select (KernelName) as name, ("gpu-id") as pid, (0) as tid, (BeginNs/1000) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % table);
+  db.dump_json('B', table, outfile)
+  db.execute('DROP VIEW B')
+##############################################################################################

projects/roctracer/bin/rpl_run.sh

@@ -0,0 +1,379 @@
+#!/bin/sh
+
+################################################################################
+# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+################################################################################
+
+time_stamp=`date +%y%m%d_%H%M%S`
+BIN_DIR=`dirname $0`
+BIN_DIR=`cd $BIN_DIR; pwd`
+RUN_DIR=`pwd`
+TMP_DIR="/tmp"
+DATA_DIR="rpl_data_${time_stamp}_$$"
+
+PKG_DIR=`echo $BIN_DIR | sed "s/\/bin\/*//"`
+BIN_DIR=$PKG_DIR/bin
+
+# PATH to custom HSA and OpenCl runtimes
+HSA_PATH=$PKG_DIR/lib/hsa
+
+# runtime API trace
+HSA_TRACE=0
+HIP_TRACE=0
+
+# Generate stats
+GEN_STATS=0
+
+export LD_LIBRARY_PATH=$PKG_DIR/lib:$PKG_DIR/tool:$PKG_DIR/roctracer/lib:$PKG_DIR/roctracer/tool:$HSA_PATH
+export PATH=.:$PATH
+
+# enable error logging
+export HSA_TOOLS_REPORT_LOAD_FAILURE=1
+export HSA_VEN_AMD_AQLPROFILE_LOG=1
+export ROCPROFILER_LOG=1
+unset ROCPROFILER_SESS
+
+# ROC Profiler environment
+# Loading of ROC Profiler by HSA runtime
+export HSA_TOOLS_LIB=librocprofiler64.so
+# Loading of the test tool by ROC Profiler
+export ROCP_TOOL_LIB=libtool.so
+# Enabling HSA dispatches intercepting by ROC PRofiler
+export ROCP_HSA_INTERCEPT=1
+# Disabling internal ROC Profiler proxy queue (simple version supported for testing purposes)
+unset ROCP_PROXY_QUEUE
+# ROC Profiler metrics definition
+export ROCP_METRICS=$PKG_DIR/lib/metrics.xml
+# Disable AQL-profile read API
+export AQLPROFILE_READ_API=0
+# ROC Profiler package path
+export ROCP_PACKAGE_DIR=$PKG_DIR
+
+# error handling
+fatal() {
+  echo "$0: Error: $1"
+  echo ""
+  usage
+}
+
+error() {
+  echo "$0: Error: $1"
+  echo ""
+  exit 1
+}
+
+# usage method
+usage() {
+  bin_name=`basename $0`
+  echo "ROCm Profiling Library (RPL) run script, a part of ROCprofiler library package."
+  echo "Full path: $BIN_DIR/$bin_name"
+  echo "Metrics definition: $PKG_DIR/lib/metrics.xml"
+  echo ""
+  echo "Usage:"
+  echo "  $bin_name [-h] [--list-basic] [--list-derived] [-i <input .txt/.xml file>] [-o <output CSV file>] <app command line>"
+  echo ""
+  echo "Options:"
+  echo "  -h - this help"
+  echo "  --verbose - verbose mode, dumping all base counters used in the input metrics"
+  echo "  --list-basic - to print the list of basic HW counters"
+  echo "  --list-derived - to print the list of derived metrics with formulas"
+  echo ""
+  echo "  -i <.txt|.xml file> - input file"
+  echo "      Input file .txt format, automatically rerun application for every pmc line:"
+  echo ""
+  echo "        # Perf counters group 1"
+  echo "        pmc : Wavefronts VALUInsts SALUInsts SFetchInsts FlatVMemInsts LDSInsts FlatLDSInsts GDSInsts VALUUtilization FetchSize"
+  echo "        # Perf counters group 2"
+  echo "        pmc : WriteSize L2CacheHit"
+  echo "        # Filter by dispatches range, GPU index and kernel names"
+  echo "        # supported range formats: \"3:9\", \"3:\", \"3\""
+  echo "        range: 1 : 4"
+  echo "        gpu: 0 1 2 3"
+  echo "        kernel: simple Pass1 simpleConvolutionPass2"
+  echo ""
+  echo "      Input file .xml format, for single profiling run:"
+  echo ""
+  echo "        # Metrics list definition, also the form \"<block-name>:<event-id>\" can be used"
+  echo "        # All defined metrics can be found in the 'metrics.xml'"
+  echo "        # There are basic metrics for raw HW counters and high-level metrics for derived counters"
+  echo "        <metric name=SQ:4,SQ_WAVES,VFetchInsts"
+  echo "        ></metric>"
+  echo ""
+  echo "        # Filter by dispatches range, GPU index and kernel names"
+  echo "        <metric"
+  echo "          # range formats: \"3:9\", \"3:\", \"3\""
+  echo "          range=\"\""
+  echo "          # list of gpu indexes \"0,1,2,3\""
+  echo "          gpu_index=\"\""
+  echo "          # list of matched sub-strings \"Simple1,Conv1,SimpleConvolution\""
+  echo "          kernel=\"\""
+  echo "        ></metric>"
+  echo ""
+  echo "  -o <output file> - output CSV file [<input file base>.csv]"
+  echo "  -d <data directory> - directory where profiler store profiling data including thread treaces [/tmp]"
+  echo "      The data directory is renoving autonatically if the directory is matching the temporary one, which is the default."
+  echo "  -t <temporary directory> - to change the temporary directory [/tmp]"
+  echo "      By changing the temporary directory you can prevent removing the profiling data from /tmp or enable removing from not '/tmp' directory."
+  echo ""
+  echo "  --basenames <on|off> - to turn on/off truncating of the kernel full function names till the base ones [off]"
+  echo "  --timestamp <on|off> - to turn on/off the kernel disoatches timestamps, dispatch/begin/end/complete [off]"
+  echo "  --ctx-wait <on|off> - to wait for outstanding contexts on profiler exit [on]"
+  echo "  --ctx-limit <max number> - maximum number of outstanding contexts [0 - unlimited]"
+  echo "  --heartbeat <rate sec> - to print progress heartbeats [0 - disabled]"
+  echo ""
+  echo "  --stats - generating stats and json trace output"
+  echo "  --hsa-trace - to trace HSA"
+  echo "  --hip-trace - to trace HIP"
+  echo ""
+  echo "Configuration file:"
+  echo "  You can set your parameters defaults preferences in the configuration file 'rpl_rc.xml'. The search path sequence: .:${HOME}:<package path>"
+  echo "  First the configuration file is looking in the current directory, then in your home, and then in the package directory."
+  echo "  Configurable options: 'basenames', 'timestamp', 'ctx-limit', 'heartbeat'."
+  echo "  An example of 'rpl_rc.xml':"
+  echo "    <defaults"
+  echo "      basenames=off"
+  echo "      timestamp=off"
+  echo "      ctx-limit=0"
+  echo "      heartbeat=0"
+  echo "    ></defaults>"
+  echo ""
+  exit 1
+}
+
+# profiling run method
+OUTPUT_LIST=""
+run() {
+  export ROCP_INPUT="$1"
+  OUTPUT_DIR="$2"
+  shift
+  shift
+  APP_CMD=$*
+
+  if [ "$OUTPUT_DIR" = "-" ] ; then
+    input_tag=`echo $ROCP_INPUT | sed "s/\.xml//"`
+    export ROCP_OUTPUT_DIR=${input_tag}_results_${time_stamp}
+  elif [ "$OUTPUT_DIR" = "--" ] ; then
+    unset ROCP_OUTPUT_DIR
+  else
+    export ROCP_OUTPUT_DIR=$OUTPUT_DIR
+  fi
+  echo "RPL: result dir '$ROCP_OUTPUT_DIR'"
+
+  if [ ! -e "$ROCP_INPUT" ] ; then
+    error "Input file '$ROCP_INPUT' not found"
+  fi
+
+  if [ -n "$ROCP_OUTPUT_DIR" ] ; then
+    if [ "$OUTPUT_DIR" = "-" ] ; then
+      if [ -e "$ROCP_OUTPUT_DIR" ] ; then
+        error "generated dir '$ROCP_OUTPUT_DIR' exists"
+      fi
+    fi
+    mkdir -p "$ROCP_OUTPUT_DIR"
+  fi
+
+  if [ "$HSA_TRACE" = 1 ] ; then
+    export ROCTRACER_DOMAIN="hsa"
+    export HSA_TOOLS_LIB="libtracer_tool.so libroctracer64.so $HSA_TOOLS_LIB"
+  fi
+
+  redirection_cmd=""
+  if [ -n "$ROCP_OUTPUT_DIR" ] ; then
+    OUTPUT_LIST="$OUTPUT_LIST $ROCP_OUTPUT_DIR/results.txt"
+    redirection_cmd="2>&1 | tee $ROCP_OUTPUT_DIR/log.txt"
+  fi
+
+  #unset ROCP_OUTPUT_DIR
+  eval "LD_PRELOAD='$HSA_TOOLS_LIB' $APP_CMD $redirection_cmd"
+}
+
+# main
+echo "RPL: on '$time_stamp' from '$PKG_DIR' in '$RUN_DIR'"
+# Parsing arguments
+if [ -z "$1" ] ; then
+  usage
+fi
+
+INPUT_FILE=""
+DATA_PATH="-"
+OUTPUT_DIR="-"
+output=""
+csv_output=""
+
+ARG_IN=""
+while [ 1 ] ; do
+  ARG_IN=$1
+  ARG_VAL=1
+  if [ "$1" = "-h" ] ; then
+    usage
+  elif [ "$1" = "-i" ] ; then
+    INPUT_FILE="$2"
+  elif [ "$1" = "-o" ] ; then
+    output="$2"
+  elif [ "$1" = "-d" ] ; then
+    DATA_PATH=$2
+  elif [ "$1" = "-t" ] ; then
+    TMP_DIR="$2"
+    if [ "$OUTPUT_DIR" = "-" ] ; then
+      DATA_PATH=$TMP_DIR
+    fi
+  elif [ "$1" = "--list-basic" ] ; then
+    export ROCP_INFO=b
+    eval "$PKG_DIR/tool/ctrl"
+    exit 1
+  elif [ "$1" = "--list-derived" ] ; then
+    export ROCP_INFO=d
+    eval "$PKG_DIR/tool/ctrl"
+    exit 1
+  elif [ "$1" = "--basenames" ] ; then
+    if [ "$2" = "on" ] ; then
+      export ROCP_TRUNCATE_NAMES=1
+    else
+      export ROCP_TRUNCATE_NAMES=0
+    fi
+  elif [ "$1" = "--timestamp" ] ; then
+    if [ "$2" = "on" ] ; then
+      export ROCP_TIMESTAMP_ON=1
+    else
+      export ROCP_TIMESTAMP_ON=0
+    fi
+  elif [ "$1" = "--ctx-wait" ] ; then
+    if [ "$2" = "on" ] ; then
+      export ROCP_OUTSTANDING_WAIT=1
+    else
+      export ROCP_OUTSTANDING_WAIT=0
+    fi
+  elif [ "$1" = "--ctx-limit" ] ; then
+    export ROCP_OUTSTANDING_MAX="$2"
+  elif [ "$1" = "--heartbeat" ] ; then
+    export ROCP_OUTSTANDING_MON="$2"
+  elif [ "$1" = "--hsa-trace" ] ; then
+    ARG_VAL=0
+    HSA_TRACE=1
+  elif [ "$1" = "--hip-trace" ] ; then
+    ARG_VAL=0
+    HIP_TRACE=1
+  elif [ "$1" = "--verbose" ] ; then
+    ARG_VAL=0
+    export ROCP_VERBOSE_MODE=1
+  elif [ "$1" = "--stats" ] ; then
+    ARG_VAL=0
+    GEN_STATS=1
+  else
+    break
+  fi
+  shift
+  if [ "$ARG_VAL" = 1 ] ; then shift; fi
+done
+
+ARG_CK=`echo $ARG_IN | sed "s/^-.*$/-/"`
+if [ "$ARG_CK" = "-" ] ; then
+  fatal "Wrong option '$ARG_IN'"
+fi
+
+if [ -z "$INPUT_FILE" ] ; then
+  input_base="results"
+  input_type="none"
+else
+  input_base=`echo "$INPUT_FILE" | sed "s/^\(.*\)\.\([^\.]*\)$/\1/"`
+  input_type=`echo "$INPUT_FILE" | sed "s/^\(.*\)\.\([^\.]*\)$/\2/"`
+  if [ -z "${input_base}" -o -z "${input_type}" ] ; then
+    fatal "Bad input file '$INPUT_FILE'"
+  fi
+  input_base=`basename $input_base`
+fi
+
+if [ "$DATA_PATH" = "-" ] ; then
+  DATA_PATH=$TMP_DIR
+fi
+
+if [ -n "$output" ] ; then
+  if [ "$output" = "--" ] ; then
+    OUTPUT_DIR="--"
+  else
+    csv_output=$output
+  fi
+else
+  csv_output=$RUN_DIR/${input_base}.csv
+fi
+
+APP_CMD=$*
+
+echo "RPL: profiling '$APP_CMD'"
+echo "RPL: input file '$INPUT_FILE'"
+
+input_list=""
+RES_DIR=""
+if [ "$input_type" = "xml" ] ; then
+  OUTPUT_DIR=$DATA_PATH
+  input_list=$INPUT_FILE
+elif [ "$input_type" = "txt" -o "$input_type" = "none" ] ; then
+  RES_DIR=$DATA_PATH/$DATA_DIR
+  if [ -e $RES_DIR ] ; then
+    error "Rundir '$RES_DIR' exists"
+  fi
+  mkdir -p $RES_DIR
+  echo "RPL: output dir '$RES_DIR'"
+  if [ "$input_type" = "txt" ] ; then
+    $BIN_DIR/txt2xml.sh $INPUT_FILE $RES_DIR
+  else
+    echo "<metric></metric>" > $RES_DIR/input.xml
+  fi
+  input_list=`/bin/ls $RES_DIR/input*.xml`
+  export ROCPROFILER_SESS=$RES_DIR
+else
+  fatal "Bad input file type '$INPUT_FILE'"
+fi
+
+if [ -n "$csv_output" ] ; then
+  rm -f $csv_output
+fi
+
+for name in $input_list; do
+  run $name $OUTPUT_DIR $APP_CMD
+  if [ -n "$ROCPROFILER_SESS" -a -e "$ROCPROFILER_SESS/error" ] ; then
+    echo "Error found, profiling aborted."
+    csv_output=""
+    break
+  fi
+done
+
+if [ -n "$csv_output" ] ; then
+  if [ "$GEN_STATS" = "1" ] ; then
+    db_output=$(echo $csv_output | sed "s/\.csv/.db/")
+    python $BIN_DIR/tblextr.py $db_output $OUTPUT_LIST
+  else
+    python $BIN_DIR/tblextr.py $csv_output $OUTPUT_LIST
+  fi
+  if [ "$?" -eq 0 ] ; then
+    echo "RPL: '$csv_output' is generated"
+  else
+    echo "Data extracting error: $OUTPUT_LIST'"
+  fi
+fi
+
+if [ "$DATA_PATH" = "$TMP_DIR" ] ; then
+  if [ -e "$RES_DIR" ] ; then
+    rm -rf $RES_DIR
+  fi
+fi
+
+exit 0

projects/roctracer/bin/sqlitedb.py

@@ -0,0 +1,196 @@
+import csv, sqlite3, re
+
+# SQLite Database class
+class SQLiteDB:
+  def __init__(self, file_name):
+    self.connection = sqlite3.connect(file_name)
+    self.tables = {}
+
+  def __del__(self):
+    self.connection.close()
+
+  # add DB table
+  def add_table(self, name, descr, extra = ()):
+    (field_list, field_dict) = descr
+    if name in self.tables: raise Exception('table is already added: "' + name + '"')
+
+    # create DB table
+    table_descr = []
+    for field in field_list: table_descr.append('"%s" %s' % (field, field_dict[field]))
+    for item in extra: table_descr.append('"%s" %s' % (item[0], item[1]))
+    stm = 'CREATE TABLE ' + name + ' (%s)' % ', '.join(table_descr)
+    cursor = self.connection.cursor()
+    cursor.execute(stm)
+    self.connection.commit()
+
+    # register table
+    fields_str = ','.join(map(lambda x: '"' + x + '"', field_list))
+    templ_str = ','.join('?' * len(field_list))
+    stm = 'INSERT INTO ' + name + '(' + fields_str + ') VALUES(' + templ_str + ');'
+    self.tables[name] = stm
+
+    return (cursor, stm);
+
+  # add columns to table
+  def add_columns(self, name, columns):
+    cursor = self.connection.cursor()
+    for item in columns:
+      stm = 'ALTER TABLE ' + name + ' ADD COLUMN "%s" %s' % (item[0], item[1])
+      cursor.execute(stm)
+    self.connection.commit()
+
+  # add columns with expression
+  def add_data_column(self, table_name, data_label, data_type, data_expr):
+    cursor = self.connection.cursor()
+    cursor.execute('ALTER TABLE %s ADD COLUMN "%s" %s' % (table_name, data_label, data_type))
+    cursor.execute('UPDATE %s SET %s = (%s);' % (table_name, data_label, data_expr))
+
+  # populate DB table entry
+  def insert_entry(self, table, val_list):
+    (cursor, stm) = table
+    cursor.execute(stm, val_list)
+
+  # populate DB table entry
+  def commit_entry(self, table, val_list):
+    self.insert_entry(table, val_list)
+    self.connection.commit()
+
+  # populate DB table data
+  def insert_table(self, table, reader):
+    for val_list in reader:
+      if not val_list[-1]: val_list.pop()
+      self.insert_entry(table, val_list)
+    self.connection.commit()
+
+  # return table fields list
+  def _get_fields(self, table_name):
+    cursor = self.connection.execute('SELECT * FROM ' + table_name)
+    return list(map(lambda x: '"%s"' % (x[0]), cursor.description))
+
+  # return table raws list
+  def _get_raws(self, table_name):
+    cursor = self.connection.execute('SELECT * FROM ' + table_name)
+    return cursor.fetchall()
+
+  # dump CSV table
+  def dump_csv(self, table_name, file_name):
+    if not re.search(r'\.csv$', file_name):
+      raise Exception('wrong output file type: "' + file_name + '"' )
+
+    fields = self._get_fields(table_name)
+    stm = 'select ' + ','.join(fields) + ' from ' + table_name
+
+    with open(file_name, mode='w') as fd:
+      fd.write(','.join(fields) + '\n')
+      for values in self.connection.execute(stm):
+        fd.write(reduce(lambda a, b: str(a) + ',' + str(b), values) + '\n')
+
+  # dump JSON trace
+  def open_json(self, file_name):
+    if not re.search(r'\.json$', file_name):
+      raise Exception('wrong output file type: "' + file_name + '"' )
+    with open(file_name, mode='w') as fd:
+      fd.write('{ "traceEvents":[{}\n');
+  def close_json(self, file_name):
+    if not re.search(r'\.json$', file_name):
+      raise Exception('wrong output file type: "' + file_name + '"' )
+    with open(file_name, mode='a') as fd:
+      fd.write(']}\n');
+
+  def label_json(self, pid, label, file_name):
+    if not re.search(r'\.json$', file_name):
+      raise Exception('wrong output file type: "' + file_name + '"' )
+    with open(file_name, mode='a') as fd:
+      fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name"}\n' %(label, pid));
+
+  def dump_json(self, table_name, data_name, file_name):
+    if not re.search(r'\.json$', file_name):
+      raise Exception('wrong output file type: "' + file_name + '"' )
+
+    sub_ptrn = re.compile(r'(^"|"$)')
+    name_ptrn = re.compile(r'(name|Name)')
+
+    table_fields = self._get_fields(table_name)
+    table_raws = self._get_raws(table_name)
+    data_fields = self._get_fields(data_name)
+    data_raws = self._get_raws(data_name)
+
+    with open(file_name, mode='a') as fd:
+      for raw_index in range(len(table_raws)):
+        values = list(table_raws[raw_index])
+        data = list(data_raws[raw_index])
+
+        vals_list = []
+        for value_index in range(len(values)):
+          label = table_fields[value_index]
+          value = values[value_index]
+          if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
+          vals_list.append('%s:"%s"' % (label, value))
+
+        args_list = []
+        for value_index in range(len(data)):
+          label = data_fields[value_index]
+          value = data[value_index]
+          if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
+          args_list.append('%s:"%s"' % (label, value))
+
+        fd.write(',{"ph":"%s",%s,\n  "args":{\n    %s\n  }\n}\n' % ('X', ','.join(vals_list), ',\n    '.join(args_list)))
+
+  # execute query on DB
+  def execute(self, cmd):
+    cursor = self.connection.cursor()
+    cursor.execute(cmd)
+
+  # commit DB
+  def commit(self):
+    self.connection.commit()
+
+  # close DB
+  def close(self):
+    self.connection.close()
+
+  # access DB
+  def get_raws(self, table_name):
+    cur = self.connection.cursor()
+    cur.execute("SELECT * FROM %s" % table_name)
+    return cur.fetchall()
+
+  # return CSV descriptor
+  # list of fields and dictionaly for the fields types
+  def _get_csv_descr(self, table_name, fd):
+    reader = csv.DictReader(fd)
+    field_names = reader.fieldnames
+    if not field_names[-1]: field_names.pop()
+    field_types = {}
+
+    for entry in reader:
+      fields_left = [f for f in field_names if f not in field_types.keys()]
+      # all fields processed
+      if not fields_left: break
+
+      for field in fields_left:
+          data = entry[field]
+          # need data for the field to be processed
+          if len(data) == 0: continue
+
+          if data.isdigit():
+              field_types[field] = "INTEGER"
+          else:
+              field_types[field] = "TEXT"
+
+    if len(fields_left) > 0: raise Exception('types not found for fields: ', fields_left)
+    return (field_names, field_types)
+
+  # add CSV table
+  def add_csv_table(self, table_name, file_name, extra = ()):
+    with open(file_name, mode='r') as fd:
+      # get CSV table descriptor
+      descr = self._get_csv_descr(table_name, fd)
+      # reader to populate the table
+      fd.seek(0)
+      reader = csv.reader(fd)
+      reader.next()
+      table = self.add_table(table_name, descr, extra)
+      self.insert_table(table, reader)
+
+##############################################################################################

projects/roctracer/bin/tblextr.py

@@ -0,0 +1,259 @@
+#!/usr/bin/python
+
+################################################################################
+# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+################################################################################
+
+import os, sys, re
+from sqlitedb import SQLiteDB
+import dform
+
+# Parsing results in the format:
+#dispatch[0], queue_index(0), kernel_name("SimpleConvolution"), time(1048928000311041,1048928006154674,1048928006168274,1048928006170503):
+#  GRBM_GUI_ACTIVE (74332)
+#  SQ_WAVES (4096)
+#  SQ_INSTS_VMEM_RD (36864)
+
+COPY_PID = 0
+HSA_PID = 1
+GPU_BASE_PID = 2
+max_gpu_id = 0
+
+# global vars
+table_descr = [
+  ['Index', 'KernelName'],
+  {'Index': 'INTEGER', 'KernelName': 'TEXT'}
+]
+var_list = table_descr[0]
+var_table = {}
+#############################################################
+
+def fatal(msg):
+  sys.stderr.write(sys.argv[0] + ": " + msg + "\n");
+  sys.exit(1)
+#############################################################
+
+# parse results method
+def parse_res(infile):
+  global max_gpu_id
+  if not os.path.isfile(infile): fatal("Error: input file '" + infile + "' not found")
+  inp = open(infile, 'r')
+
+  beg_pattern = re.compile("^dispatch\[(\d*)\], (.*) kernel-name\(\"([^\"]*)\"\)")
+  prop_pattern = re.compile("([\w-]+)\((\w+)\)");
+  ts_pattern = re.compile(", time\((\d*),(\d*),(\d*),(\d*)\)")
+  var_pattern = re.compile("^\s*([^\s]*)\s+\((\d*)\)")
+
+  dispatch_number = 0
+  for line in inp.readlines():
+    record = line[:-1]
+
+    m = var_pattern.match(record)
+    if m:
+      if not dispatch_number in var_table: fatal("Error: dispatch number not found '" + str(dispatch_number) + "'")
+      var = m.group(1)
+      val = m.group(2)
+      var_table[dispatch_number][m.group(1)] = m.group(2)
+      if not var in var_list: var_list.append(var)
+
+    m = beg_pattern.match(record)
+    if m:
+      dispatch_number = m.group(1)
+      if not dispatch_number in var_table:
+        var_table[dispatch_number] = {
+          'Index': dispatch_number,
+          'KernelName': "\"" + m.group(3) + "\""
+        }
+        kernel_properties = m.group(2)
+        for prop in kernel_properties.split(', '):
+          m = prop_pattern.match(prop)
+          if m:
+            var = m.group(1)
+            val = m.group(2)
+            var_table[dispatch_number][var] = val
+            if not var in var_list: var_list.append(var);
+            if re.search(r'gpu-id', var):
+              if (val > max_gpu_id): max_gpu_id = val
+          else: fatal('wrong kernel property "' + prop + '" in "'+ kernel_properties + '"')
+        m = ts_pattern.search(record)
+        if m:
+          var_table[dispatch_number]['DispatchNs'] = m.group(1)
+          var_table[dispatch_number]['BeginNs'] = m.group(2)
+          var_table[dispatch_number]['EndNs'] = m.group(3)
+          var_table[dispatch_number]['CompleteNs'] = m.group(4)
+
+  inp.close()
+#############################################################
+
+# merge results table
+def merge_table():
+  global var_list
+  keys = sorted(var_table.keys(), key=int)
+
+  fields = set(var_table[keys[0]])
+  if 'DispatchNs' in fields:
+    var_list.append('DispatchNs')
+    var_list.append('BeginNs')
+    var_list.append('EndNs')
+    var_list.append('CompleteNs')
+  var_list = [x for x in var_list if x in fields]
+#############################################################
+
+# dump CSV results
+def dump_csv(file_name):
+  global var_list
+  keys = sorted(var_table.keys(), key=int)
+
+  with open(file_name, mode='w') as fd:
+    fd.write(','.join(var_list) + '\n');
+    for ind in keys:
+      entry = var_table[ind]
+      dispatch_number = entry['Index']
+      if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n")
+      val_list = [entry[var] for var in var_list]
+      fd.write(','.join(val_list) + '\n');
+#############################################################
+
+# fill kernels DB
+def fill_kernel_db(table_name, db):
+  global var_list
+  keys = sorted(var_table.keys(), key=int)
+
+  for var in set(var_list).difference(set(table_descr[1])):
+    table_descr[1][var] = 'INTEGER'
+  table_descr[0] = var_list;
+
+  table_handle = db.add_table(table_name, table_descr)
+
+  for ind in keys:
+    entry = var_table[ind]
+    dispatch_number = entry['Index']
+    if ind != dispatch_number: fatal("Dispatch #" + ind + " index mismatch (" + dispatch_number + ")\n")
+    val_list = [entry[var] for var in var_list]
+    db.insert_entry(table_handle, val_list)
+#############################################################
+
+# fill HSA DB
+hsa_table_descr = [
+  ['BeginNs', 'EndNs', 'pid', 'tid', 'Name', 'args'],
+  {'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'}
+]
+def fill_hsa_db(table_name, db, indir):
+  file_name = indir + '/' + 'hsa_api_trace.txt'
+  ptrn = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$')
+
+  table_handle = db.add_table(table_name, hsa_table_descr)
+  with open(file_name, mode='r') as fd:
+    for line in fd.readlines():
+      record = line[:-1]
+      m = ptrn.match(record)
+      if m:
+        rec_vals = []
+        for ind in range(1,7):
+          if hsa_table_descr[0][ind - 1] == 'pid':
+            rec_vals.append(HSA_PID)
+          else:
+            rec_vals.append(m.group(ind))
+        db.insert_entry(table_handle, rec_vals)
+#############################################################
+
+# fill COPY DB
+copy_table_descr = [
+  ['BeginNs', 'EndNs', 'Name', 'pid', 'tid'],
+  {'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'}
+]
+def fill_copy_db(table_name, db, indir):
+  file_name = indir + '/' + 'async_copy_trace.txt'
+  ptrn = re.compile(r'(\d+):(\d+) (.*)$')
+
+  table_handle = db.add_table(table_name, copy_table_descr)
+  with open(file_name, mode='r') as fd:
+    for line in fd.readlines():
+      record = line[:-1]
+      m = ptrn.match(record)
+      if m:
+        rec_vals = []
+        for ind in range(1,4): rec_vals.append(m.group(ind))
+        rec_vals.append(COPY_PID)
+        rec_vals.append(0)
+        db.insert_entry(table_handle, rec_vals)
+#############################################################
+# main
+if (len(sys.argv) < 3): fatal("Usage: " + sys.argv[0] + " <output CSV file> <input result files list>")
+
+outfile = sys.argv[1]
+infiles = sys.argv[2:]
+indir = re.sub(r'\/[^\/]*$', r'', infiles[0])
+print "indir: '" + indir + "'"
+
+dbfile = ''
+csvfile = ''
+
+if re.search(r'\.csv$', outfile):
+  csvfile = outfile
+elif re.search(r'\.db$', outfile):
+  dbfile = outfile
+  csvfile = re.sub(r'\.db$', '.csv', outfile)
+else:
+  fatal("Bad output file '" + outfile + "'")
+
+for f in infiles: parse_res(f)
+if len(var_table) == 0: sys.exit(1)
+merge_table()
+
+if dbfile == '':
+  dump_csv(csvfile)
+else:
+  statfile = re.sub(r'\.csv$', '.stats.csv', csvfile)
+  jsonfile = re.sub(r'\.csv$', '.json', csvfile)
+
+  with open(dbfile, mode='w') as fd: fd.truncate()
+  db = SQLiteDB(dbfile)
+
+  fill_hsa_db('HSA', db, indir) 
+  fill_copy_db('COPY', db, indir) 
+  fill_kernel_db('A', db)
+
+  db.open_json(jsonfile);
+  db.label_json(HSA_PID, "CPU", jsonfile)
+  db.label_json(COPY_PID, "COPY", jsonfile)
+  for ind in range(0, int(max_gpu_id) + 1): db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile)
+
+  if 'BeginNs' in var_list:
+    dform.post_process_data(db, 'A', csvfile)
+    dform.gen_table_bins(db, 'A', statfile, 'KernelName', 'DurationNs')
+    dform.gen_kernel_json_trace(db, 'A', jsonfile)
+  else:
+    db.dump_csv('A', csvfile)
+
+  statfile = re.sub(r'stats', r'hsa_stats', statfile)
+  dform.post_process_data(db, 'HSA')
+  dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs')
+  dform.gen_api_json_trace(db, 'HSA', jsonfile)
+
+  dform.post_process_data(db, 'COPY')
+  dform.gen_api_json_trace(db, 'COPY', jsonfile)
+
+  db.close_json(jsonfile);
+  db.close()
+
+sys.exit(0)
+#############################################################

projects/roctracer/bin/txt2xml.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+################################################################################
+# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+################################################################################
+
+timestamp=`date +%y%m%d_%H%M%S`
+
+if [ $# = 0 ] ; then
+  echo "Usage: $0 <input text file> [output dir]"
+  exit -1
+fi
+
+input=$1
+outdir=$2
+if [ -z "$outdir" ] ; then
+  outdir="."
+fi
+
+range=""
+kernel=""
+gpu_index=""
+
+parse() {
+  scan="$1"
+  index=0
+  while read -r line || [[ -n "$line" ]] ; do
+    line=`echo $line | sed "s/\s*#.*$//"`
+    if [ -z "$line" ] ; then
+      continue
+    fi
+
+    feature=`echo $line | sed -n "s/^\s*\([a-z]*\)\s*:.*$/\1/p"`
+    line=`echo $line | sed "s/^[^:]*:\s*//"`
+    line=`echo "$line" | sed -e "s/\s*=\s*/=/g" -e "s/\s*:\s*/:/g" -e "s/,\{1,\}/ /g" -e "s/\s\{1,\}/ /g" -e "s/\s*$//"`
+
+    if [ "$scan" = 0 ] ; then
+      line=`echo "$line" | sed -e "s/ /,/g"`
+      if [ "$feature" == "range" ] ; then
+        range=$line
+      fi
+      if [ "$feature" == "kernel" ] ; then
+        kernel=$line
+      fi
+      if [ "$feature" == "gpu" ] ; then
+        gpu_index=$line
+      fi
+    else
+      output=$outdir/input${index}.xml
+      header="# $timestamp '$output' generated with '$0 $*'"
+      echo $header > $output
+
+      if [ "$feature" == "pmc" ] ; then
+        line=`echo "$line" | sed -e "s/ /,/g"`
+        cat >> $output <<EOF
+<metric range="$range" kernel="$kernel" gpu_index="$gpu_index"></metric>
+<metric name=$line ></metric>
+EOF
+      fi
+
+      if [ "$feature" == "sqtt" ] ; then
+        cat >> $output <<EOF
+<metric range="$range" kernel="$kernel" gpu_index="$gpu_index"></metric>
+<trace name="SQTT"><parameters $line ></parameters></trace>
+EOF
+      fi
+
+      if [ "$feature" == "hsa" ] ; then
+        cat >> $output <<EOF
+<trace name="HSA"><parameters api="$line"></parameters></trace>
+EOF
+      fi
+    fi
+
+    index=$((index + 1))
+  done < $input
+}
+
+parse 0
+parse 1
+
+exit 0