fixing stats without hsa trace

[ROCm/roctracer commit: a8e799d92b]

projects/roctracer/bin/rpl_run.sh

@@ -23,18 +23,19 @@
 ################################################################################
 
 time_stamp=`date +%y%m%d_%H%M%S`
-BIN_DIR=`dirname $0`
+BIN_DIR=$(dirname $(realpath $0))
-BIN_DIR=`cd $BIN_DIR; pwd`
+PKG_DIR=$(dirname $BIN_DIR)
+ROOT_DIR=$(dirname $PKG_DIR)
 RUN_DIR=`pwd`
 TMP_DIR="/tmp"
 DATA_DIR="rpl_data_${time_stamp}_$$"
 
-PKG_DIR=`echo $BIN_DIR | sed "s/\/bin\/*//"`
-BIN_DIR=$PKG_DIR/bin
-
 # PATH to custom HSA and OpenCl runtimes
 HSA_PATH=$PKG_DIR/lib/hsa
 
+# roctracer path
+if [ -z "$ROCTRACER_PATH" ] ; then ROCTRACER_PATH=$ROOT_DIR/roctracer; fi
+
 # runtime API trace
 HSA_TRACE=0
 HIP_TRACE=0
@@ -42,7 +43,7 @@ HIP_TRACE=0
 # Generate stats
 GEN_STATS=0
 
-export LD_LIBRARY_PATH=$PKG_DIR/lib:$PKG_DIR/tool:$PKG_DIR/roctracer/lib:$PKG_DIR/roctracer/tool:$HSA_PATH
+export LD_LIBRARY_PATH=$PKG_DIR/lib:$PKG_DIR/tool:$ROCTRACER_PATH/lib:$ROCTRACER_PATH/tool:$HSA_PATH
 export PATH=.:$PATH
 
 # enable error logging

projects/roctracer/bin/sqlitedb.py

@@ -1,10 +1,11 @@
-import csv, sqlite3, re
+import csv, sqlite3, re, sys
 
 # SQLite Database class
 class SQLiteDB:
   def __init__(self, file_name):
     self.connection = sqlite3.connect(file_name)
     self.tables = {}
+    self.json_arg_list_enabled = 0
 
   def __del__(self):
     self.connection.close()
@@ -71,6 +72,9 @@ class SQLiteDB:
   def _get_raws(self, table_name):
     cursor = self.connection.execute('SELECT * FROM ' + table_name)
     return cursor.fetchall()
+  def _get_raws_indexed(self, table_name):
+    cursor = self.connection.execute('SELECT * FROM ' + table_name + ' order by "Index" asc;')
+    return cursor.fetchall()
   def _get_raw_by_id(self, table_name, req_id):
     cursor = self.connection.execute('SELECT * FROM ' + table_name + ' WHERE "Index"=?', (req_id,))
     raws = cursor.fetchall()
@@ -128,31 +132,38 @@ class SQLiteDB:
     name_ptrn = re.compile(r'(name|Name)')
 
     table_fields = self._get_fields(table_name)
-    table_raws = self._get_raws(table_name)
+    table_raws = self._get_raws_indexed(table_name)
     data_fields = self._get_fields(data_name)
+    data_raws = self._get_raws_indexed(data_name)
 
     with open(file_name, mode='a') as fd:
-      for raw_index in range(len(table_raws)):
+      table_raws_len = len(table_raws)
-        values = list(table_raws[raw_index])
+      for raw_index in range(table_raws_len):
+        if (raw_index == table_raws_len - 1) or (raw_index % 1000 == 0):
+          sys.stdout.write( \
+            "\rdump json " + str(raw_index) + ":" + str(len(table_raws))  + " "*100 \
+          )
+
         vals_list = []
-        raw_id = 0;
+        values = list(table_raws[raw_index])
         for value_index in range(len(values)):
           label = table_fields[value_index]
           value = values[value_index]
           if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
-          if label == '"Index"': raw_id = value
+          if label != '"Index"': vals_list.append('%s:"%s"' % (label, value))
-          else: vals_list.append('%s:"%s"' % (label, value))
 
-        data = self._get_raw_by_id(data_name, raw_id)
         args_list = []
+        data = list(data_raws[raw_index])
         for value_index in range(len(data)):
           label = data_fields[value_index]
           value = data[value_index]
           if name_ptrn.search(label): value = sub_ptrn.sub(r'', value)
-          args_list.append('%s:"%s"' % (label, value))
+          if label != '"Index"': args_list.append('%s:"%s"' % (label, value))
 
         fd.write(',{"ph":"%s",%s,\n  "args":{\n    %s\n  }\n}\n' % ('X', ','.join(vals_list), ',\n    '.join(args_list)))
 
+    sys.stdout.write('\n')
+
   # execute query on DB
   def execute(self, cmd):
     cursor = self.connection.cursor()

projects/roctracer/bin/tblextr.py

@@ -110,21 +110,20 @@ def parse_res(infile):
           var_table[dispatch_number]['BeginNs'] = m.group(2)
           var_table[dispatch_number]['EndNs'] = m.group(3)
           var_table[dispatch_number]['CompleteNs'] = m.group(4)
-        else: fatal('bad kernel record "' + record + '"')
 
-        gpu_pid = GPU_BASE_PID + int(gpu_id)
+          gpu_pid = GPU_BASE_PID + int(gpu_id)
-        if not gpu_pid in dep_dict: dep_dict[gpu_pid] = {}
+          if not gpu_pid in dep_dict: dep_dict[gpu_pid] = {}
-        dep_str = dep_dict[gpu_pid]
+          dep_str = dep_dict[gpu_pid]
-        if not 'tid' in dep_str: dep_str['tid'] = []
+          if not 'tid' in dep_str: dep_str['tid'] = []
-        if not 'from' in dep_str: dep_str['from'] = []
+          if not 'from' in dep_str: dep_str['from'] = []
-        if not 'to' in dep_str: dep_str['to'] = {}
+          if not 'to' in dep_str: dep_str['to'] = {}
-        to_id = len(dep_str['tid'])
+          to_id = len(dep_str['tid'])
-        from_us = int(m.group(1)) / 1000
+          from_us = int(m.group(1)) / 1000
-        to_us = int(m.group(2)) / 1000
+          to_us = int(m.group(2)) / 1000
-        dep_str['to'][to_id] = to_us
+          dep_str['to'][to_id] = to_us
-        dep_str['from'].append(from_us)
+          dep_str['from'].append(from_us)
-        dep_str['tid'].append(disp_tid)
+          dep_str['tid'].append(disp_tid)
-        kern_dep_list.append((disp_tid, m.group(1)))
+          kern_dep_list.append((disp_tid, m.group(1)))
 
   inp.close()
 #############################################################
@@ -187,6 +186,8 @@ def fill_hsa_db(table_name, db, indir):
   ptrn_val = re.compile(r'(\d+):(\d+) (\d+):(\d+) ([^\(]+)(\(.*)$')
   ptrn_ac = re.compile(r'hsa_amd_memory_async_copy')
 
+  if not os.path.isfile(file_name): return 0
+
   if not COPY_PID in dep_dict: dep_dict[COPY_PID] = {}
   dep_tid_list = []
   dep_from_us_list = []
@@ -227,6 +228,8 @@ def fill_hsa_db(table_name, db, indir):
 
   dep_dict[COPY_PID]['tid'] = dep_tid_list
   dep_dict[COPY_PID]['from'] = dep_from_us_list
+
+  return 1
 #############################################################
 
 # fill COPY DB
@@ -267,7 +270,6 @@ if (len(sys.argv) < 3): fatal("Usage: " + sys.argv[0] + " <output CSV file> <inp
 outfile = sys.argv[1]
 infiles = sys.argv[2:]
 indir = re.sub(r'\/[^\/]*$', r'', infiles[0])
-print "indir: '" + indir + "'"
 
 dbfile = ''
 csvfile = ''
@@ -292,15 +294,19 @@ else:
 
   with open(dbfile, mode='w') as fd: fd.truncate()
   db = SQLiteDB(dbfile)
+  db.open_json(jsonfile);
 
-  fill_hsa_db('HSA', db, indir) 
+  hsa_trace_found = fill_hsa_db('HSA', db, indir)
-  fill_copy_db('COPY', db, indir) 
+  if hsa_trace_found:
+    fill_copy_db('COPY', db, indir)
   fill_kernel_db('A', db)
 
-  db.open_json(jsonfile);
+  if hsa_trace_found:
-  db.label_json(HSA_PID, "CPU", jsonfile)
+    db.label_json(HSA_PID, "CPU", jsonfile)
-  db.label_json(COPY_PID, "COPY", jsonfile)
+    db.label_json(COPY_PID, "COPY", jsonfile)
-  for ind in range(0, int(max_gpu_id) + 1): db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile)
+
+  for ind in range(0, int(max_gpu_id) + 1):
+    db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile)
 
   if 'BeginNs' in var_list:
     dform.post_process_data(db, 'A', csvfile)
@@ -309,13 +315,14 @@ else:
   else:
     db.dump_csv('A', csvfile)
 
-  statfile = re.sub(r'stats', r'hsa_stats', statfile)
+  if hsa_trace_found:
-  dform.post_process_data(db, 'HSA')
+    statfile = re.sub(r'stats', r'hsa_stats', statfile)
-  dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs')
+    dform.post_process_data(db, 'HSA')
-  dform.gen_api_json_trace(db, 'HSA', START_US, jsonfile)
+    dform.gen_table_bins(db, 'HSA', statfile, 'Name', 'DurationNs')
+    dform.gen_api_json_trace(db, 'HSA', START_US, jsonfile)
 
-  dform.post_process_data(db, 'COPY')
+    dform.post_process_data(db, 'COPY')
-  dform.gen_api_json_trace(db, 'COPY', START_US, jsonfile)
+    dform.gen_api_json_trace(db, 'COPY', START_US, jsonfile)
 
   dep_id = 0
   for (to_pid, dep_str) in dep_dict.items():