diff --git a/bin/dform.py b/bin/dform.py index 1e5c63b1c5..f797d63703 100644 --- a/bin/dform.py +++ b/bin/dform.py @@ -24,6 +24,11 @@ def gen_api_json_trace(db, table, start_us, outfile): db.dump_json('B', table, outfile) db.execute('DROP VIEW B') +def gen_ext_json_trace(db, table, start_us, outfile): + db.execute('create view B as select Name as name, pid, tid, (BeginNs/1000 - %d) as ts, ((EndNs - BeginNs)/1000) as dur from %s order by ts asc;' % (start_us, table)); + db.dump_json('B', table, outfile) + db.execute('DROP VIEW B') + def gen_ops_json_trace(db, table, base_pid, start_us, outfile): db.execute('create view B as select "Index", Name as name, ("dev-id" + %d) as pid, tid, (BeginNs/1000 - %d) as ts, (DurationNs/1000) as dur from %s order by ts asc;' % (base_pid, start_us, table)); db.dump_json('B', table, outfile) diff --git a/bin/rpl_run.sh b/bin/rpl_run.sh index 8331ffc8cb..9655d78d9f 100755 --- a/bin/rpl_run.sh +++ b/bin/rpl_run.sh @@ -45,6 +45,7 @@ if [ -z "$HCC_HOME" ] ; then fi # runtime API trace +ROCTX_TRACE=0 HSA_TRACE=0 SYS_TRACE=0 HIP_TRACE=0 @@ -97,28 +98,6 @@ errck() { fi } -parse_time_val() { - local time_maxumim_us=$((0xffffffff)) - local __resultvar=$1 - eval "local val=$"$__resultvar - val_m=`echo $val | sed -n "s/^\([0-9]*\)m$/\1/p"` - val_s=`echo $val | sed -n "s/^\([0-9]*\)s$/\1/p"` - val_ms=`echo $val | sed -n "s/^\([0-9]*\)ms$/\1/p"` - val_us=`echo $val | sed -n "s/^\([0-9]*\)us$/\1/p"` - if [ -n "$val_m" ] ; then val_us=$((val_m*60000000)) - elif [ -n "$val_s" ] ; then val_us=$((val_s*1000000)) - elif [ -n "$val_ms" ] ; then val_us=$((val_ms*1000)) - fi - - if [ -z "$val_us" ] ; then - error_message="invalid time value format ($val)" - elif [ "$val_us" -gt "$time_maxumim_us" ] ; then - error_message="time value exceeds maximum supported ($val > ${time_maxumim_us}us)" - else - eval $__resultvar="'$val_us'" - fi -} - # usage method usage() { bin_name=`basename $0` @@ -180,6 +159,7 @@ usage() { echo " --heartbeat - to print progress heartbeats [0 - disabled]" echo "" echo " --stats - generating kernel execution stats, file .stats.csv" + echo " --roctx-trace - to enable rocTX trace" echo " --hsa-trace - to trace HSA, generates API execution stats and JSON file chrome-tracing compatible" echo " --sys-trace - to trace HIP/HSA APIs and GPU activity, generates stats and JSON trace chrome-tracing compatible" echo " --hip-trace - to trace HIP, generates API execution stats and JSON file chrome-tracing compatible" @@ -246,23 +226,22 @@ run() { fi API_TRACE="" - if [ "$HSA_TRACE" = 1 ] ; then - API_TRACE="hsa" + if [ "$ROCTX_TRACE" = 1 ] ; then + API_TRACE=${API_TRACE}":roctx" fi - if [ "$SYS_TRACE" = 1 ] ; then - API_TRACE="sys" + if [ "$HSA_TRACE" = 1 ] ; then + API_TRACE=${API_TRACE}":hsa" fi if [ "$HIP_TRACE" = 1 ] ; then - if [ -z "$API_TRACE" ] ; then - API_TRACE="hip"; - else - API_TRACE="all" - fi + API_TRACE=${API_TRACE}":hip" fi + if [ "$SYS_TRACE" = 1 ] ; then + API_TRACE=${API_TRACE}":sys" + fi + if [ -n "$API_TRACE" ] ; then - API_TRACE=$(echo $API_TRACE | sed 's/all//') - if [ -n "$API_TRACE" ] ; then export ROCTRACER_DOMAIN=$API_TRACE; fi - if [ "$API_TRACE" = "hip" -o "$API_TRACE" = "sys" ] ; then + export ROCTRACER_DOMAIN=$API_TRACE + if [ "$HSA_TRACE" = 0 ] ; then OUTPUT_LIST="$ROCP_OUTPUT_DIR/" fi export HSA_TOOLS_LIB="$TTLIB_PATH/libtracer_tool.so" @@ -273,7 +252,6 @@ run() { redirection_cmd="2>&1 | tee $ROCP_OUTPUT_DIR/log.txt" fi - #unset ROCP_OUTPUT_DIR CMD_LINE="$APP_CMD $redirection_cmd" eval "$CMD_LINE" } @@ -291,6 +269,29 @@ merge_output() { done } +convert_time_val() { + local time_maxumim_us=$((0xffffffff)) + local __resultvar=$1 + eval "local val=$"$__resultvar + val_m=`echo $val | sed -n "s/^\([0-9]*\)m$/\1/p"` + val_s=`echo $val | sed -n "s/^\([0-9]*\)s$/\1/p"` + val_ms=`echo $val | sed -n "s/^\([0-9]*\)ms$/\1/p"` + val_us=`echo $val | sed -n "s/^\([0-9]*\)us$/\1/p"` + if [ -n "$val_m" ] ; then val_us=$((val_m*60000000)) + elif [ -n "$val_s" ] ; then val_us=$((val_s*1000000)) + elif [ -n "$val_ms" ] ; then val_us=$((val_ms*1000)) + fi + + if [ -z "$val_us" ] ; then + error_message="invalid time value format ($val)" + elif [ "$val_us" -gt "$time_maxumim_us" ] ; then + error_message="time value exceeds maximum supported ($val > ${time_maxumim_us}us)" + else + eval $__resultvar="'$val_us'" + fi +} + +################################################################################################ # main echo "RPL: on '$time_stamp' from '$PKG_DIR' in '$RUN_DIR'" # Parsing arguments @@ -358,6 +359,9 @@ while [ 1 ] ; do ARG_VAL=0 export ROCP_TIMESTAMP_ON=1 GEN_STATS=1 + elif [ "$1" = "--roctx-trace" ] ; then + ARG_VAL=0 + ROCTX_TRACE=1 elif [ "$1" = "--hsa-trace" ] ; then ARG_VAL=0 export ROCP_TIMESTAMP_ON=1 @@ -382,11 +386,11 @@ while [ 1 ] ; do period_delay=`echo "$2" | sed -n "s/"${period_expr}"/\1/p"` period_rate=`echo "$2" | sed -n "s/"${period_expr}"/\2/p"` period_len=`echo "$2" | sed -n "s/"${period_expr}"/\3/p"` - parse_time_val period_delay + convert_time_val period_delay errck "Option '$ARG_IN', delay value" - parse_time_val period_rate + convert_time_val period_rate errck "Option '$ARG_IN', rate value" - parse_time_val period_len + convert_time_val period_len errck "Option '$ARG_IN', length value" export ROCP_CTRL_RATE="$period_delay:$period_rate:$period_len" elif [ "$1" = "--verbose" ] ; then diff --git a/bin/tblextr.py b/bin/tblextr.py index 7a30cd56c2..895f41b112 100755 --- a/bin/tblextr.py +++ b/bin/tblextr.py @@ -36,7 +36,8 @@ COPY_PID = 0 OPS_PID = 1 HSA_PID = 2 HIP_PID = 3 -GPU_BASE_PID = 4 +EXT_PID = 4 +GPU_BASE_PID = 5 max_gpu_id = -1 START_US = 0 @@ -186,10 +187,66 @@ def fill_kernel_db(table_name, db): db.insert_entry(table_handle, val_list) ############################################################# -# fill HSA DB -hsa_table_descr = [ +# Fill Ext DB +ext_table_descr = [ + ['BeginNs', 'EndNs', 'pid', 'tid', 'Name', 'Index'], + {'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER', 'Name':'TEXT', 'Index':'INTEGER'} +] +def fill_ext_db(table_name, db, indir, trace_name, api_pid): + file_name = indir + '/' + trace_name + '_trace.txt' + ptrn_val = re.compile(r'(\d+) (\d+):(\d+) (\d+):(.*)$') + + if not os.path.isfile(file_name): return 0 + + range_stack = {} + + record_id = 0 + table_handle = db.add_table(table_name, ext_table_descr) + with open(file_name, mode='r') as fd: + for line in fd.readlines(): + record = line[:-1] + m = ptrn_val.match(record) + if m: + tms = int(m.group(1)) + pid = m.group(2) + tid = m.group(3) + cid = int(m.group(4)) + msg = m.group(5) + + rec_vals = [] + + if cid != 2: + rec_vals.append(tms) + rec_vals.append(tms + 1) + rec_vals.append(api_pid) + rec_vals.append(tid) + rec_vals.append(msg) + rec_vals.append(record_id) + + if cid == 1: + if not pid in range_stack: range_stack[pid] = {} + pid_stack = range_stack[pid] + if not tid in pid_stack: pid_stack[tid] = [] + rec_stack = pid_stack[tid] + rec_stack.append(rec_vals) + continue + + if cid == 2: + pid_stack = range_stack[pid] + rec_stack = pid_stack[tid] + rec_vals = rec_stack.pop() + rec_vals[1] = tms + + db.insert_entry(table_handle, rec_vals) + record_id += 1 + + return 1 +############################################################# + +# Fill API DB +api_table_descr = [ ['BeginNs', 'EndNs', 'pid', 'tid', 'Name', 'args', 'Index'], - {'Index':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER'} + {'BeginNs':'INTEGER', 'EndNs':'INTEGER', 'pid':'INTEGER', 'tid':'INTEGER', 'Name':'TEXT', 'args':'TEXT', 'Index':'INTEGER'} ] def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep_filtr, expl_id): file_name = indir + '/' + api_name + '_api_trace.txt' @@ -211,14 +268,15 @@ def fill_api_db(table_name, db, indir, api_name, api_pid, dep_pid, dep_list, dep START_US = 0 record_id = 0 - table_handle = db.add_table(table_name, hsa_table_descr) + table_handle = db.add_table(table_name, api_table_descr) with open(file_name, mode='r') as fd: for line in fd.readlines(): record = line[:-1] m = ptrn_val.match(record) if m: rec_vals = [] - for ind in range(1,7): + rec_len = len(api_table_descr[0]) + for ind in range(1,rec_len): rec_vals.append(m.group(ind)) rec_vals[2] = api_pid rec_vals.append(record_id) @@ -359,6 +417,8 @@ else: with open(dbfile, mode='w') as fd: fd.truncate() db = SQLiteDB(dbfile) + ext_trace_found = fill_ext_db('rocTX', db, indir, 'roctx', EXT_PID) + hsa_trace_found = fill_api_db('HSA', db, indir, 'hsa', HSA_PID, COPY_PID, kern_dep_list, {}, 0) hsa_activity_found = fill_copy_db('COPY', db, indir) @@ -367,10 +427,13 @@ else: fill_kernel_db('A', db) - any_trace_found = hsa_trace_found | hip_trace_found + any_trace_found = ext_trace_found | hsa_trace_found | hip_trace_found if any_trace_found: db.open_json(jsonfile) + if ext_trace_found: + db.label_json(EXT_PID, "Markers and Ranges", jsonfile) + if hsa_trace_found: db.label_json(HSA_PID, "CPU HSA API", jsonfile) if hsa_activity_found: @@ -383,6 +446,9 @@ else: for ind in range(0, int(max_gpu_id) + 1): db.label_json(int(ind) + int(GPU_BASE_PID), "GPU" + str(ind), jsonfile) + if ext_trace_found: + dform.gen_ext_json_trace(db, 'rocTX', START_US, jsonfile) + if len(var_table) != 0: dform.post_process_data(db, 'A', csvfile) dform.gen_table_bins(db, 'A', statfile, 'KernelName', 'DurationNs')