AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding system info

Browse Source 
Change-Id: I1f7d23bd7c7e317eed9e177f0c7681226be4bfca
This commit is contained in:
Evgeny
2020-03-24 22:01:34 -05:00
parent 299a08e4be
commit c77a6236e7
4 changed files with 148 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/run_tool.sh
-1
View File
@@ -1,5 +1,4 @@
#!/bin/sh
BIN_DIR=`dirname $0`
BIN_DIR=`realpath $BIN_DIR`
PKG_DIR=${BIN_DIR%/bin}
bin/sqlitedb.py
+15 -2
View File
@@ -22,6 +22,7 @@
import csv, sqlite3, re, sys
from functools import reduce
from txt2params import gen_params
# SQLite Database class
class SQLiteDB:
@@ -133,13 +134,13 @@ class SQLiteDB:
if not re.search(r'\.json$', file_name):
raise Exception('wrong output file type: "' + file_name + '"' )
with open(file_name, mode='a') as fd:
fd.write(']}\n');
fd.write('}')
def label_json(self, pid, label, file_name):
if not re.search(r'\.json$', file_name):
raise Exception('wrong output file type: "' + file_name + '"' )
with open(file_name, mode='a') as fd:
fd.write(',{"args":{"name":"%s %s"},"ph":"M","pid":%s,"name":"process_name"}\n' %(self.section_index, label, pid));
fd.write(',{"args":{"name":"%s"},"ph":"M","pid":%s,"name":"process_name","sort_index":%d}\n' %(label, pid, self.section_index))
self.section_index += 1
def flow_json(self, base_id, from_pid, from_tid, from_us_list, to_pid, to_us_dict, corr_id_list, start_us, file_name):
@@ -158,6 +159,18 @@ class SQLiteDB:
fd.write(',{"ts":%d,"ph":"t","cat":"DataFlow","id":%d,"pid":%s,"tid":0,"name":"dep"}\n' % (to_ts, dep_id, str(to_pid)))
dep_id += 1
def metadata_json(self, jsonfile, sysinfo_file):
params = gen_params(sysinfo_file);
with open(jsonfile, mode='a') as fd:
fd.write('],\n')
fd.write('"otherData": {\n')
cnt = 0
for key in params:
if cnt != 0: fd.write(',\n')
fd.write(' "' + key + '": "' + params[key] + '"')
cnt = cnt + 1
fd.write('\n }\n')
def dump_json(self, table_name, data_name, file_name):
if not re.search(r'\.json$', file_name):
raise Exception('wrong output file type: "' + file_name + '"' )
bin/tblextr.py
+25 -4
View File
@@ -20,7 +20,7 @@
# THE SOFTWARE.
################################################################################
import os, sys, re
import os, sys, re, subprocess
from sqlitedb import SQLiteDB
import dform
@@ -63,10 +63,26 @@ def dbglog(msg):
fatal("error")
#############################################################
# Dumping sysinfo
sysinfo_begin = 1
def metadata_gen(sysinfo_file, sysinfo_cmd):
global sysinfo_begin
if not re.search(r'\.txt$', sysinfo_file):
raise Exception('wrong output file type: "' + sysinfo_file + '"' )
if sysinfo_begin == 1:
sysinfo_begin = 0
with open(sysinfo_file, mode='w') as fd: fd.write('')
with open(sysinfo_file, mode='a') as fd: fd.write('CMD: ' + sysinfo_cmd + '\n')
status = subprocess.call(sysinfo_cmd + ' >> ' + sysinfo_file,
stderr=subprocess.STDOUT,
shell=True)
if status != 0:
raise Exception('Could not run command: "' + sysinfo_cmd + '"')
# parse results method
def parse_res(infile):
global max_gpu_id
if not os.path.isfile(infile): return # fatal("Error: input file '" + infile + "' not found")
if not os.path.isfile(infile): return
inp = open(infile, 'r')
beg_pattern = re.compile("^dispatch\[(\d*)\], (.*) kernel-name\(\"([^\"]*)\"\)")
@@ -231,9 +247,9 @@ def fill_ext_db(table_name, db, indir, trace_name, api_pid):
continue
if cid == 2:
if not pid in range_stack: fatal("ROCTX range begin missed, pid(" + pid + ")");
if not pid in range_stack: fatal("ROCTX range begin not found, pid(" + pid + ")");
pid_stack = range_stack[pid]
if not tid in pid_stack: fatal("ROCTX range begin missed, tid(" + tid + ")");
if not tid in pid_stack: fatal("ROCTX range begin not found, tid(" + tid + ")");
rec_stack = pid_stack[tid]
rec_vals = rec_stack.pop()
rec_vals[1] = tms
@@ -495,6 +511,8 @@ else:
kfd_statfile = re.sub(r'\.stats\.csv$', r'.kfd_stats.csv', statfile)
ops_statfile = statfile
copy_statfile = re.sub(r'\.stats\.csv$', r'.copy_stats.csv', statfile)
sysinfo_file = re.sub(r'\.stats\.csv$', r'.sysinfo.txt', statfile)
metadata_gen(sysinfo_file, '/opt/rocm/bin/rocminfo')
with open(dbfile, mode='w') as fd: fd.truncate()
db = SQLiteDB(dbfile)
@@ -565,6 +583,8 @@ else:
dform.gen_table_bins(db, 'OPS', ops_statfile, 'Name', 'DurationNs')
dform.gen_ops_json_trace(db, 'OPS', GPU_BASE_PID, START_US, jsonfile)
metadata_gen(sysinfo_file, '/opt/rocm/bin/hipcc --version')
if kfd_trace_found:
dform.post_process_data(db, 'KFD')
dform.gen_table_bins(db, 'KFD', kfd_statfile, 'Name', 'DurationNs')
@@ -595,6 +615,7 @@ else:
dep_id += len(tid_list)
if any_trace_found:
db.metadata_json(jsonfile, sysinfo_file)
db.close_json(jsonfile);
db.close()
bin/txt2params.py
+108
View File
@@ -0,0 +1,108 @@
#!/usr/bin/python
################################################################################
# Copyright (c) 2018 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
################################################################################
import os, sys, re
# gen_params() takes a text file like the output of rocminfo cmd and parses it into a map {key,value}
# where key is the param and value is the value of this param
# for example: Threadmodel : "posix"
# it also processes encompasing sections to generate a full param name such as (section names separated by '_'):
# "Agent2_PoolInfo_ISAInfo_ISA1_WorkgroupMaxSizeperDimension_x": "1024(0x400)",
def gen_params(txtfile):
fields = {}
parent_field = ''
nbr_indent = 0
nbr_indent_prev = 0
check_for_dims = False
with open(txtfile) as fp:
for line in fp:
me = re.match(r'\*\*\* Done \*\*\*',line) #Marks the end of cmd
if me:
parent_field = ''
nbr_indent = 0
nbr_indent_prev = 0
check_for_dims = False
continue
mv = re.match(r'HCC clang version\s+(.*)',line) # outlier: only line with a version number and no ':', special case
if mv:
key = 'HCCclangversion'
val = mv.group(1)
fields[key] = val
continue
# Variable 'check_for_dims' is True for text like this:
# Workgroup Max Size per Dimension:
# x 1024(0x400)
# y 1024(0x400)
# z 1024(0x400)
if check_for_dims == True:
mc = re.match(r'\s*([x|y|z])\s+(.*)',line)
if mc:
key_sav = mc.group(1)
if parent_field != '':
key = parent_field + '_' + mc.group(1)
else:
key = mc.group(1)
val = re.sub(r"\s+", "", mc.group(2))
fields[key] = val
if key_sav == 'z':
check_for_dims = False
nbr_indent_prev = nbr_indent
mi = re.search(r'^(\s+)\w+', line)
md = re.search(':', line)
if mi:
nbr_indent = len(mi.group(1)) / 2 #indentation cnt
else:
if not md:
tmp = re.sub(r"\s+", "", line)
if tmp.isalnum():
parent_field = tmp
continue
if nbr_indent < nbr_indent_prev:
pos = parent_field.rfind('_')
if pos != -1:
parent_field = parent_field[:pos]
# Process lines such as :
# Segment: GLOBAL; FLAGS: KERNARG, FINE GRAINED
# Size: 131897644(0x7dc992c) KB
for lin in line.split(';'):
lin = re.sub(r"\s+", "", lin)
m = re.match(r'(.*):(.*)', lin)
if m:
key, val = m.group(1), m.group(2)
if parent_field != '':
key = parent_field + '_' + key
if val == '':
mk = re.match(r'.*Dimension',key)
if mk: # expect x,y,z on next 3 lines
check_for_dims = True
parent_field = key
else:
fields[key] = val
else:
if nbr_indent != nbr_indent_prev and not check_for_dims :
parent_field = parent_field + '_' + lin.replace(':','')
return fields