Brandon Potter
296 строки
14 KiB
Python
296 строки
14 KiB
Python
#!/tool/pandora64/.package/python-3.8.0/bin/python3
|
|
|
|
import dictionary
|
|
import log
|
|
#import matplotlib.pyplot
|
|
import numpy
|
|
import csv
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
class Plot():
|
|
def __init__(self, args, archives, changeset_to_build):
|
|
self._args = args
|
|
self._archives = archives
|
|
self._changelog = dictionary.ChangelogToMostRecentBuild()
|
|
self._changelog.generate(changeset_to_build)
|
|
print(self._changelog.dump())
|
|
|
|
def abbreviate_changesets(self, changesets):
|
|
return [changeset[0:8] for changeset in changesets]
|
|
|
|
@staticmethod
|
|
def write_dict_to_file(tracker, field_names, file_name):
|
|
with open(file_name, 'w') as csvfile:
|
|
writer = csv.DictWriter(csvfile, fieldnames=field_names)
|
|
writer.writeheader()
|
|
writer.writerows(tracker)
|
|
|
|
@staticmethod
|
|
def check_and_add_to_dict(dictionary, key, array):
|
|
if len(array) > 0:
|
|
dictionary[key] = array[0]
|
|
else:
|
|
dictionary[key] = 0
|
|
|
|
def changeset_slice(self):
|
|
self._log_tracker = log.Tracker(self._args, self._archives)
|
|
for changeset in self._changelog._all_changesets:
|
|
if changeset in self._changelog.data.keys():
|
|
build_id = self._changelog.data[changeset]
|
|
self._log_tracker.add(changeset, build_id)
|
|
print(self._log_tracker.dump())
|
|
|
|
"""
|
|
separate out dictionaries based on operation
|
|
and prepare them in a format that works with
|
|
the csv module
|
|
"""
|
|
put_tracker = []
|
|
put_nbi_tracker = []
|
|
get_tracker = []
|
|
get_nbi_tracker = []
|
|
amo_tracker = []
|
|
ping_pong_tracker = []
|
|
prev_commit = list(self._log_tracker._data.keys())[0][0]
|
|
amo_dict = {}
|
|
for key, value in self._log_tracker._data.items():
|
|
if (key[1] == "put.log"):
|
|
put_tracker.append({'Commit':key[0][0:7],
|
|
'b1':value.latency[0],
|
|
'b2':value.latency[1],
|
|
'b4':value.latency[2],
|
|
'b8':value.latency[3],
|
|
'b16':value.latency[4],
|
|
'b32':value.latency[5],
|
|
'b64':value.latency[6],
|
|
'b128':value.latency[7],
|
|
'b256':value.latency[8],
|
|
'b512':value.latency[9],
|
|
'b1024':value.latency[10],
|
|
'b2048':value.latency[11],
|
|
'b4096':value.latency[12],
|
|
'b8192':value.latency[13],
|
|
'b16384':value.latency[14],
|
|
'b32768':value.latency[15]
|
|
})
|
|
if (key[1] == "put_nbi.log"):
|
|
put_nbi_tracker.append({'Commit':key[0][0:7],
|
|
'b1':value.latency[0],
|
|
'b2':value.latency[1],
|
|
'b4':value.latency[2],
|
|
'b8':value.latency[3],
|
|
'b16':value.latency[4],
|
|
'b32':value.latency[5],
|
|
'b64':value.latency[6],
|
|
'b128':value.latency[7],
|
|
'b256':value.latency[8],
|
|
'b512':value.latency[9],
|
|
'b1024':value.latency[10],
|
|
'b2048':value.latency[11],
|
|
'b4096':value.latency[12],
|
|
'b8192':value.latency[13],
|
|
'b16384':value.latency[14],
|
|
'b32768':value.latency[15]
|
|
})
|
|
if (key[1] == "get.log"):
|
|
get_tracker.append({'Commit':key[0][0:7],
|
|
'b1':value.latency[0],
|
|
'b2':value.latency[1],
|
|
'b4':value.latency[2],
|
|
'b8':value.latency[3],
|
|
'b16':value.latency[4],
|
|
'b32':value.latency[5],
|
|
'b64':value.latency[6],
|
|
'b128':value.latency[7],
|
|
'b256':value.latency[8],
|
|
'b512':value.latency[9],
|
|
'b1024':value.latency[10],
|
|
'b2048':value.latency[11],
|
|
'b4096':value.latency[12],
|
|
'b8192':value.latency[13],
|
|
'b16384':value.latency[14],
|
|
'b32768':value.latency[15]
|
|
})
|
|
if (key[1] == "get_nbi.log"):
|
|
get_nbi_tracker.append({'Commit':key[0][0:7],
|
|
'b1':value.latency[0],
|
|
'b2':value.latency[1],
|
|
'b4':value.latency[2],
|
|
'b8':value.latency[3],
|
|
'b16':value.latency[4],
|
|
'b32':value.latency[5],
|
|
'b64':value.latency[6],
|
|
'b128':value.latency[7],
|
|
'b256':value.latency[8],
|
|
'b512':value.latency[9],
|
|
'b1024':value.latency[10],
|
|
'b2048':value.latency[11],
|
|
'b4096':value.latency[12],
|
|
'b8192':value.latency[13],
|
|
'b16384':value.latency[14],
|
|
'b32768':value.latency[15]
|
|
})
|
|
if (key[1] == "ping_pong.log"):
|
|
ping_pong_tracker.append({'Commit':key[0][0:7],
|
|
'latency':value.latency[0]
|
|
})
|
|
|
|
# check to see if we have moved to a new commit
|
|
# if we have, store the dict in the amo_tracker
|
|
if (key[0] != prev_commit):
|
|
amo_dict['Commit'] = prev_commit[0:7]
|
|
amo_tracker.append(amo_dict.copy())
|
|
amo_dict.clear()
|
|
|
|
prev_commit = key[0]
|
|
|
|
if (key[1] == "amo_add.log"):
|
|
self.check_and_add_to_dict(amo_dict, 'add', value.latency)
|
|
if (key[1] == "amo_cswap.log"):
|
|
self.check_and_add_to_dict(amo_dict, 'cswap', value.latency)
|
|
if (key[1] == "amo_fadd.log"):
|
|
self.check_and_add_to_dict(amo_dict, 'fadd', value.latency)
|
|
if (key[1] == "amo_fcswap.log"):
|
|
self.check_and_add_to_dict(amo_dict, 'fcswap', value.latency)
|
|
if (key[1] == "amo_fetch.log"):
|
|
self.check_and_add_to_dict(amo_dict, 'fetch', value.latency)
|
|
if (key[1] == "amo_finc.log"):
|
|
self.check_and_add_to_dict(amo_dict, 'finc', value.latency)
|
|
if (key[1] == "amo_inc.log"):
|
|
self.check_and_add_to_dict(amo_dict, 'inc', value.latency)
|
|
|
|
# store the last commit's amo data
|
|
amo_dict['Commit'] = prev_commit[0:7]
|
|
amo_tracker.append(amo_dict.copy())
|
|
|
|
# write put results into a file:
|
|
size_field_names= ['Commit','b1','b2','b4','b8','b16','b32','b64','b128','b256','b512','b1024','b2048','b4096','b8192','b16384','b32768']
|
|
amo_field_names= ['Commit','add','cswap','fadd','fcswap','fetch','finc','inc']
|
|
ping_pong_field_names= ['Commit','latency']
|
|
|
|
self.write_dict_to_file(put_tracker, size_field_names, "put.csv")
|
|
self.write_dict_to_file(put_nbi_tracker, size_field_names, "put_nbi.csv")
|
|
self.write_dict_to_file(get_tracker, size_field_names, "get.csv")
|
|
self.write_dict_to_file(get_nbi_tracker, size_field_names, "get_nbi.csv")
|
|
self.write_dict_to_file(amo_tracker, amo_field_names, "amo.csv")
|
|
self.write_dict_to_file(ping_pong_tracker, ping_pong_field_names, "ping_pong.csv")
|
|
|
|
# make a directory and execute the R script to generate plots in that directory
|
|
current_dir = os.getcwd()
|
|
plot_dir = os.path.join(current_dir, 'plots')
|
|
if not os.path.exists(plot_dir):
|
|
os.makedirs(plot_dir)
|
|
|
|
changeset_a = list(self._log_tracker._data.keys())[0][0]
|
|
changeset_b = list(self._log_tracker._data.keys())[-1][0]
|
|
|
|
# check if the provided changesets are correct
|
|
if (self._args.changeset_range):
|
|
found_changeset_a = False
|
|
found_changeset_b = False
|
|
for key, value in self._log_tracker._data.items():
|
|
if (found_changeset_a and found_changeset_b):
|
|
break
|
|
if (not found_changeset_a):
|
|
if (self._args.changeset_range[0] == key[0]):
|
|
found_changeset_a = True
|
|
if (not found_changeset_b):
|
|
if (self._args.changeset_range[1] == key[0]):
|
|
found_changeset_b = True
|
|
|
|
if ((not found_changeset_a) and (not found_changeset_b)):
|
|
sys.exit("One of the specified changesets was not found. Please specify correct/complete commit IDs.")
|
|
else:
|
|
changeset_a = self._args.changeset_range[0]
|
|
changeset_b = self._args.changeset_range[1]
|
|
|
|
r_command = "Rscript ./plotter.R -o ./plots -a " + changeset_a[0:7] + " -b " + changeset_b[0:7]
|
|
|
|
print(r_command)
|
|
subprocess.check_call(r_command, shell=True)
|
|
|
|
|
|
def one_changeset_plot(self):
|
|
found_changeset = 0
|
|
non_amo_tracker = []
|
|
amo_tracker = []
|
|
ping_pong_tracker = []
|
|
for key, value in self._log_tracker._data.items():
|
|
if (key[0] == self._args.one_changeset):
|
|
found_changeset = 1
|
|
if (key[1] == "put.log"):
|
|
put_vals = value.latency
|
|
if (key[1] == "put_nbi.log"):
|
|
put_nbi_vals = value.latency
|
|
if (key[1] == "get.log"):
|
|
get_vals = value.latency
|
|
if (key[1] == "get_nbi.log"):
|
|
get_nbi_vals = value.latency
|
|
if (key[1] == "amo_add.log"):
|
|
amo_tracker.append({'op':'add',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "amo_add.log"):
|
|
amo_tracker.append({'op':'add',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "amo_cswap.log"):
|
|
amo_tracker.append({'op':'cswap',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "amo_fadd.log"):
|
|
amo_tracker.append({'op':'fadd',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "amo_fcswap.log"):
|
|
amo_tracker.append({'op':'fcswap',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "amo_fetch.log"):
|
|
amo_tracker.append({'op':'fetch',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "amo_finc.log"):
|
|
amo_tracker.append({'op':'finc',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "amo_inc.log"):
|
|
amo_tracker.append({'op':'inc',
|
|
'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
if (key[1] == "ping_pong.log"):
|
|
ping_pong_tracker.append({'latency': value.latency[0] if len(value.latency) > 0 else 0
|
|
})
|
|
|
|
|
|
if (not found_changeset):
|
|
sys.exit("The requested changeset was not found. Please specify correct/complete commit IDs.")
|
|
|
|
index = 0
|
|
for size in [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]:
|
|
non_amo_tracker.append({'size':size,
|
|
'put':put_vals[index],
|
|
'put_nbi':put_nbi_vals[index],
|
|
'get':get_vals[index],
|
|
'get_nbi':get_nbi_vals[index]
|
|
})
|
|
index = index + 1
|
|
|
|
# write results into a file:
|
|
non_amo_field_names= ['size','put','put_nbi','get','get_nbi']
|
|
amo_field_names= ['op','latency']
|
|
ping_pong_field_names= ['latency']
|
|
|
|
self.write_dict_to_file(non_amo_tracker, non_amo_field_names, "non_amo_one_changeset.csv")
|
|
self.write_dict_to_file(amo_tracker, amo_field_names, "amo_one_changeset.csv")
|
|
self.write_dict_to_file(ping_pong_tracker, ping_pong_field_names, "ping_pong_one_changeset.csv")
|
|
|
|
# call the R script with an option that tells it to plot figures for
|
|
r_command = "Rscript ./plotter.R -o ./plots -c " + self._args.one_changeset
|
|
|
|
print(r_command)
|
|
subprocess.check_call(r_command, shell=True)
|
|
|