Remove unused scripts from functional tests (#237)
[ROCm/rocshmem commit: 38a7820aa8]
Cette révision appartient à :
Aurelien Bouteiller
révisé par
GitHub
GitHub
Parent
282fc7fe71
révision
e607bfbb7b
@@ -1,363 +0,0 @@
|
||||
###############################################################################
|
||||
# Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to
|
||||
# deal in the Software without restriction, including without limitation the
|
||||
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
# sell copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
# IN THE SOFTWARE.
|
||||
###############################################################################
|
||||
|
||||
#!/usr/bin/python3
|
||||
|
||||
import argparse
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
###############################################################################
|
||||
############################ TEST SUITE VARIABLES #############################
|
||||
###############################################################################
|
||||
algorithms = {
|
||||
0: 'get',
|
||||
1: 'get_nbi',
|
||||
2: 'put',
|
||||
3: 'put_nbi',
|
||||
4: 'get_swarm',
|
||||
5: 'reduction',
|
||||
6: 'amo_fadd',
|
||||
7: 'amo_finc',
|
||||
8: 'amo_fetch',
|
||||
9: 'amo_fcswap',
|
||||
10: 'amo_add',
|
||||
11: 'amo_inc',
|
||||
12: 'amo_cswap',
|
||||
13: 'init',
|
||||
14: 'ping_pong',
|
||||
15: 'barrier',
|
||||
16: 'random',
|
||||
17: 'barrier_all',
|
||||
18: 'sync_all',
|
||||
19: 'sync_test',
|
||||
20: 'broadcast',
|
||||
21: 'collect',
|
||||
22: 'fcollect',
|
||||
23: 'all_to_all',
|
||||
24: 'all_to_alls'
|
||||
}
|
||||
|
||||
primitives = [0, 1, 2, 3]
|
||||
threaded = [4]
|
||||
atomics = [6, 7, 8, 9, 10, 11, 12]
|
||||
inits = [13]
|
||||
point_to_points = [14]
|
||||
collectives = [5, 15, 17, 18, 19, 20, 21, 22, 23, 24]
|
||||
|
||||
single_thread_algorithms = primitives + \
|
||||
atomics + \
|
||||
inits + \
|
||||
point_to_points + \
|
||||
[5] # reduction
|
||||
|
||||
multi_thread_algorithms = primitives + \
|
||||
threaded + \
|
||||
atomics + \
|
||||
inits + \
|
||||
point_to_points + \
|
||||
[5] # reduction
|
||||
|
||||
reverse_offload_algorithms = primitives + \
|
||||
threaded + \
|
||||
inits + \
|
||||
point_to_points + \
|
||||
[5]
|
||||
|
||||
###############################################################################
|
||||
############################## PARSER FUNCTIONS ###############################
|
||||
###############################################################################
|
||||
def parse_command_line():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Execute rocshmem integration tests.')
|
||||
|
||||
parser.add_argument('--mpirun_procs',
|
||||
dest='mpirun_procs',
|
||||
type=int,
|
||||
nargs='+',
|
||||
default=[2, 4, 8])
|
||||
|
||||
parser.add_argument('--mpirun_machines',
|
||||
dest='mpirun_machines',
|
||||
type=str,
|
||||
nargs='+',
|
||||
default=['ast-rocm1',
|
||||
'ast-rocm2',
|
||||
'ast-rocm3',
|
||||
'ast-rocm4'])
|
||||
|
||||
parser.add_argument('--library_build_config_type',
|
||||
dest='library_build_config_type',
|
||||
type=str,
|
||||
default='rc_single')
|
||||
|
||||
parser.add_argument('--client_binary_path',
|
||||
dest='client_binary_path',
|
||||
type=str,
|
||||
default=os.getcwd()+'/build/rocshmem_functional_tests')
|
||||
|
||||
parser.add_argument('--output_directory_path',
|
||||
dest='output_directory_path',
|
||||
type=str,
|
||||
default=os.getcwd()+'/build')
|
||||
|
||||
parser.add_argument('--number_threads',
|
||||
dest='number_threads',
|
||||
type=int,
|
||||
nargs='+',
|
||||
default=1)
|
||||
|
||||
parser.add_argument('--number_workgroups',
|
||||
dest='number_workgroups',
|
||||
type=int,
|
||||
nargs='+',
|
||||
default=1)
|
||||
|
||||
parser.add_argument('--workgroup_size',
|
||||
dest='workgroup_size',
|
||||
type=int,
|
||||
nargs='+',
|
||||
default=[64, 1024])
|
||||
|
||||
parser.add_argument('--workgroup_context_type',
|
||||
dest='workgroup_context_type',
|
||||
type=int,
|
||||
nargs='+',
|
||||
default=[0, 8])
|
||||
|
||||
parser.add_argument('--max_message_size',
|
||||
dest='max_message_size',
|
||||
type=int,
|
||||
default=1048576)
|
||||
|
||||
parser.add_argument('--algorithms',
|
||||
dest='algorithms',
|
||||
type=int,
|
||||
nargs='*',
|
||||
default=None)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
def convert_arguments_to_dictionary(args):
|
||||
return vars(args)
|
||||
|
||||
def determine_algos_from_library_config_type(config):
|
||||
if config['algorithms']:
|
||||
return config
|
||||
|
||||
thread_single = re.match('.*single.*', config['library_build_config_type'])
|
||||
|
||||
if thread_single:
|
||||
config['algorithms'] = single_thread_algorithms
|
||||
else:
|
||||
config['algorithms'] = multi_thread_algorithms
|
||||
|
||||
return config
|
||||
|
||||
def process_arguments(args):
|
||||
config = convert_arguments_to_dictionary(args)
|
||||
config = determine_algos_from_library_config_type(config)
|
||||
return config
|
||||
|
||||
###############################################################################
|
||||
######################## OPTION CONVERSION DICTIONARY #########################
|
||||
###############################################################################
|
||||
option_keyword_dictionary = {
|
||||
'mpirun_procs' : '-np',
|
||||
'mpirun_machines' : '--host',
|
||||
'number_threads' : '-t',
|
||||
'number_workgroups' : '-w',
|
||||
'max_message_size' : '-s',
|
||||
'algorithms' : '-a',
|
||||
'workgroup_size' : '-z',
|
||||
'workgroup_context_type' : '-x',
|
||||
}
|
||||
|
||||
###############################################################################
|
||||
########################## TEST GENERATION FUNCTIONS ##########################
|
||||
###############################################################################
|
||||
def separate_dictionaries(dictionary):
|
||||
library_build_config = {}
|
||||
mpirun = {}
|
||||
path = {}
|
||||
driver = {}
|
||||
|
||||
library_build_config_keys = [
|
||||
'library_build_config_type'
|
||||
]
|
||||
mpirun_keys = [
|
||||
'mpirun_machines',
|
||||
'mpirun_procs'
|
||||
]
|
||||
path_keys = [
|
||||
'output_directory_path',
|
||||
'client_binary_path'
|
||||
]
|
||||
|
||||
for key, value in dictionary.items():
|
||||
if key in library_build_config_keys:
|
||||
library_build_config[key] = value
|
||||
elif key in mpirun_keys:
|
||||
mpirun[key] = value
|
||||
elif key in path_keys:
|
||||
path[key] = value
|
||||
else:
|
||||
driver[key] = value
|
||||
|
||||
return library_build_config, mpirun, path, driver
|
||||
|
||||
def convert_dict_values_to_list_of_lists(dictionary):
|
||||
values = dictionary.values()
|
||||
list_of_lists = []
|
||||
for element in values:
|
||||
if isinstance(element, list):
|
||||
list_of_lists.append(element)
|
||||
else:
|
||||
single_element_list = [element]
|
||||
list_of_lists.append(single_element_list)
|
||||
return list_of_lists
|
||||
|
||||
def cartesian_product(list_of_lists):
|
||||
return list(itertools.product(*list_of_lists))
|
||||
|
||||
def create_config_combinations(dictionary):
|
||||
keys = []
|
||||
for k,v in dictionary.items():
|
||||
keys.append(k)
|
||||
|
||||
values_list_of_lists = convert_dict_values_to_list_of_lists(dictionary)
|
||||
combinations = cartesian_product(values_list_of_lists)
|
||||
return keys, combinations
|
||||
|
||||
def create_all_mpirun_machine_combinations(mpirun_dict):
|
||||
mpirun_machine_combinations = []
|
||||
number_of_machines = len(mpirun_dict['mpirun_machines'])
|
||||
|
||||
for i in range(1, number_of_machines + 1):
|
||||
combinations_object = \
|
||||
itertools.combinations(mpirun_dict['mpirun_machines'], i)
|
||||
combinations_list = list(combinations_object)
|
||||
mpirun_machine_combinations += combinations_list
|
||||
|
||||
machine_combo_strings = stringify_machines(mpirun_machine_combinations)
|
||||
mpirun_dict['mpirun_machines'] = machine_combo_strings
|
||||
|
||||
return mpirun_dict
|
||||
|
||||
def stringify_machines(machine_list_of_tuples):
|
||||
list_of_strings = []
|
||||
for tup in machine_list_of_tuples:
|
||||
list_of_strings.append(','.join(map(str, tup)))
|
||||
return list_of_strings
|
||||
|
||||
def stringify_config(keys, combo_list):
|
||||
string_list = []
|
||||
for combo_tuple in combo_list:
|
||||
one_string = ''
|
||||
for position in range(len(keys)):
|
||||
substr = '{key} {combo}'.format(key=keys[position],
|
||||
combo=combo_tuple[position])
|
||||
one_string += substr + ' '
|
||||
string_list.append(one_string)
|
||||
return string_list
|
||||
|
||||
def generate_test_commands(config_templates):
|
||||
(build_config, mpirun, path, driver) = \
|
||||
separate_dictionaries(config_templates)
|
||||
|
||||
driver_keys, driver_combos = create_config_combinations(driver)
|
||||
driver_commands = stringify_config(driver_keys, driver_combos)
|
||||
|
||||
mpirun = create_all_mpirun_machine_combinations(mpirun)
|
||||
mpirun_keys, mpirun_combos = create_config_combinations(mpirun)
|
||||
mpirun_commands = stringify_config(mpirun_keys, mpirun_combos)
|
||||
|
||||
commands = []
|
||||
for mpirun_command in mpirun_commands:
|
||||
for driver_command in driver_commands:
|
||||
mpirun_bin = 'mpirun '
|
||||
driver_bin = path['client_binary_path'] + ' '
|
||||
command = mpirun_bin + \
|
||||
mpirun_command + \
|
||||
driver_bin + \
|
||||
driver_command
|
||||
|
||||
commands.append(command)
|
||||
|
||||
return commands
|
||||
|
||||
def replace_option_keywords(commands):
|
||||
altered_commands = []
|
||||
for command in commands:
|
||||
for k,v in option_keyword_dictionary.items():
|
||||
command = command.replace(k, v)
|
||||
altered_commands.append(command)
|
||||
return altered_commands
|
||||
|
||||
def generate_tests():
|
||||
command_line_arguments = parse_command_line()
|
||||
config_template = process_arguments(command_line_arguments)
|
||||
commands = generate_test_commands(config_template)
|
||||
commands = replace_option_keywords(commands)
|
||||
return commands
|
||||
|
||||
###############################################################################
|
||||
############################ FILE HELPER FUNCTIONS ############################
|
||||
###############################################################################
|
||||
def open_output_file(filepath):
|
||||
try:
|
||||
f = open(filepath, 'r')
|
||||
except IOError:
|
||||
sys.exit('failed to open: ' + filepath)
|
||||
return f
|
||||
|
||||
def create_file_name():
|
||||
return
|
||||
|
||||
def write_output():
|
||||
return
|
||||
|
||||
###############################################################################
|
||||
########################## TEST EXECUTION FUNCTIONS ###########################
|
||||
###############################################################################
|
||||
def single_test(test):
|
||||
try:
|
||||
subprocess.run(test, shell=True, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(e)
|
||||
return
|
||||
|
||||
def all_tests():
|
||||
for test in generate_tests():
|
||||
print(test)
|
||||
#single_test(test)
|
||||
return
|
||||
|
||||
###############################################################################
|
||||
############################## SCRIPT MAIN BODY ###############################
|
||||
###############################################################################
|
||||
all_tests()
|
||||
@@ -1,139 +0,0 @@
|
||||
###############################################################################
|
||||
# Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
|
||||
#
|
||||
# SPDX-License-Identifier: MIT
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to
|
||||
# deal in the Software without restriction, including without limitation the
|
||||
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
# sell copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||
# IN THE SOFTWARE.
|
||||
###############################################################################
|
||||
|
||||
#!/bin/bash
|
||||
###############################################
|
||||
# Script : shmem_allLib_build_and_test.sh
|
||||
# Description : the script is to build and execute test for
|
||||
# given libraries as parameter to the script.
|
||||
# the script used in math_ci jenkins pipeline
|
||||
# version 1.0 (Version 1)
|
||||
# Assumption : the script is placed under clients/functional_tests
|
||||
###############################################
|
||||
echo ""
|
||||
echo ""
|
||||
echo "Starting script .."$0
|
||||
echo ""
|
||||
#check parameters. parameter of libs that need to be built and tested will be passed
|
||||
if [ $# -ne 1 ]
|
||||
then
|
||||
echo "Invalid Options for $0 ";
|
||||
echo "Usage : $0 <liblist> "
|
||||
echo "example : $0 RC_SINGLE,RC_MULTI "
|
||||
exit 1
|
||||
fi
|
||||
|
||||
liblist=$1
|
||||
tliblist=`echo $liblist | sed 's/,/ /g'` # replace , with space, so that can be passed for loop
|
||||
echo "Parameters passed ==> $tliblist"
|
||||
echo
|
||||
|
||||
for libi in $tliblist
|
||||
do
|
||||
echo
|
||||
echo "==> Processing Lib...$libi"
|
||||
case $libi in
|
||||
RC_SINGLE)
|
||||
libnm=rc_single
|
||||
libBuildDir=$libnm"_build"
|
||||
threadType="single_thread"
|
||||
;;
|
||||
RC_MULTI)
|
||||
libnm=rc_multi
|
||||
libBuildDir=$libnm"_build"
|
||||
threadType="multi_thread"
|
||||
;;
|
||||
RC_MULTI_WF_COAL)
|
||||
libnm=rc_multi_wf_coal
|
||||
libBuildDir=$libnm"_build"
|
||||
threadType="multi_thread"
|
||||
;;
|
||||
DC_SINGLE)
|
||||
libnm=dc_single
|
||||
libBuildDir=$libnm"_build"
|
||||
threadType="single_thread"
|
||||
;;
|
||||
DC_MULTI)
|
||||
libnm=dc_multi
|
||||
libBuildDir=$libnm"_build"
|
||||
threadType="multi_thread"
|
||||
;;
|
||||
RO_NET)
|
||||
libnm=ro_net
|
||||
libBuildDir=$libnm"_build"
|
||||
threadType="single_thread"
|
||||
;;
|
||||
*)
|
||||
echo "Unable to find the option for $libi. Plesae check shmem admin"
|
||||
echo "** warning ** Skipping the process for $libi "
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$libnm" != "" ] #process only if libname found
|
||||
then
|
||||
echo "+-------------------------------------------------------------------------------------------------+"
|
||||
echo `date +%Y%m%d%H%M%S`" ==> Start | $threadType - build_configs/$libnm <=="
|
||||
echo "starting with params==> $libnm ; $libBuildDir ; $threadType"
|
||||
echo
|
||||
|
||||
echo "Library build at ==> "$libBuildDir
|
||||
|
||||
mkdir $libBuildDir
|
||||
cd $libBuildDir
|
||||
../scripts/build_configs/$libnm # from build directory generating the build
|
||||
|
||||
# test exeuction based on lib
|
||||
if [ "$libnm" == "ro_net" ]
|
||||
then
|
||||
ROCSHMEM_RO=1
|
||||
ROC_NET_CPU_QUEUE=1
|
||||
UCX_TLS=rc
|
||||
#echo $ROCSHMEM_RO"--"$ROC_NET_CPU_QUEUE "--"$UCX_TLS
|
||||
../scripts/functional_tests/driver.sh tests/functional_tests/rocshmem_functional_tests $threadType .
|
||||
else
|
||||
../scripts/functional_tests/driver.sh tests/functional_tests/rocshmem_functional_tests $threadType .
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "Lib $libnm functional Test exited with ** error $? **"
|
||||
else
|
||||
echo "Lib $libnm sucessfull "
|
||||
fi
|
||||
|
||||
cd ../ # move to base directory
|
||||
|
||||
echo
|
||||
echo `date +%Y%m%d%H%M%S`" ==> End | $threadType - build_configs/$libnm <=="
|
||||
echo "+-------------------------------------------------------------------------------------------------+"
|
||||
fi
|
||||
|
||||
libnm="" #reset the parameter after completion of build
|
||||
libBuildDir=""
|
||||
threadType=""
|
||||
done
|
||||
echo
|
||||
echo "Script execution ==> $0 <== is done"
|
||||
echo
|
||||
|
||||
Référencer dans un nouveau ticket
Bloquer un utilisateur