AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove unused scripts from functional tests (#237)

Browse Source 
[ROCm/rocshmem commit: 38a7820aa8]
This commit is contained in:
Aurelien Bouteiller
2025-09-12 10:14:33 -04:00
committed by GitHub
parent 282fc7fe71
commit e607bfbb7b
2 changed files with 0 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocshmem/scripts/functional_tests/driver.py
-363
View File
@@ -1,363 +0,0 @@
###############################################################################
# Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
#
# SPDX-License-Identifier: MIT
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
###############################################################################
#!/usr/bin/python3
import argparse
import itertools
import os
import re
import subprocess
import sys
###############################################################################
############################ TEST SUITE VARIABLES #############################
###############################################################################
algorithms = {
0: 'get',
1: 'get_nbi',
2: 'put',
3: 'put_nbi',
4: 'get_swarm',
5: 'reduction',
6: 'amo_fadd',
7: 'amo_finc',
8: 'amo_fetch',
9: 'amo_fcswap',
10: 'amo_add',
11: 'amo_inc',
12: 'amo_cswap',
13: 'init',
14: 'ping_pong',
15: 'barrier',
16: 'random',
17: 'barrier_all',
18: 'sync_all',
19: 'sync_test',
20: 'broadcast',
21: 'collect',
22: 'fcollect',
23: 'all_to_all',
24: 'all_to_alls'
}
primitives = [0, 1, 2, 3]
threaded = [4]
atomics = [6, 7, 8, 9, 10, 11, 12]
inits = [13]
point_to_points = [14]
collectives = [5, 15, 17, 18, 19, 20, 21, 22, 23, 24]
single_thread_algorithms = primitives + \
atomics + \
inits + \
point_to_points + \
[5] # reduction
multi_thread_algorithms = primitives + \
threaded + \
atomics + \
inits + \
point_to_points + \
[5] # reduction
reverse_offload_algorithms = primitives + \
threaded + \
inits + \
point_to_points + \
[5]
###############################################################################
############################## PARSER FUNCTIONS ###############################
###############################################################################
def parse_command_line():
parser = argparse.ArgumentParser(
description='Execute rocshmem integration tests.')
parser.add_argument('--mpirun_procs',
dest='mpirun_procs',
type=int,
nargs='+',
default=[2, 4, 8])
parser.add_argument('--mpirun_machines',
dest='mpirun_machines',
type=str,
nargs='+',
default=['ast-rocm1',
'ast-rocm2',
'ast-rocm3',
'ast-rocm4'])
parser.add_argument('--library_build_config_type',
dest='library_build_config_type',
type=str,
default='rc_single')
parser.add_argument('--client_binary_path',
dest='client_binary_path',
type=str,
default=os.getcwd()+'/build/rocshmem_functional_tests')
parser.add_argument('--output_directory_path',
dest='output_directory_path',
type=str,
default=os.getcwd()+'/build')
parser.add_argument('--number_threads',
dest='number_threads',
type=int,
nargs='+',
default=1)
parser.add_argument('--number_workgroups',
dest='number_workgroups',
type=int,
nargs='+',
default=1)
parser.add_argument('--workgroup_size',
dest='workgroup_size',
type=int,
nargs='+',
default=[64, 1024])
parser.add_argument('--workgroup_context_type',
dest='workgroup_context_type',
type=int,
nargs='+',
default=[0, 8])
parser.add_argument('--max_message_size',
dest='max_message_size',
type=int,
default=1048576)
parser.add_argument('--algorithms',
dest='algorithms',
type=int,
nargs='*',
default=None)
return parser.parse_args()
def convert_arguments_to_dictionary(args):
return vars(args)
def determine_algos_from_library_config_type(config):
if config['algorithms']:
return config
thread_single = re.match('.*single.*', config['library_build_config_type'])
if thread_single:
config['algorithms'] = single_thread_algorithms
else:
config['algorithms'] = multi_thread_algorithms
return config
def process_arguments(args):
config = convert_arguments_to_dictionary(args)
config = determine_algos_from_library_config_type(config)
return config
###############################################################################
######################## OPTION CONVERSION DICTIONARY #########################
###############################################################################
option_keyword_dictionary = {
'mpirun_procs' : '-np',
'mpirun_machines' : '--host',
'number_threads' : '-t',
'number_workgroups' : '-w',
'max_message_size' : '-s',
'algorithms' : '-a',
'workgroup_size' : '-z',
'workgroup_context_type' : '-x',
}
###############################################################################
########################## TEST GENERATION FUNCTIONS ##########################
###############################################################################
def separate_dictionaries(dictionary):
library_build_config = {}
mpirun = {}
path = {}
driver = {}
library_build_config_keys = [
'library_build_config_type'
]
mpirun_keys = [
'mpirun_machines',
'mpirun_procs'
]
path_keys = [
'output_directory_path',
'client_binary_path'
]
for key, value in dictionary.items():
if key in library_build_config_keys:
library_build_config[key] = value
elif key in mpirun_keys:
mpirun[key] = value
elif key in path_keys:
path[key] = value
else:
driver[key] = value
return library_build_config, mpirun, path, driver
def convert_dict_values_to_list_of_lists(dictionary):
values = dictionary.values()
list_of_lists = []
for element in values:
if isinstance(element, list):
list_of_lists.append(element)
else:
single_element_list = [element]
list_of_lists.append(single_element_list)
return list_of_lists
def cartesian_product(list_of_lists):
return list(itertools.product(*list_of_lists))
def create_config_combinations(dictionary):
keys = []
for k,v in dictionary.items():
keys.append(k)
values_list_of_lists = convert_dict_values_to_list_of_lists(dictionary)
combinations = cartesian_product(values_list_of_lists)
return keys, combinations
def create_all_mpirun_machine_combinations(mpirun_dict):
mpirun_machine_combinations = []
number_of_machines = len(mpirun_dict['mpirun_machines'])
for i in range(1, number_of_machines + 1):
combinations_object = \
itertools.combinations(mpirun_dict['mpirun_machines'], i)
combinations_list = list(combinations_object)
mpirun_machine_combinations += combinations_list
machine_combo_strings = stringify_machines(mpirun_machine_combinations)
mpirun_dict['mpirun_machines'] = machine_combo_strings
return mpirun_dict
def stringify_machines(machine_list_of_tuples):
list_of_strings = []
for tup in machine_list_of_tuples:
list_of_strings.append(','.join(map(str, tup)))
return list_of_strings
def stringify_config(keys, combo_list):
string_list = []
for combo_tuple in combo_list:
one_string = ''
for position in range(len(keys)):
substr = '{key} {combo}'.format(key=keys[position],
combo=combo_tuple[position])
one_string += substr + ' '
string_list.append(one_string)
return string_list
def generate_test_commands(config_templates):
(build_config, mpirun, path, driver) = \
separate_dictionaries(config_templates)
driver_keys, driver_combos = create_config_combinations(driver)
driver_commands = stringify_config(driver_keys, driver_combos)
mpirun = create_all_mpirun_machine_combinations(mpirun)
mpirun_keys, mpirun_combos = create_config_combinations(mpirun)
mpirun_commands = stringify_config(mpirun_keys, mpirun_combos)
commands = []
for mpirun_command in mpirun_commands:
for driver_command in driver_commands:
mpirun_bin = 'mpirun '
driver_bin = path['client_binary_path'] + ' '
command = mpirun_bin + \
mpirun_command + \
driver_bin + \
driver_command
commands.append(command)
return commands
def replace_option_keywords(commands):
altered_commands = []
for command in commands:
for k,v in option_keyword_dictionary.items():
command = command.replace(k, v)
altered_commands.append(command)
return altered_commands
def generate_tests():
command_line_arguments = parse_command_line()
config_template = process_arguments(command_line_arguments)
commands = generate_test_commands(config_template)
commands = replace_option_keywords(commands)
return commands
###############################################################################
############################ FILE HELPER FUNCTIONS ############################
###############################################################################
def open_output_file(filepath):
try:
f = open(filepath, 'r')
except IOError:
sys.exit('failed to open: ' + filepath)
return f
def create_file_name():
return
def write_output():
return
###############################################################################
########################## TEST EXECUTION FUNCTIONS ###########################
###############################################################################
def single_test(test):
try:
subprocess.run(test, shell=True, check=True)
except subprocess.CalledProcessError as e:
print(e)
return
def all_tests():
for test in generate_tests():
print(test)
#single_test(test)
return
###############################################################################
############################## SCRIPT MAIN BODY ###############################
###############################################################################
all_tests()
projects/rocshmem/scripts/functional_tests/shmem_allLib_build_and_test.sh
-139
View File
@@ -1,139 +0,0 @@
###############################################################################
# Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
#
# SPDX-License-Identifier: MIT
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
###############################################################################
#!/bin/bash
###############################################
# Script : shmem_allLib_build_and_test.sh
# Description : the script is to build and execute test for
# given libraries as parameter to the script.
# the script used in math_ci jenkins pipeline
# version 1.0 (Version 1)
# Assumption : the script is placed under clients/functional_tests
###############################################
echo ""
echo ""
echo "Starting script .."$0
echo ""
#check parameters. parameter of libs that need to be built and tested will be passed
if [ $# -ne 1 ]
then
echo "Invalid Options for $0 ";
echo "Usage : $0 <liblist> "
echo "example : $0 RC_SINGLE,RC_MULTI "
exit 1
fi
liblist=$1
tliblist=`echo $liblist | sed 's/,/ /g'` # replace , with space, so that can be passed for loop
echo "Parameters passed ==> $tliblist"
echo
for libi in $tliblist
do
echo
echo "==> Processing Lib...$libi"
case $libi in
RC_SINGLE)
libnm=rc_single
libBuildDir=$libnm"_build"
threadType="single_thread"
;;
RC_MULTI)
libnm=rc_multi
libBuildDir=$libnm"_build"
threadType="multi_thread"
;;
RC_MULTI_WF_COAL)
libnm=rc_multi_wf_coal
libBuildDir=$libnm"_build"
threadType="multi_thread"
;;
DC_SINGLE)
libnm=dc_single
libBuildDir=$libnm"_build"
threadType="single_thread"
;;
DC_MULTI)
libnm=dc_multi
libBuildDir=$libnm"_build"
threadType="multi_thread"
;;
RO_NET)
libnm=ro_net
libBuildDir=$libnm"_build"
threadType="single_thread"
;;
*)
echo "Unable to find the option for $libi. Plesae check shmem admin"
echo "** warning ** Skipping the process for $libi "
;;
esac
if [ "$libnm" != "" ] #process only if libname found
then
echo "+-------------------------------------------------------------------------------------------------+"
echo `date +%Y%m%d%H%M%S`" ==> Start | $threadType - build_configs/$libnm <=="
echo "starting with params==> $libnm ; $libBuildDir ; $threadType"
echo
echo "Library build at ==> "$libBuildDir
mkdir $libBuildDir
cd $libBuildDir
../scripts/build_configs/$libnm # from build directory generating the build
# test exeuction based on lib
if [ "$libnm" == "ro_net" ]
then
ROCSHMEM_RO=1
ROC_NET_CPU_QUEUE=1
UCX_TLS=rc
#echo $ROCSHMEM_RO"--"$ROC_NET_CPU_QUEUE "--"$UCX_TLS
../scripts/functional_tests/driver.sh tests/functional_tests/rocshmem_functional_tests $threadType .
else
../scripts/functional_tests/driver.sh tests/functional_tests/rocshmem_functional_tests $threadType .
fi
if [ $? -ne 0 ]
then
echo "Lib $libnm functional Test exited with ** error $? **"
else
echo "Lib $libnm sucessfull "
fi
cd ../ # move to base directory
echo
echo `date +%Y%m%d%H%M%S`" ==> End | $threadType - build_configs/$libnm <=="
echo "+-------------------------------------------------------------------------------------------------+"
fi
libnm="" #reset the parameter after completion of build
libBuildDir=""
threadType=""
done
echo
echo "Script execution ==> $0 <== is done"
echo