diff --git a/projects/rocshmem/scripts/functional_tests/driver.py b/projects/rocshmem/scripts/functional_tests/driver.py deleted file mode 100755 index 95ae4fe5de..0000000000 --- a/projects/rocshmem/scripts/functional_tests/driver.py +++ /dev/null @@ -1,363 +0,0 @@ -############################################################################### -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -############################################################################### - -#!/usr/bin/python3 - -import argparse -import itertools -import os -import re -import subprocess -import sys - -############################################################################### -############################ TEST SUITE VARIABLES ############################# -############################################################################### -algorithms = { - 0: 'get', - 1: 'get_nbi', - 2: 'put', - 3: 'put_nbi', - 4: 'get_swarm', - 5: 'reduction', - 6: 'amo_fadd', - 7: 'amo_finc', - 8: 'amo_fetch', - 9: 'amo_fcswap', - 10: 'amo_add', - 11: 'amo_inc', - 12: 'amo_cswap', - 13: 'init', - 14: 'ping_pong', - 15: 'barrier', - 16: 'random', - 17: 'barrier_all', - 18: 'sync_all', - 19: 'sync_test', - 20: 'broadcast', - 21: 'collect', - 22: 'fcollect', - 23: 'all_to_all', - 24: 'all_to_alls' -} - -primitives = [0, 1, 2, 3] -threaded = [4] -atomics = [6, 7, 8, 9, 10, 11, 12] -inits = [13] -point_to_points = [14] -collectives = [5, 15, 17, 18, 19, 20, 21, 22, 23, 24] - -single_thread_algorithms = primitives + \ - atomics + \ - inits + \ - point_to_points + \ - [5] # reduction - -multi_thread_algorithms = primitives + \ - threaded + \ - atomics + \ - inits + \ - point_to_points + \ - [5] # reduction - -reverse_offload_algorithms = primitives + \ - threaded + \ - inits + \ - point_to_points + \ - [5] - -############################################################################### -############################## PARSER FUNCTIONS ############################### -############################################################################### -def parse_command_line(): - parser = argparse.ArgumentParser( - description='Execute rocshmem integration tests.') - - parser.add_argument('--mpirun_procs', - dest='mpirun_procs', - type=int, - nargs='+', - default=[2, 4, 8]) - - parser.add_argument('--mpirun_machines', - dest='mpirun_machines', - type=str, - nargs='+', - default=['ast-rocm1', - 'ast-rocm2', - 'ast-rocm3', - 'ast-rocm4']) - - parser.add_argument('--library_build_config_type', - dest='library_build_config_type', - type=str, - default='rc_single') - - parser.add_argument('--client_binary_path', - dest='client_binary_path', - type=str, - default=os.getcwd()+'/build/rocshmem_functional_tests') - - parser.add_argument('--output_directory_path', - dest='output_directory_path', - type=str, - default=os.getcwd()+'/build') - - parser.add_argument('--number_threads', - dest='number_threads', - type=int, - nargs='+', - default=1) - - parser.add_argument('--number_workgroups', - dest='number_workgroups', - type=int, - nargs='+', - default=1) - - parser.add_argument('--workgroup_size', - dest='workgroup_size', - type=int, - nargs='+', - default=[64, 1024]) - - parser.add_argument('--workgroup_context_type', - dest='workgroup_context_type', - type=int, - nargs='+', - default=[0, 8]) - - parser.add_argument('--max_message_size', - dest='max_message_size', - type=int, - default=1048576) - - parser.add_argument('--algorithms', - dest='algorithms', - type=int, - nargs='*', - default=None) - - return parser.parse_args() - -def convert_arguments_to_dictionary(args): - return vars(args) - -def determine_algos_from_library_config_type(config): - if config['algorithms']: - return config - - thread_single = re.match('.*single.*', config['library_build_config_type']) - - if thread_single: - config['algorithms'] = single_thread_algorithms - else: - config['algorithms'] = multi_thread_algorithms - - return config - -def process_arguments(args): - config = convert_arguments_to_dictionary(args) - config = determine_algos_from_library_config_type(config) - return config - -############################################################################### -######################## OPTION CONVERSION DICTIONARY ######################### -############################################################################### -option_keyword_dictionary = { - 'mpirun_procs' : '-np', - 'mpirun_machines' : '--host', - 'number_threads' : '-t', - 'number_workgroups' : '-w', - 'max_message_size' : '-s', - 'algorithms' : '-a', - 'workgroup_size' : '-z', - 'workgroup_context_type' : '-x', -} - -############################################################################### -########################## TEST GENERATION FUNCTIONS ########################## -############################################################################### -def separate_dictionaries(dictionary): - library_build_config = {} - mpirun = {} - path = {} - driver = {} - - library_build_config_keys = [ - 'library_build_config_type' - ] - mpirun_keys = [ - 'mpirun_machines', - 'mpirun_procs' - ] - path_keys = [ - 'output_directory_path', - 'client_binary_path' - ] - - for key, value in dictionary.items(): - if key in library_build_config_keys: - library_build_config[key] = value - elif key in mpirun_keys: - mpirun[key] = value - elif key in path_keys: - path[key] = value - else: - driver[key] = value - - return library_build_config, mpirun, path, driver - -def convert_dict_values_to_list_of_lists(dictionary): - values = dictionary.values() - list_of_lists = [] - for element in values: - if isinstance(element, list): - list_of_lists.append(element) - else: - single_element_list = [element] - list_of_lists.append(single_element_list) - return list_of_lists - -def cartesian_product(list_of_lists): - return list(itertools.product(*list_of_lists)) - -def create_config_combinations(dictionary): - keys = [] - for k,v in dictionary.items(): - keys.append(k) - - values_list_of_lists = convert_dict_values_to_list_of_lists(dictionary) - combinations = cartesian_product(values_list_of_lists) - return keys, combinations - -def create_all_mpirun_machine_combinations(mpirun_dict): - mpirun_machine_combinations = [] - number_of_machines = len(mpirun_dict['mpirun_machines']) - - for i in range(1, number_of_machines + 1): - combinations_object = \ - itertools.combinations(mpirun_dict['mpirun_machines'], i) - combinations_list = list(combinations_object) - mpirun_machine_combinations += combinations_list - - machine_combo_strings = stringify_machines(mpirun_machine_combinations) - mpirun_dict['mpirun_machines'] = machine_combo_strings - - return mpirun_dict - -def stringify_machines(machine_list_of_tuples): - list_of_strings = [] - for tup in machine_list_of_tuples: - list_of_strings.append(','.join(map(str, tup))) - return list_of_strings - -def stringify_config(keys, combo_list): - string_list = [] - for combo_tuple in combo_list: - one_string = '' - for position in range(len(keys)): - substr = '{key} {combo}'.format(key=keys[position], - combo=combo_tuple[position]) - one_string += substr + ' ' - string_list.append(one_string) - return string_list - -def generate_test_commands(config_templates): - (build_config, mpirun, path, driver) = \ - separate_dictionaries(config_templates) - - driver_keys, driver_combos = create_config_combinations(driver) - driver_commands = stringify_config(driver_keys, driver_combos) - - mpirun = create_all_mpirun_machine_combinations(mpirun) - mpirun_keys, mpirun_combos = create_config_combinations(mpirun) - mpirun_commands = stringify_config(mpirun_keys, mpirun_combos) - - commands = [] - for mpirun_command in mpirun_commands: - for driver_command in driver_commands: - mpirun_bin = 'mpirun ' - driver_bin = path['client_binary_path'] + ' ' - command = mpirun_bin + \ - mpirun_command + \ - driver_bin + \ - driver_command - - commands.append(command) - - return commands - -def replace_option_keywords(commands): - altered_commands = [] - for command in commands: - for k,v in option_keyword_dictionary.items(): - command = command.replace(k, v) - altered_commands.append(command) - return altered_commands - -def generate_tests(): - command_line_arguments = parse_command_line() - config_template = process_arguments(command_line_arguments) - commands = generate_test_commands(config_template) - commands = replace_option_keywords(commands) - return commands - -############################################################################### -############################ FILE HELPER FUNCTIONS ############################ -############################################################################### -def open_output_file(filepath): - try: - f = open(filepath, 'r') - except IOError: - sys.exit('failed to open: ' + filepath) - return f - -def create_file_name(): - return - -def write_output(): - return - -############################################################################### -########################## TEST EXECUTION FUNCTIONS ########################### -############################################################################### -def single_test(test): - try: - subprocess.run(test, shell=True, check=True) - except subprocess.CalledProcessError as e: - print(e) - return - -def all_tests(): - for test in generate_tests(): - print(test) - #single_test(test) - return - -############################################################################### -############################## SCRIPT MAIN BODY ############################### -############################################################################### -all_tests() diff --git a/projects/rocshmem/scripts/functional_tests/shmem_allLib_build_and_test.sh b/projects/rocshmem/scripts/functional_tests/shmem_allLib_build_and_test.sh deleted file mode 100755 index 7f54a946c4..0000000000 --- a/projects/rocshmem/scripts/functional_tests/shmem_allLib_build_and_test.sh +++ /dev/null @@ -1,139 +0,0 @@ -############################################################################### -# Copyright (c) Advanced Micro Devices, Inc. All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -############################################################################### - -#!/bin/bash -############################################### -# Script : shmem_allLib_build_and_test.sh -# Description : the script is to build and execute test for -# given libraries as parameter to the script. -# the script used in math_ci jenkins pipeline -# version 1.0 (Version 1) -# Assumption : the script is placed under clients/functional_tests -############################################### -echo "" -echo "" -echo "Starting script .."$0 -echo "" -#check parameters. parameter of libs that need to be built and tested will be passed -if [ $# -ne 1 ] -then - echo "Invalid Options for $0 "; - echo "Usage : $0 " - echo "example : $0 RC_SINGLE,RC_MULTI " - exit 1 -fi - -liblist=$1 -tliblist=`echo $liblist | sed 's/,/ /g'` # replace , with space, so that can be passed for loop -echo "Parameters passed ==> $tliblist" -echo - -for libi in $tliblist -do - echo - echo "==> Processing Lib...$libi" - case $libi in - RC_SINGLE) - libnm=rc_single - libBuildDir=$libnm"_build" - threadType="single_thread" - ;; - RC_MULTI) - libnm=rc_multi - libBuildDir=$libnm"_build" - threadType="multi_thread" - ;; - RC_MULTI_WF_COAL) - libnm=rc_multi_wf_coal - libBuildDir=$libnm"_build" - threadType="multi_thread" - ;; - DC_SINGLE) - libnm=dc_single - libBuildDir=$libnm"_build" - threadType="single_thread" - ;; - DC_MULTI) - libnm=dc_multi - libBuildDir=$libnm"_build" - threadType="multi_thread" - ;; - RO_NET) - libnm=ro_net - libBuildDir=$libnm"_build" - threadType="single_thread" - ;; - *) - echo "Unable to find the option for $libi. Plesae check shmem admin" - echo "** warning ** Skipping the process for $libi " - ;; - esac - - if [ "$libnm" != "" ] #process only if libname found - then - echo "+-------------------------------------------------------------------------------------------------+" - echo `date +%Y%m%d%H%M%S`" ==> Start | $threadType - build_configs/$libnm <==" - echo "starting with params==> $libnm ; $libBuildDir ; $threadType" - echo - - echo "Library build at ==> "$libBuildDir - - mkdir $libBuildDir - cd $libBuildDir - ../scripts/build_configs/$libnm # from build directory generating the build - - # test exeuction based on lib - if [ "$libnm" == "ro_net" ] - then - ROCSHMEM_RO=1 - ROC_NET_CPU_QUEUE=1 - UCX_TLS=rc - #echo $ROCSHMEM_RO"--"$ROC_NET_CPU_QUEUE "--"$UCX_TLS - ../scripts/functional_tests/driver.sh tests/functional_tests/rocshmem_functional_tests $threadType . - else - ../scripts/functional_tests/driver.sh tests/functional_tests/rocshmem_functional_tests $threadType . - fi - - if [ $? -ne 0 ] - then - echo "Lib $libnm functional Test exited with ** error $? **" - else - echo "Lib $libnm sucessfull " - fi - - cd ../ # move to base directory - - echo - echo `date +%Y%m%d%H%M%S`" ==> End | $threadType - build_configs/$libnm <==" - echo "+-------------------------------------------------------------------------------------------------+" - fi - - libnm="" #reset the parameter after completion of build - libBuildDir="" - threadType="" -done -echo -echo "Script execution ==> $0 <== is done" -echo -