diff --git a/projects/rocprofiler-systems/examples/openmp/CMakeLists.txt b/projects/rocprofiler-systems/examples/openmp/CMakeLists.txt index 79d167d07e..3d5388007c 100644 --- a/projects/rocprofiler-systems/examples/openmp/CMakeLists.txt +++ b/projects/rocprofiler-systems/examples/openmp/CMakeLists.txt @@ -98,8 +98,6 @@ set(DEFAULT_GPU_TARGETS "gfx906" "gfx908" "gfx90a" - "gfx940" - "gfx941" "gfx942" "gfx950" "gfx1030" diff --git a/projects/rocprofiler-systems/examples/openmp/external/CMakeLists.txt b/projects/rocprofiler-systems/examples/openmp/external/CMakeLists.txt index dad3ece0a9..31c47147e2 100644 --- a/projects/rocprofiler-systems/examples/openmp/external/CMakeLists.txt +++ b/projects/rocprofiler-systems/examples/openmp/external/CMakeLists.txt @@ -7,8 +7,8 @@ # # * Currently, GNU's libgomp cannot be used to capture traces. Only LLVM's # LIBOMP can be used for trace capture -# * amdflang uses LIBOMP 201611 (5.0), but not all OpenMP 5.0 features are -# implemented. Only a subset of tests can be compiled. +# * As of implementation, amdflang uses LIBOMP 201611 (5.0), but not all +# OpenMP 5.0 features are implemented. Only a subset of tests can be compiled. # ----------------------------------------------------------------------------------------# set(OMPVV_USE_OFFLOAD_TESTS FALSE) @@ -40,54 +40,87 @@ rocprofiler_systems_checkout_git_submodule( ) set(ROCPROFSYS_OMPVV_SUBMODULE_DIR "${PROJECT_SOURCE_DIR}/external/ompvv") -set(ROCPROFSYS_OMPVV_SOURCE_DIR "${CMAKE_BINARY_DIR}/examples/openmp/external/ompvv") +set(ROCPROFSYS_OPENMP_EXTERNAL_DIR "${CMAKE_BINARY_DIR}/examples/openmp/external") +set(ROCPROFSYS_OMPVV_SOURCE_DIR "${ROCPROFSYS_OPENMP_EXTERNAL_DIR}/ompvv") set(OMPVV_MAKEDEF_FILE "sys/make/make.def") +set(OMPVV_BIN_DEST "${CMAKE_BINARY_DIR}") # ---------------------------------------------------------------------------------------# -# Helper functions +# Helper Function # ---------------------------------------------------------------------------------------# -# Updates the make.def FOFFLOADING flag -function(update_make_def_foffloading FOFFLOADING_FLAGS) - file(READ ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} MAKE_DEF_CONTENT) - - string( - REGEX REPLACE - "FOFFLOADING[ \t]*=[ \t]*-fopenmp[^\n\r]*" - "FOFFLOADING = ${FOFFLOADING_FLAGS}" - MAKE_DEF_CONTENT - "${MAKE_DEF_CONTENT}" - ) - - file(WRITE ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} "${MAKE_DEF_CONTENT}") -endfunction() - -# Individually compiles a given list of tests -function(compile_ompvv_tests TEST_LIST_VAR) - # OMPVV doesn't support building subsets of test folders - foreach(test IN LISTS ${TEST_LIST_VAR}) - rocprofiler_systems_message(STATUS "Compiling OMPVV test: ${test}") - +function(configure_ompvv_tests TEST_TYPE TEST_LIST_VAR) + # Precreate bin dir to avoid race condition + if(NOT EXISTS "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin") execute_process( + COMMAND + ${CMAKE_COMMAND} -E make_directory "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin" + ) + endif() + if(TEST_TYPE STREQUAL "HOST") + set(TARGET_PREFIX "openmp-vv-host") + set(FOFFLOADING_FLAGS "-fopenmp --offload-host-only") + elseif(TEST_TYPE STREQUAL "OFFLOAD") + set(TARGET_PREFIX "openmp-vv-offload") + set(FOFFLOADING_FLAGS "-fopenmp") + foreach(arch IN LISTS DEFAULT_GPU_TARGETS) + set(FOFFLOADING_FLAGS "${FOFFLOADING_FLAGS} --offload-arch=${arch}") + endforeach() + else() + rocprofiler_systems_message(FATAL_ERROR "Unknown TEST_TYPE - Only HOST and OFFLOAD supported") + endif() + + set(LOCAL_TEST_LIST "") + + foreach(TEST IN LISTS ${TEST_LIST_VAR}) + get_filename_component(TEST_NAME ${TEST} NAME_WE) + string(REPLACE "_" "-" TARGET_NAME ${TEST_NAME}) + set(TARGET_NAME "${TARGET_PREFIX}-${TARGET_NAME}") + + set(CUSTOM_FFLAGS "-lm ${FOFFLOADING_FLAGS}") + set(CUSTOM_FLINKFLAGS "-lm ${FOFFLOADING_FLAGS}") + + add_custom_command( + OUTPUT "${OMPVV_BIN_DEST}/${TARGET_NAME}" COMMAND make FC=${OMPVV_FC} DEVICE_TYPE=amd OMP_VERSION=${OMPVV_OPENMP_VERSION} NUM_THREADS_HOST=${OMPVV_NUM_THREADS_HOST} NUM_THREADS_DEVICE=${OMPVV_NUM_THREADS_DEVICE} - NUM_TEAMS_DEVICE=${OMPVV_NUM_TEAMS_DEVICE} "SOURCES=${test}" compile + NUM_TEAMS_DEVICE=${OMPVV_NUM_TEAMS_DEVICE} + "FOFFLOADING=${FOFFLOADING_FLAGS}" "FFLAGS=${CUSTOM_FFLAGS}" + "FLINKFLAGS=${CUSTOM_FLINKFLAGS}" "SOURCES=${TEST}" compile > + ${CMAKE_CURRENT_BINARY_DIR}/ompvv-compile-${TARGET_NAME}.log 2>&1 + COMMAND + ${CMAKE_COMMAND} -E copy_if_different + "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin/${TEST_NAME}.F90.o" + "${OMPVV_BIN_DEST}/${TARGET_NAME}" + COMMENT "Compiling OMPVV test: ${TEST_NAME}" WORKING_DIRECTORY ${ROCPROFSYS_OMPVV_SOURCE_DIR} - RESULT_VARIABLE OMPVV_BUILD_RESULT - OUTPUT_VARIABLE OMPVV_BUILD_OUTPUT - ERROR_VARIABLE OMPVV_BUILD_ERROR - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_STRIP_TRAILING_WHITESPACE + VERBATIM + DEPENDS + ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${TEST} + ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} ) - if(NOT OMPVV_BUILD_RESULT EQUAL 0) - rocprofiler_systems_message( - FATAL_ERROR "OMPVV build failed for ${test}: ${OMPVV_BUILD_ERROR}" - ) - endif() + list(APPEND LOCAL_TEST_LIST ${TARGET_NAME}) + + add_custom_target( + "${TARGET_NAME}-build" + ALL + DEPENDS "${OMPVV_BIN_DEST}/${TARGET_NAME}" + ) + add_executable("${TARGET_NAME}-exec" IMPORTED GLOBAL) + set_target_properties( + "${TARGET_NAME}-exec" + PROPERTIES IMPORTED_LOCATION "${OMPVV_BIN_DEST}/${TARGET_NAME}" + ) endforeach() + + if(TEST_TYPE STREQUAL "HOST") + set(ROCPROFSYS_OMPVV_HOST_TESTS "${LOCAL_TEST_LIST}" PARENT_SCOPE) + else() + set(ROCPROFSYS_OMPVV_OFFLOAD_TESTS "${LOCAL_TEST_LIST}" PARENT_SCOPE) + endif() endfunction() # ---------------------------------------------------------------------------------------# @@ -212,6 +245,7 @@ if(OMPVV_USE_OFFLOAD_TESTS) ) endif() +# To be added once task_detach race condition resolved in SDK # if(AMDFLANG_VERSION_MAJOR EQUAL 20) # list(APPEND OMPVV_HOST_TESTS_TO_COMPILE # "${OMPVV_TDIR}/task/test_task_detach.F90" @@ -219,92 +253,16 @@ endif() # endif() # ---------------------------------------------------------------------------------------# -# Compile tests +# Setup build time compilation # ---------------------------------------------------------------------------------------# -# Remove all -O3 flags to prevent function inlining -file(READ ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} MAKE_DEF_CONTENT) -string(REGEX REPLACE "-O3[ \t]*" "" MAKE_DEF_CONTENT "${MAKE_DEF_CONTENT}") -file(WRITE ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} "${MAKE_DEF_CONTENT}") - -# OMPVV makefile option NO_OFFLOADING not supported for amdflang, -# must manually do it -set(OMPVV_HOST_FOFFLOADING_FLAGS "-fopenmp --offload-host-only") -update_make_def_foffloading("${OMPVV_HOST_FOFFLOADING_FLAGS}") -compile_ompvv_tests(OMPVV_HOST_TESTS_TO_COMPILE) - -if(OMPVV_USE_OFFLOAD_TESTS) - set(OMPVV_OFFLOAD_FOFFLOADING_FLAGS "-fopenmp") - foreach(arch IN LISTS DEFAULT_GPU_TARGETS) - set(OMPVV_OFFLOAD_FOFFLOADING_FLAGS - "${OMPVV_OFFLOAD_FOFFLOADING_FLAGS} --offload-arch=${arch}" - ) - endforeach() - update_make_def_foffloading("${OMPVV_OFFLOAD_FOFFLOADING_FLAGS}") - compile_ompvv_tests(OMPVV_OFFLOAD_TESTS_TO_COMPILE) -endif() - -# ---------------------------------------------------------------------------------------# -# Copy generated binaries to top level of build folder -# ---------------------------------------------------------------------------------------# - -set(OMPVV_BIN_SOURCE "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin") -set(OMPVV_BIN_DEST "${CMAKE_BINARY_DIR}") - set(ROCPROFSYS_OMPVV_HOST_TESTS "") set(ROCPROFSYS_OMPVV_OFFLOAD_TESTS "") -file(GLOB OMPVV_BINARIES "${OMPVV_BIN_SOURCE}/*") - -foreach(OMPVV_BIN ${OMPVV_BINARIES}) - get_filename_component(BIN_NAME ${OMPVV_BIN} NAME_WE) - - # Determine type of test - set(IS_HOST_TEST FALSE) - set(IS_OFFLOAD_TEST FALSE) - foreach(host_test IN LISTS OMPVV_HOST_TESTS_TO_COMPILE) - get_filename_component(HOST_TEST_NAME ${host_test} NAME_WE) - if(BIN_NAME STREQUAL HOST_TEST_NAME) - set(IS_HOST_TEST TRUE) - break() - endif() - endforeach() - - if(NOT IS_HOST_TEST AND OMPVV_USE_OFFLOAD_TESTS) - foreach(offload_test IN LISTS OMPVV_OFFLOAD_TESTS_TO_COMPILE) - get_filename_component(OFFLOAD_TEST_NAME ${offload_test} NAME_WE) - if(BIN_NAME STREQUAL OFFLOAD_TEST_NAME) - set(IS_OFFLOAD_TEST TRUE) - break() - endif() - endforeach() - endif() - - if(NOT IS_HOST_TEST AND NOT IS_OFFLOAD_TEST) - continue() - endif() - - # For consistent naming with other tests - string(REGEX REPLACE "\\.F90$" "" NEW_BIN_NAME ${BIN_NAME}) - string(REPLACE "_" "-" NEW_BIN_NAME ${NEW_BIN_NAME}) - - file(COPY "${OMPVV_BIN}" DESTINATION "${OMPVV_BIN_DEST}") - - if(IS_HOST_TEST) - set(TARGET_NAME "openmp-vv-host-${NEW_BIN_NAME}") - list(APPEND ROCPROFSYS_OMPVV_HOST_TESTS ${TARGET_NAME}) - elseif(IS_OFFLOAD_TEST) - set(TARGET_NAME "openmp-vv-offload-${NEW_BIN_NAME}") - list(APPEND ROCPROFSYS_OMPVV_OFFLOAD_TESTS ${TARGET_NAME}) - endif() - - file(RENAME "${OMPVV_BIN_DEST}/${BIN_NAME}.F90.o" "${OMPVV_BIN_DEST}/${TARGET_NAME}") - add_executable(${TARGET_NAME} IMPORTED GLOBAL) - set_target_properties( - ${TARGET_NAME} - PROPERTIES IMPORTED_LOCATION "${OMPVV_BIN_DEST}/${TARGET_NAME}" - ) -endforeach() +configure_ompvv_tests("HOST" OMPVV_HOST_TESTS_TO_COMPILE) +if(OMPVV_USE_OFFLOAD_TESTS) + configure_ompvv_tests("OFFLOAD" OMPVV_OFFLOAD_TESTS_TO_COMPILE) +endif() set(ROCPROFSYS_OMPVV_HOST_TESTS "${ROCPROFSYS_OMPVV_HOST_TESTS}" @@ -321,5 +279,5 @@ set(ROCPROFSYS_OMPVV_OFFLOAD_TESTS ) rocprofiler_systems_message(STATUS - "Successfully built and compiled OMPVV tests" + "Successfully configured OMPVV" ) diff --git a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake index 5fa0a38703..c3778964ec 100644 --- a/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake +++ b/projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake @@ -114,7 +114,7 @@ if(ROCPROFSYS_OMPVV_HOST_TESTS) rocprofiler_systems_add_test( SKIP_RUNTIME NAME ${HOST_TEST_NAME} - TARGET ${HOST_TEST_NAME} + TARGET ${HOST_TEST_NAME}-exec LABELS "openmp;ompvv" REWRITE_ARGS -e -v 2 --instrument-loops @@ -141,7 +141,7 @@ if(ROCPROFSYS_OMPVV_HOST_TESTS) rocprofiler_systems_add_test( SKIP_RUNTIME NAME ${OFFLOAD_TEST_NAME} - TARGET ${OFFLOAD_TEST_NAME} + TARGET ${OFFLOAD_TEST_NAME}-exec GPU ON LABELS "openmp;ompvv;openmp-target" REWRITE_ARGS -e -v 2