[rocprofiler-systems] Overhaul OpenMP-VV Test compilation (#1389)

* Reworked Compilation

* Formatting

* Change compile log name

* Optimize Code

* Remove gfx940 and gfx941

projects/rocprofiler-systems/examples/openmp/CMakeLists.txt

@@ -98,8 +98,6 @@ set(DEFAULT_GPU_TARGETS
     "gfx906"
     "gfx908"
     "gfx90a"
-    "gfx940"
-    "gfx941"
     "gfx942"
     "gfx950"
     "gfx1030"

projects/rocprofiler-systems/examples/openmp/external/CMakeLists.txt

@@ -7,8 +7,8 @@
 #
 # * Currently, GNU's libgomp cannot be used to capture traces. Only LLVM's
 #   LIBOMP can be used for trace capture
-# * amdflang uses LIBOMP 201611 (5.0), but not all OpenMP 5.0 features are
-#   implemented. Only a subset of tests can be compiled.
+# * As of implementation, amdflang uses LIBOMP 201611 (5.0), but not all
+#   OpenMP 5.0 features are implemented. Only a subset of tests can be compiled.
 # ----------------------------------------------------------------------------------------#
 
 set(OMPVV_USE_OFFLOAD_TESTS FALSE)
@@ -40,54 +40,87 @@ rocprofiler_systems_checkout_git_submodule(
 )
 
 set(ROCPROFSYS_OMPVV_SUBMODULE_DIR "${PROJECT_SOURCE_DIR}/external/ompvv")
-set(ROCPROFSYS_OMPVV_SOURCE_DIR "${CMAKE_BINARY_DIR}/examples/openmp/external/ompvv")
+set(ROCPROFSYS_OPENMP_EXTERNAL_DIR "${CMAKE_BINARY_DIR}/examples/openmp/external")
+set(ROCPROFSYS_OMPVV_SOURCE_DIR "${ROCPROFSYS_OPENMP_EXTERNAL_DIR}/ompvv")
 set(OMPVV_MAKEDEF_FILE "sys/make/make.def")
+set(OMPVV_BIN_DEST "${CMAKE_BINARY_DIR}")
 
 # ---------------------------------------------------------------------------------------#
-# Helper functions
+# Helper Function
 # ---------------------------------------------------------------------------------------#
 
-# Updates the make.def FOFFLOADING flag
-function(update_make_def_foffloading FOFFLOADING_FLAGS)
-    file(READ ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} MAKE_DEF_CONTENT)
-
-    string(
-        REGEX REPLACE
-        "FOFFLOADING[ \t]*=[ \t]*-fopenmp[^\n\r]*"
-        "FOFFLOADING   =  ${FOFFLOADING_FLAGS}"
-        MAKE_DEF_CONTENT
-        "${MAKE_DEF_CONTENT}"
-    )
-
-    file(WRITE ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} "${MAKE_DEF_CONTENT}")
-endfunction()
-
-# Individually compiles a given list of tests
-function(compile_ompvv_tests TEST_LIST_VAR)
-    # OMPVV doesn't support building subsets of test folders
-    foreach(test IN LISTS ${TEST_LIST_VAR})
-        rocprofiler_systems_message(STATUS "Compiling OMPVV test: ${test}")
-
+function(configure_ompvv_tests TEST_TYPE TEST_LIST_VAR)
+    # Precreate bin dir to avoid race condition
+    if(NOT EXISTS "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin")
         execute_process(
+            COMMAND
+                ${CMAKE_COMMAND} -E make_directory "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin"
+        )
+    endif()
+    if(TEST_TYPE STREQUAL "HOST")
+        set(TARGET_PREFIX "openmp-vv-host")
+        set(FOFFLOADING_FLAGS "-fopenmp --offload-host-only")
+    elseif(TEST_TYPE STREQUAL "OFFLOAD")
+        set(TARGET_PREFIX "openmp-vv-offload")
+        set(FOFFLOADING_FLAGS "-fopenmp")
+        foreach(arch IN LISTS DEFAULT_GPU_TARGETS)
+            set(FOFFLOADING_FLAGS "${FOFFLOADING_FLAGS} --offload-arch=${arch}")
+        endforeach()
+    else()
+        rocprofiler_systems_message(FATAL_ERROR "Unknown TEST_TYPE - Only HOST and OFFLOAD supported")
+    endif()
+
+    set(LOCAL_TEST_LIST "")
+
+    foreach(TEST IN LISTS ${TEST_LIST_VAR})
+        get_filename_component(TEST_NAME ${TEST} NAME_WE)
+        string(REPLACE "_" "-" TARGET_NAME ${TEST_NAME})
+        set(TARGET_NAME "${TARGET_PREFIX}-${TARGET_NAME}")
+
+        set(CUSTOM_FFLAGS "-lm ${FOFFLOADING_FLAGS}")
+        set(CUSTOM_FLINKFLAGS "-lm ${FOFFLOADING_FLAGS}")
+
+        add_custom_command(
+            OUTPUT "${OMPVV_BIN_DEST}/${TARGET_NAME}"
             COMMAND
                 make FC=${OMPVV_FC} DEVICE_TYPE=amd OMP_VERSION=${OMPVV_OPENMP_VERSION}
                 NUM_THREADS_HOST=${OMPVV_NUM_THREADS_HOST}
                 NUM_THREADS_DEVICE=${OMPVV_NUM_THREADS_DEVICE}
-                NUM_TEAMS_DEVICE=${OMPVV_NUM_TEAMS_DEVICE} "SOURCES=${test}" compile
+                NUM_TEAMS_DEVICE=${OMPVV_NUM_TEAMS_DEVICE}
+                "FOFFLOADING=${FOFFLOADING_FLAGS}" "FFLAGS=${CUSTOM_FFLAGS}"
+                "FLINKFLAGS=${CUSTOM_FLINKFLAGS}" "SOURCES=${TEST}" compile >
+                ${CMAKE_CURRENT_BINARY_DIR}/ompvv-compile-${TARGET_NAME}.log 2>&1
+            COMMAND
+                ${CMAKE_COMMAND} -E copy_if_different
+                "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin/${TEST_NAME}.F90.o"
+                "${OMPVV_BIN_DEST}/${TARGET_NAME}"
+            COMMENT "Compiling OMPVV test: ${TEST_NAME}"
             WORKING_DIRECTORY ${ROCPROFSYS_OMPVV_SOURCE_DIR}
-            RESULT_VARIABLE OMPVV_BUILD_RESULT
-            OUTPUT_VARIABLE OMPVV_BUILD_OUTPUT
-            ERROR_VARIABLE OMPVV_BUILD_ERROR
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-            ERROR_STRIP_TRAILING_WHITESPACE
+            VERBATIM
+            DEPENDS
+                ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${TEST}
+                ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE}
         )
 
-        if(NOT OMPVV_BUILD_RESULT EQUAL 0)
-            rocprofiler_systems_message(
-              FATAL_ERROR "OMPVV build failed for ${test}: ${OMPVV_BUILD_ERROR}"
-            )
-        endif()
+        list(APPEND LOCAL_TEST_LIST ${TARGET_NAME})
+
+        add_custom_target(
+            "${TARGET_NAME}-build"
+            ALL
+            DEPENDS "${OMPVV_BIN_DEST}/${TARGET_NAME}"
+        )
+        add_executable("${TARGET_NAME}-exec" IMPORTED GLOBAL)
+        set_target_properties(
+            "${TARGET_NAME}-exec"
+            PROPERTIES IMPORTED_LOCATION "${OMPVV_BIN_DEST}/${TARGET_NAME}"
+        )
     endforeach()
+
+    if(TEST_TYPE STREQUAL "HOST")
+        set(ROCPROFSYS_OMPVV_HOST_TESTS "${LOCAL_TEST_LIST}" PARENT_SCOPE)
+    else()
+        set(ROCPROFSYS_OMPVV_OFFLOAD_TESTS "${LOCAL_TEST_LIST}" PARENT_SCOPE)
+    endif()
 endfunction()
 
 # ---------------------------------------------------------------------------------------#
@@ -212,6 +245,7 @@ if(OMPVV_USE_OFFLOAD_TESTS)
     )
 endif()
 
+# To be added once task_detach race condition resolved in SDK
 # if(AMDFLANG_VERSION_MAJOR EQUAL 20)
 #     list(APPEND OMPVV_HOST_TESTS_TO_COMPILE
 #         "${OMPVV_TDIR}/task/test_task_detach.F90"
@@ -219,92 +253,16 @@ endif()
 # endif()
 
 # ---------------------------------------------------------------------------------------#
-# Compile tests
+# Setup build time compilation
 # ---------------------------------------------------------------------------------------#
 
-# Remove all -O3 flags to prevent function inlining
-file(READ ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} MAKE_DEF_CONTENT)
-string(REGEX REPLACE "-O3[ \t]*" "" MAKE_DEF_CONTENT "${MAKE_DEF_CONTENT}")
-file(WRITE ${ROCPROFSYS_OMPVV_SOURCE_DIR}/${OMPVV_MAKEDEF_FILE} "${MAKE_DEF_CONTENT}")
-
-# OMPVV makefile option NO_OFFLOADING not supported for amdflang,
-# must manually do it
-set(OMPVV_HOST_FOFFLOADING_FLAGS "-fopenmp --offload-host-only")
-update_make_def_foffloading("${OMPVV_HOST_FOFFLOADING_FLAGS}")
-compile_ompvv_tests(OMPVV_HOST_TESTS_TO_COMPILE)
-
-if(OMPVV_USE_OFFLOAD_TESTS)
-    set(OMPVV_OFFLOAD_FOFFLOADING_FLAGS "-fopenmp")
-    foreach(arch IN LISTS DEFAULT_GPU_TARGETS)
-        set(OMPVV_OFFLOAD_FOFFLOADING_FLAGS
-            "${OMPVV_OFFLOAD_FOFFLOADING_FLAGS} --offload-arch=${arch}"
-        )
-    endforeach()
-    update_make_def_foffloading("${OMPVV_OFFLOAD_FOFFLOADING_FLAGS}")
-    compile_ompvv_tests(OMPVV_OFFLOAD_TESTS_TO_COMPILE)
-endif()
-
-# ---------------------------------------------------------------------------------------#
-# Copy generated binaries to top level of build folder
-# ---------------------------------------------------------------------------------------#
-
-set(OMPVV_BIN_SOURCE "${ROCPROFSYS_OMPVV_SOURCE_DIR}/bin")
-set(OMPVV_BIN_DEST "${CMAKE_BINARY_DIR}")
-
 set(ROCPROFSYS_OMPVV_HOST_TESTS "")
 set(ROCPROFSYS_OMPVV_OFFLOAD_TESTS "")
 
-file(GLOB OMPVV_BINARIES "${OMPVV_BIN_SOURCE}/*")
-
-foreach(OMPVV_BIN ${OMPVV_BINARIES})
-    get_filename_component(BIN_NAME ${OMPVV_BIN} NAME_WE)
-
-    # Determine type of test
-    set(IS_HOST_TEST FALSE)
-    set(IS_OFFLOAD_TEST FALSE)
-    foreach(host_test IN LISTS OMPVV_HOST_TESTS_TO_COMPILE)
-        get_filename_component(HOST_TEST_NAME ${host_test} NAME_WE)
-        if(BIN_NAME STREQUAL HOST_TEST_NAME)
-            set(IS_HOST_TEST TRUE)
-            break()
-        endif()
-    endforeach()
-
-    if(NOT IS_HOST_TEST AND OMPVV_USE_OFFLOAD_TESTS)
-        foreach(offload_test IN LISTS OMPVV_OFFLOAD_TESTS_TO_COMPILE)
-            get_filename_component(OFFLOAD_TEST_NAME ${offload_test} NAME_WE)
-            if(BIN_NAME STREQUAL OFFLOAD_TEST_NAME)
-                set(IS_OFFLOAD_TEST TRUE)
-                break()
-            endif()
-        endforeach()
-    endif()
-
-    if(NOT IS_HOST_TEST AND NOT IS_OFFLOAD_TEST)
-        continue()
-    endif()
-
-    # For consistent naming with other tests
-    string(REGEX REPLACE "\\.F90$" "" NEW_BIN_NAME ${BIN_NAME})
-    string(REPLACE "_" "-" NEW_BIN_NAME ${NEW_BIN_NAME})
-
-    file(COPY "${OMPVV_BIN}" DESTINATION "${OMPVV_BIN_DEST}")
-
-    if(IS_HOST_TEST)
-        set(TARGET_NAME "openmp-vv-host-${NEW_BIN_NAME}")
-        list(APPEND ROCPROFSYS_OMPVV_HOST_TESTS ${TARGET_NAME})
-    elseif(IS_OFFLOAD_TEST)
-        set(TARGET_NAME "openmp-vv-offload-${NEW_BIN_NAME}")
-        list(APPEND ROCPROFSYS_OMPVV_OFFLOAD_TESTS ${TARGET_NAME})
-    endif()
-
-    file(RENAME "${OMPVV_BIN_DEST}/${BIN_NAME}.F90.o" "${OMPVV_BIN_DEST}/${TARGET_NAME}")
-    add_executable(${TARGET_NAME} IMPORTED GLOBAL)
-    set_target_properties(
-        ${TARGET_NAME}
-        PROPERTIES IMPORTED_LOCATION "${OMPVV_BIN_DEST}/${TARGET_NAME}"
-    )
-endforeach()
+configure_ompvv_tests("HOST" OMPVV_HOST_TESTS_TO_COMPILE)
+if(OMPVV_USE_OFFLOAD_TESTS)
+    configure_ompvv_tests("OFFLOAD" OMPVV_OFFLOAD_TESTS_TO_COMPILE)
+endif()
 
 set(ROCPROFSYS_OMPVV_HOST_TESTS
     "${ROCPROFSYS_OMPVV_HOST_TESTS}"
@@ -321,5 +279,5 @@ set(ROCPROFSYS_OMPVV_OFFLOAD_TESTS
 )
 
 rocprofiler_systems_message(STATUS
-                            "Successfully built and compiled OMPVV tests"
+                            "Successfully configured OMPVV"
 )

projects/rocprofiler-systems/tests/rocprof-sys-openmp-tests.cmake

@@ -114,7 +114,7 @@ if(ROCPROFSYS_OMPVV_HOST_TESTS)
         rocprofiler_systems_add_test(
             SKIP_RUNTIME
             NAME ${HOST_TEST_NAME}
-            TARGET ${HOST_TEST_NAME}
+            TARGET ${HOST_TEST_NAME}-exec
             LABELS "openmp;ompvv"
             REWRITE_ARGS
               -e -v 2 --instrument-loops
@@ -141,7 +141,7 @@ if(ROCPROFSYS_OMPVV_HOST_TESTS)
         rocprofiler_systems_add_test(
             SKIP_RUNTIME
             NAME ${OFFLOAD_TEST_NAME}
-            TARGET ${OFFLOAD_TEST_NAME}
+            TARGET ${OFFLOAD_TEST_NAME}-exec
             GPU ON
             LABELS "openmp;ompvv;openmp-target"
             REWRITE_ARGS -e -v 2