From f9cfe0989483a3233e78838466b6a010cbefdcd1 Mon Sep 17 00:00:00 2001 From: Aaron Enye Shi Date: Mon, 13 Jul 2020 13:56:25 -0400 Subject: [PATCH] Update to use new bitcode library structure Rather than manually linking to the device libraries, the compiler can now handle linking with them. Allow the build to continue using old layout if the build system still uses it. Therefore maintain compatibility with ROCm 3.7 and earlier. Change-Id: Ida81775da3d0f7c2c67386a71cb057ede31a1545 [ROCm/ROCR-Runtime commit: d23b26f76077d835a3e9f8057ccbde5171c8ae80] --- .../rocrtst/Kernels/CMakeLists.txt | 26 +++++----- .../rocrtst/samples/CMakeLists.txt | 40 +++++++++++----- .../rocrtst/suites/test_common/CMakeLists.txt | 32 +++++++++---- .../hsa-runtime/image/blit_src/CMakeLists.txt | 48 ++++++++++++------- .../hsa-runtime/image/blit_src/README.md | 18 +++---- 5 files changed, 107 insertions(+), 57 deletions(-) diff --git a/projects/rocr-runtime/rocrtst/Kernels/CMakeLists.txt b/projects/rocr-runtime/rocrtst/Kernels/CMakeLists.txt index 15c8a95cbb..c65f5d79bc 100644 --- a/projects/rocr-runtime/rocrtst/Kernels/CMakeLists.txt +++ b/projects/rocr-runtime/rocrtst/Kernels/CMakeLists.txt @@ -9,7 +9,7 @@ cmake_minimum_required(VERSION 2.8.0) # # export LLVM_DIR="Path to Lightning build artifacts" # -# export OCL_BITCODE_DIR="Path containing Opencl Bitcode libraries" +# export OCL_BITCODE_DIR="Path containing AMDGCN Bitcode libraries" # # 2) Make an new folder called build under root folder # @@ -79,10 +79,16 @@ set (ROCM_CODEOBJ_LIST "" CACHE INTERNAL ROCM_CODEOBJ_LIST) # Options that are passed along to Clang to enable code object generation # set(KERN_SUFFIX "kernels.hsaco") -set(BITCODE_PREF "-Xclang -mlink-bitcode-file -Xclang") -set(COMMON_BITCODE_LIB1 "${BITCODE_PREF} $ENV{OCL_BITCODE_DIR}/opencl.amdgcn.bc") -set(COMMON_BITCODE_LIB2 "${BITCODE_PREF} $ENV{OCL_BITCODE_DIR}/ockl.amdgcn.bc") -set(COMMON_BITCODE_LIB3 "${BITCODE_PREF} $ENV{OCL_BITCODE_DIR}/ocml.amdgcn.bc") +# Check if device-libs bitcode is following old or new layout +set(BITCODE_DIR "$ENV{OCL_BITCODE_DIR}") +if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc") + set(BITCODE_ARGS "-nogpulib + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc") +else() + set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}") +endif() # # Compiles Opencl kernel into a AMDGcn code object @@ -110,15 +116,13 @@ function(CompileKernel KRNL_NAME TARGET_DEV) # following strings are used to generate a code object and code # asm files # - string(CONCAT CODE_ARG_STR "-Xclang -finclude-default-header -nogpulib " + string(CONCAT CODE_ARG_STR "-Xclang -finclude-default-header " "-target amdgcn-amdh-amdhsa -mcpu=${TARGET_DEV} -mno-code-object-v3 " - "${COMMON_BITCODE_LIB1} ${COMMON_BITCODE_LIB2} " - "${COMMON_BITCODE_LIB3} -cl-std=CL${OPENCL_VER} " + "${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} " "${PROJECT_SOURCE_DIR}/${CL_FILE} -o ${KERNEL_DIR}/${CODEOBJ_FILE}") - string(CONCAT ASM_ARG_STR "-S -Xclang -finclude-default-header -nogpulib " + string(CONCAT ASM_ARG_STR "-S -Xclang -finclude-default-header " "-target amdgcn-amdh-amdhsa -mcpu=${TARGET_DEV} -mno-code-object-v3 " - "${COMMON_BITCODE_LIB1} ${COMMON_BITCODE_LIB2} " - "${COMMON_BITCODE_LIB3} -cl-std=CL${OPENCL_VER} " + "${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} " "${PROJECT_SOURCE_DIR}/${CL_FILE} -o ${KERNEL_DIR}/${CODEASM_FILE}") set(ASM_ARG_LIST ${ASM_ARG_STR}) set(CODE_ARG_LIST ${CODE_ARG_STR}) diff --git a/projects/rocr-runtime/rocrtst/samples/CMakeLists.txt b/projects/rocr-runtime/rocrtst/samples/CMakeLists.txt index e165f2ae6d..a16e04dae3 100755 --- a/projects/rocr-runtime/rocrtst/samples/CMakeLists.txt +++ b/projects/rocr-runtime/rocrtst/samples/CMakeLists.txt @@ -126,7 +126,7 @@ if (NOT DEFINED TARGET_DEVICES) message(" e.g., cmake -DTARGET_DEVICES=\"gfx803;gfx900;gfx...\" ..") message(" Using default target of $DEFAULT_TARGET") list(APPEND TARGET_DEVICES "gfx803") -endif() +endif() string(TOLOWER "${ROCRTST_BLD_TYPE}" tmp) if("${tmp}" STREQUAL release) @@ -141,6 +141,19 @@ if(${EMULATOR_BUILD}) add_definitions(-DROCRTST_EMULATOR_BUILD=1) endif() +find_path(BITCODE_DIR NAMES "opencl.bc" "opencl.amdgcn.bc" + PATHS + "${ROCM_DIR}/amdgcn/bitcode" + "${ROCM_DIR}/lib/bitcode" + "${ROCM_DIR}/lib" + "${ROCM_DIR}/lib/x86_64/bitcode" + "${OPENCL_DIR}/amdgcn/bitcode" + "${OPENCL_DIR}/lib/x86_64/bitcode" + "${LLVM_DIR}/../lib/bitcode" + "${CMAKE_PREFIX_PATH}/amdgcn/bitcode" + "${CMAKE_PREFIX_PATH}/lib/bitcode" + "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode") + # Set Name for Samples Project # @@ -240,15 +253,15 @@ include_directories(${ROCR_INC_DIR} "${OPENCL_DIR}/include") function(process_sample S_NAME TARG_DEV HAS_KERNEL) set(KERNEL_DIR ${PROJECT_BINARY_DIR}/${TARG_DEV}) set(SNAME_KERNEL "${S_NAME}_kernels.hsaco") - + set(TARG_NAME "${S_NAME}_hsaco.${TARG_DEV}") set (HSACO_TARG_LIST ${HSACO_TARG_LIST} ${TARG_NAME} CACHE INTERNAL HSACO_TARG_LIST) - + if (${HAS_KERNEL}) # Build the kernel separate_arguments(CLANG_ARG_LIST UNIX_COMMAND - "-x cl -target amdgcn-amd-amdhsa -Xclang -finclude-default-header -nogpulib -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_LIBS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}") + "-x cl -target amdgcn-amd-amdhsa -Xclang -finclude-default-header -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}") add_custom_target("${TARG_NAME}" ${CLANG} ${CLANG_ARG_LIST} COMMAND ${CMAKE_COMMAND} -E create_symlink "../${SNAME_EXE}" "${KERNEL_DIR}/${SNAME_EXE}" @@ -291,22 +304,23 @@ foreach(td ${TARGET_DEVICES}) endforeach(td) ########################### -# SAMPLE SPECIFIC SECTION +# SAMPLE SPECIFIC SECTION ########################### set (HSACO_TARG_LIST "" CACHE INTERNAL HSACO_TARG_LIST) set(KERN_SUFFIX "kernels.hsaco") -set(BITCODE_PREF "-Xclang -mlink-bitcode-file -Xclang") -set(BITCODE_PREF "${BITCODE_PREF} ${OPENCL_LIB_DIR}/bitcode") -set(COMMON_BITCODE_LIBS "${BITCODE_PREF}/opencl.amdgcn.bc") -set(COMMON_BITCODE_LIBS - "${COMMON_BITCODE_LIBS} ${BITCODE_PREF}/ockl.amdgcn.bc") +# Check if device-libs bitcode is following old or new layout +if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc") + set(BITCODE_ARGS "-nogpulib + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc") +else() + set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}") +endif() -# Binary Search -set(BITCODE_LIBS "${COMMON_BITCODE_LIBS}") -set(BITCODE_LIBS "${BITCODE_LIBS} ${BITCODE_PREF}/ocml.amdgcn.bc") set(CL_FILE_LIST "${PROJECT_SOURCE_DIR}/binary_search/binary_search_kernels.cl") build_sample_for_devices("binary_search" TRUE) diff --git a/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt b/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt index 91ddb6d1ad..b17810284e 100755 --- a/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt +++ b/projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt @@ -137,6 +137,19 @@ else() set(ISDEBUG 1) endif() +find_path(BITCODE_DIR NAMES "opencl.bc" "opencl.amdgcn.bc" + PATHS + "${ROCM_DIR}/amdgcn/bitcode" + "${ROCM_DIR}/lib/bitcode" + "${ROCM_DIR}/lib" + "${ROCM_DIR}/lib/x86_64/bitcode" + "${OPENCL_DIR}/amdgcn/bitcode" + "${OPENCL_DIR}/lib/x86_64/bitcode" + "${LLVM_DIR}/../lib/bitcode" + "${CMAKE_PREFIX_PATH}/amdgcn/bitcode" + "${CMAKE_PREFIX_PATH}/lib/bitcode" + "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode") + # # Print out the build configuration being used: # @@ -205,7 +218,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") # # Extend the compiler flags for 64-bit builds # -if (IS64BIT) +if (IS64BIT) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m64 -msse -msse2") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") @@ -277,7 +290,7 @@ function(build_kernel S_NAME TARG_DEV) set(HSACO_TARG_LIST ${HSACO_TARG_LIST} ${TARG_NAME} CACHE INTERNAL HSA_TARG_LIST) separate_arguments(CLANG_ARG_LIST UNIX_COMMAND - "-x cl -target amdgcn-amd-amdhsa -include ${OPENCL_INC_DIR}opencl-c.h -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_LIBS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}") + "-x cl -target amdgcn-amd-amdhsa -include ${OPENCL_INC_DIR}opencl-c.h -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}") add_custom_target("${TARG_NAME}" ${CLANG} ${CLANG_ARG_LIST} COMMAND ${CMAKE_COMMAND} -E create_symlink "../${ROCRTST}" "${KERNEL_DIR}/${ROCRTST}" @@ -305,13 +318,16 @@ endforeach(td) set (HSACO_TARG_LIST "" CACHE INTERNAL HSACO_TARG_LIST) set(KERN_SUFFIX "kernels.hsaco") -set(BITCODE_PREF "-Xclang -mlink-bitcode-file -Xclang") -set(BITCODE_PREF "${BITCODE_PREF} ${OPENCL_LIB_DIR}/bitcode") - -set(COMMON_BITCODE_LIBS "${BITCODE_PREF}/opencl.amdgcn.bc") -set(COMMON_BITCODE_LIBS - "${COMMON_BITCODE_LIBS} ${BITCODE_PREF}/ockl.amdgcn.bc") +# Check if device-libs bitcode is following old or new layout +if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc") + set(BITCODE_ARGS "-nogpulib + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc") +else() + set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}") +endif() # Test Case Template example set(BITCODE_LIBS "${COMMON_BITCODE_LIBS}") diff --git a/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/CMakeLists.txt b/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/CMakeLists.txt index ef59715a23..c3e2838a00 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/CMakeLists.txt +++ b/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/CMakeLists.txt @@ -49,10 +49,20 @@ set (QUIT 0) find_package(Clang REQUIRED HINTS ${CMAKE_INSTALL_PREFIX}/llvm ${CMAKE_PREFIX_PATH}/llvm PATHS /opt/rocm/llvm ) # Device libs doesn't support find_package yet. -get_include_path(BITCODE_DIR "Bitcode library path" RESULT FOUND NAMES "opencl.amdgcn.bc" - HINTS "${CMAKE_INSTALL_PREFIX}/lib/bitcode" "${CMAKE_INSTALL_PREFIX}/lib/x86_64/bitcode" - "${CMAKE_PREFIX_PATH}/lib/bitcode" "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode" - PATHS "/opt/rocm/lib/bitcode" "/opt/rocm/lib/x86_64/bitcode") +# FIXME: HINTS should not use CMAKE_INSTALL_PREFIX +get_include_path(BITCODE_DIR "Bitcode library path" RESULT FOUND NAMES "opencl.bc" "opencl.amdgcn.bc" + HINTS + "${CMAKE_INSTALL_PREFIX}/amdgcn/bitcode" + "${CMAKE_INSTALL_PREFIX}/lib/bitcode" + "${CMAKE_INSTALL_PREFIX}/lib/x86_64/bitcode" + "${CMAKE_PREFIX_PATH}/amdgcn/bitcode" + "${CMAKE_PREFIX_PATH}/lib/bitcode" + "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode" + PATHS + "/opt/rocm/amdgcn/bitcode" + "/opt/rocm/lib/bitcode" + "/opt/rocm/lib" + "/opt/rocm/lib/x86_64/bitcode") if (NOT ${FOUND}) set (QUIT 1) endif() @@ -92,18 +102,24 @@ endif() function(gen_kernel_bc TARGET_DEV XNACK_OPT INPUT_FILE OUTPUT_FILE) string (REPLACE "gfx" "" GFXIP "${TARGET_DEV}") + # Determine if device-libs is following old or new layout + if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc") + set(BITCODE_ARGS "-nogpulib + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_daz_opt_on.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_isa_version_${GFXIP}.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_unsafe_math_off.amdgcn.bc + -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_finite_only_off.amdgcn.bc") + else() + set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}") + endif() + separate_arguments(CLANG_ARG_LIST UNIX_COMMAND "-O2 -x cl -cl-denorms-are-zero -cl-std=CL2.0 -target amdgcn-amd-amdhsa -Xclang -finclude-default-header -mcpu=${TARGET_DEV} -m${XNACK_OPT} - -nogpulib - -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc - -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc - -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc - -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_daz_opt_on.amdgcn.bc - -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_isa_version_${GFXIP}.amdgcn.bc - -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_unsafe_math_off.amdgcn.bc - -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_finite_only_off.amdgcn.bc - -o ${OUTPUT_FILE} ${INPUT_FILE}") + ${BITCODE_ARGS} -o ${OUTPUT_FILE} ${INPUT_FILE}") ## Add custom command to produce a code object file. ## This depends on the kernel source file & compiler. @@ -125,7 +141,7 @@ endfunction(gen_kernel_bc) ## Find device code object name and forward to custom command ##========================================== function(build_kernel BLIT_NAME TARG_DEV) - + list (FIND XNACK_DEVS ${TARG_DEV} XNACK_IDX) if (${XNACK_IDX} GREATER -1) set (XNACK_OPT "xnack") @@ -137,7 +153,7 @@ function(build_kernel BLIT_NAME TARG_DEV) set (CODE_OBJECT_FILE "${BLIT_NAME}_${TARG_DEV}") set (CL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/imageblit_kernels.cl) gen_kernel_bc(${TARG_DEV} ${XNACK_OPT} ${CL_FILE} ${CODE_OBJECT_FILE}) - + ## Build a list of code object file names ## These will be target dependencies. set (HSACO_TARG_LIST ${HSACO_TARG_LIST} "${CODE_OBJECT_FILE}" PARENT_SCOPE) @@ -159,7 +175,7 @@ function(build_kernel_for_devices BLIT_NAME) endforeach(dev) set(HSACO_TARG_LIST ${HSACO_TARG_LIST} PARENT_SCOPE) - + endfunction(build_kernel_for_devices) ##========================================== diff --git a/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/README.md b/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/README.md index b97e7c35b6..b6597873a4 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/README.md +++ b/projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/README.md @@ -25,14 +25,14 @@ To add a new supported device, the following steps are required: In order to create the code object file, the bitcodes of the kernels are generated by the compiler and the following bitcode libraries are required, - opencl.amdgcn.bc - ocml.amdgcn.bc - irif.amdgcn.bc - oclc_correctly_rounded_sqrt_off.amdgcn.bc - oclc_daz_opt_on.amdgcn.bc - oclc_finite_only_off.amdgcn.bc - oclc_isa_version_.amdgcn.bc - oclc_unsafe_math_off.amdgcn.bc + opencl.bc + ocml.bc + irif.bc + oclc_correctly_rounded_sqrt_off.bc + oclc_daz_opt_on.bc + oclc_finite_only_off.bc + oclc_isa_version_.bc + oclc_unsafe_math_off.bc where is the gfxip number of the GPU. The directory contains the bitcode libraries is specified in a CMake varaible. @@ -43,7 +43,7 @@ object file. All of them have default values, and defined as following: OPENCL_DIR - the location of installed OpenCL (Default: /opt/rocm/opencl) BITCODE_DIR - the directory contains the bitcode library - (Default: ${OPENCL_DIR}/lib/x86_64/bitcode) + (Default: /opt/rocm/amdgcn/bitcode) LLVM_DIR - the directory contains the clang, llvm-link and llvm-dis executables (Default: ${PROJECT_BUILD_DIR}/../lightning/bin)