Update to use new bitcode library structure

Rather than manually linking to the device libraries, the compiler
can now handle linking with them. Allow the build to continue using
old layout if the build system still uses it. Therefore maintain
compatibility with ROCm 3.7 and earlier.

Change-Id: Ida81775da3d0f7c2c67386a71cb057ede31a1545


[ROCm/ROCR-Runtime commit: d23b26f760]

projects/rocr-runtime/rocrtst/Kernels/CMakeLists.txt

@@ -9,7 +9,7 @@ cmake_minimum_required(VERSION 2.8.0)
 #
 #     export LLVM_DIR="Path to Lightning build artifacts"
 #
-#     export OCL_BITCODE_DIR="Path containing Opencl Bitcode libraries"
+#     export OCL_BITCODE_DIR="Path containing AMDGCN Bitcode libraries"
 #
 #  2) Make an new folder called build under root folder
 #
@@ -79,10 +79,16 @@ set (ROCM_CODEOBJ_LIST "" CACHE INTERNAL ROCM_CODEOBJ_LIST)
 # Options that are passed along to Clang to enable code object generation
 #
 set(KERN_SUFFIX "kernels.hsaco")
-set(BITCODE_PREF "-Xclang -mlink-bitcode-file -Xclang")
-set(COMMON_BITCODE_LIB1 "${BITCODE_PREF} $ENV{OCL_BITCODE_DIR}/opencl.amdgcn.bc")
-set(COMMON_BITCODE_LIB2 "${BITCODE_PREF} $ENV{OCL_BITCODE_DIR}/ockl.amdgcn.bc")
-set(COMMON_BITCODE_LIB3 "${BITCODE_PREF} $ENV{OCL_BITCODE_DIR}/ocml.amdgcn.bc")
+# Check if device-libs bitcode is following old or new layout
+set(BITCODE_DIR "$ENV{OCL_BITCODE_DIR}")
+if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc")
+  set(BITCODE_ARGS "-nogpulib
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc")
+else()
+  set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}")
+endif()
 
 #
 # Compiles Opencl kernel into a AMDGcn code object
@@ -110,15 +116,13 @@ function(CompileKernel KRNL_NAME TARGET_DEV)
   # following strings are used to generate a code object and code
   # asm files
   #
-  string(CONCAT CODE_ARG_STR "-Xclang -finclude-default-header -nogpulib "
+  string(CONCAT CODE_ARG_STR "-Xclang -finclude-default-header "
                 "-target amdgcn-amdh-amdhsa -mcpu=${TARGET_DEV} -mno-code-object-v3 "
-                "${COMMON_BITCODE_LIB1} ${COMMON_BITCODE_LIB2} "
-                "${COMMON_BITCODE_LIB3} -cl-std=CL${OPENCL_VER} "
+                "${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} "
                 "${PROJECT_SOURCE_DIR}/${CL_FILE} -o ${KERNEL_DIR}/${CODEOBJ_FILE}")
-  string(CONCAT ASM_ARG_STR "-S -Xclang -finclude-default-header -nogpulib "
+  string(CONCAT ASM_ARG_STR "-S -Xclang -finclude-default-header "
                 "-target amdgcn-amdh-amdhsa -mcpu=${TARGET_DEV} -mno-code-object-v3 "
-                "${COMMON_BITCODE_LIB1} ${COMMON_BITCODE_LIB2} "
-                "${COMMON_BITCODE_LIB3} -cl-std=CL${OPENCL_VER} "
+                "${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} "
                 "${PROJECT_SOURCE_DIR}/${CL_FILE} -o ${KERNEL_DIR}/${CODEASM_FILE}")
   set(ASM_ARG_LIST ${ASM_ARG_STR})
   set(CODE_ARG_LIST ${CODE_ARG_STR})

projects/rocr-runtime/rocrtst/samples/CMakeLists.txt

@@ -126,7 +126,7 @@ if (NOT DEFINED TARGET_DEVICES)
   message("  e.g., cmake -DTARGET_DEVICES=\"gfx803;gfx900;gfx...\" ..")
   message("  Using default target of $DEFAULT_TARGET")
   list(APPEND TARGET_DEVICES "gfx803")
-endif() 
+endif()
 
 string(TOLOWER "${ROCRTST_BLD_TYPE}" tmp)
 if("${tmp}" STREQUAL release)
@@ -141,6 +141,19 @@ if(${EMULATOR_BUILD})
 add_definitions(-DROCRTST_EMULATOR_BUILD=1)
 endif()
 
+find_path(BITCODE_DIR NAMES "opencl.bc" "opencl.amdgcn.bc"
+  PATHS
+    "${ROCM_DIR}/amdgcn/bitcode"
+    "${ROCM_DIR}/lib/bitcode"
+    "${ROCM_DIR}/lib"
+    "${ROCM_DIR}/lib/x86_64/bitcode"
+    "${OPENCL_DIR}/amdgcn/bitcode"
+    "${OPENCL_DIR}/lib/x86_64/bitcode"
+    "${LLVM_DIR}/../lib/bitcode"
+    "${CMAKE_PREFIX_PATH}/amdgcn/bitcode"
+    "${CMAKE_PREFIX_PATH}/lib/bitcode"
+    "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode")
+
 # Set Name for Samples Project
 #
 
@@ -240,15 +253,15 @@ include_directories(${ROCR_INC_DIR} "${OPENCL_DIR}/include")
 function(process_sample S_NAME TARG_DEV HAS_KERNEL)
   set(KERNEL_DIR ${PROJECT_BINARY_DIR}/${TARG_DEV})
   set(SNAME_KERNEL "${S_NAME}_kernels.hsaco")
-  
+
   set(TARG_NAME "${S_NAME}_hsaco.${TARG_DEV}")
   set (HSACO_TARG_LIST ${HSACO_TARG_LIST} ${TARG_NAME}
                                                CACHE INTERNAL HSACO_TARG_LIST)
-  
+
   if (${HAS_KERNEL})
     # Build the kernel
     separate_arguments(CLANG_ARG_LIST UNIX_COMMAND
-     "-x cl -target amdgcn-amd-amdhsa -Xclang -finclude-default-header -nogpulib -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_LIBS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}")
+     "-x cl -target amdgcn-amd-amdhsa -Xclang -finclude-default-header -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}")
     add_custom_target("${TARG_NAME}" ${CLANG} ${CLANG_ARG_LIST} COMMAND
       ${CMAKE_COMMAND} -E create_symlink
         "../${SNAME_EXE}" "${KERNEL_DIR}/${SNAME_EXE}"
@@ -291,22 +304,23 @@ foreach(td ${TARGET_DEVICES})
 endforeach(td)
 
 ###########################
-# SAMPLE SPECIFIC SECTION 
+# SAMPLE SPECIFIC SECTION
 ###########################
 
 set (HSACO_TARG_LIST "" CACHE INTERNAL HSACO_TARG_LIST)
 
 set(KERN_SUFFIX "kernels.hsaco")
-set(BITCODE_PREF "-Xclang -mlink-bitcode-file -Xclang")
-set(BITCODE_PREF "${BITCODE_PREF} ${OPENCL_LIB_DIR}/bitcode")
 
-set(COMMON_BITCODE_LIBS "${BITCODE_PREF}/opencl.amdgcn.bc")
-set(COMMON_BITCODE_LIBS
-                      "${COMMON_BITCODE_LIBS} ${BITCODE_PREF}/ockl.amdgcn.bc")
+# Check if device-libs bitcode is following old or new layout
+if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc")
+  set(BITCODE_ARGS "-nogpulib
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc")
+else()
+  set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}")
+endif()
 
-# Binary Search
-set(BITCODE_LIBS "${COMMON_BITCODE_LIBS}")
-set(BITCODE_LIBS "${BITCODE_LIBS} ${BITCODE_PREF}/ocml.amdgcn.bc")
 set(CL_FILE_LIST
                "${PROJECT_SOURCE_DIR}/binary_search/binary_search_kernels.cl")
 build_sample_for_devices("binary_search" TRUE)

projects/rocr-runtime/rocrtst/suites/test_common/CMakeLists.txt

@@ -137,6 +137,19 @@ else()
   set(ISDEBUG 1)
 endif()
 
+find_path(BITCODE_DIR NAMES "opencl.bc" "opencl.amdgcn.bc"
+  PATHS
+    "${ROCM_DIR}/amdgcn/bitcode"
+    "${ROCM_DIR}/lib/bitcode"
+    "${ROCM_DIR}/lib"
+    "${ROCM_DIR}/lib/x86_64/bitcode"
+    "${OPENCL_DIR}/amdgcn/bitcode"
+    "${OPENCL_DIR}/lib/x86_64/bitcode"
+    "${LLVM_DIR}/../lib/bitcode"
+    "${CMAKE_PREFIX_PATH}/amdgcn/bitcode"
+    "${CMAKE_PREFIX_PATH}/lib/bitcode"
+    "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode")
+
 #
 # Print out the build configuration being used:
 #
@@ -205,7 +218,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
 #
 # Extend the compiler flags for 64-bit builds
 #
-if (IS64BIT) 
+if (IS64BIT)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m64  -msse -msse2")
 else()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
@@ -277,7 +290,7 @@ function(build_kernel S_NAME TARG_DEV)
   set(HSACO_TARG_LIST ${HSACO_TARG_LIST} ${TARG_NAME}
                                                  CACHE INTERNAL HSA_TARG_LIST)
   separate_arguments(CLANG_ARG_LIST UNIX_COMMAND
-   "-x cl -target amdgcn-amd-amdhsa -include ${OPENCL_INC_DIR}opencl-c.h -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_LIBS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}")
+   "-x cl -target amdgcn-amd-amdhsa -include ${OPENCL_INC_DIR}opencl-c.h -mcpu=${TARG_DEV} -mno-code-object-v3 ${BITCODE_ARGS} -cl-std=CL${OPENCL_VER} ${CL_FILE_LIST} -o ${KERNEL_DIR}/${SNAME_KERNEL}")
   add_custom_target("${TARG_NAME}" ${CLANG} ${CLANG_ARG_LIST} COMMAND
       ${CMAKE_COMMAND} -E create_symlink
         "../${ROCRTST}" "${KERNEL_DIR}/${ROCRTST}"
@@ -305,13 +318,16 @@ endforeach(td)
 set (HSACO_TARG_LIST "" CACHE INTERNAL HSACO_TARG_LIST)
 
 set(KERN_SUFFIX "kernels.hsaco")
-set(BITCODE_PREF "-Xclang -mlink-bitcode-file -Xclang")
-set(BITCODE_PREF "${BITCODE_PREF} ${OPENCL_LIB_DIR}/bitcode")
-
-set(COMMON_BITCODE_LIBS "${BITCODE_PREF}/opencl.amdgcn.bc")
-set(COMMON_BITCODE_LIBS
-                      "${COMMON_BITCODE_LIBS} ${BITCODE_PREF}/ockl.amdgcn.bc")
 
+# Check if device-libs bitcode is following old or new layout
+if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc")
+  set(BITCODE_ARGS "-nogpulib
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc
+    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc")
+else()
+  set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}")
+endif()
 
 # Test Case Template example
 set(BITCODE_LIBS "${COMMON_BITCODE_LIBS}")

projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/CMakeLists.txt

@@ -49,10 +49,20 @@ set (QUIT 0)
 find_package(Clang REQUIRED HINTS ${CMAKE_INSTALL_PREFIX}/llvm ${CMAKE_PREFIX_PATH}/llvm PATHS /opt/rocm/llvm )
 
 # Device libs doesn't support find_package yet.
-get_include_path(BITCODE_DIR "Bitcode library path" RESULT FOUND NAMES "opencl.amdgcn.bc"
-  HINTS "${CMAKE_INSTALL_PREFIX}/lib/bitcode" "${CMAKE_INSTALL_PREFIX}/lib/x86_64/bitcode"
-  "${CMAKE_PREFIX_PATH}/lib/bitcode" "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode"
-  PATHS "/opt/rocm/lib/bitcode" "/opt/rocm/lib/x86_64/bitcode")
+# FIXME: HINTS should not use CMAKE_INSTALL_PREFIX
+get_include_path(BITCODE_DIR "Bitcode library path" RESULT FOUND NAMES "opencl.bc" "opencl.amdgcn.bc"
+  HINTS
+    "${CMAKE_INSTALL_PREFIX}/amdgcn/bitcode"
+    "${CMAKE_INSTALL_PREFIX}/lib/bitcode"
+    "${CMAKE_INSTALL_PREFIX}/lib/x86_64/bitcode"
+    "${CMAKE_PREFIX_PATH}/amdgcn/bitcode"
+    "${CMAKE_PREFIX_PATH}/lib/bitcode"
+    "${CMAKE_PREFIX_PATH}/lib/x86_64/bitcode"
+  PATHS
+    "/opt/rocm/amdgcn/bitcode"
+    "/opt/rocm/lib/bitcode"
+    "/opt/rocm/lib"
+    "/opt/rocm/lib/x86_64/bitcode")
 if (NOT ${FOUND})
   set (QUIT 1)
 endif()
@@ -92,18 +102,24 @@ endif()
 function(gen_kernel_bc TARGET_DEV XNACK_OPT INPUT_FILE OUTPUT_FILE)
 
   string (REPLACE "gfx" "" GFXIP "${TARGET_DEV}")
+  # Determine if device-libs is following old or new layout
+  if(EXISTS "${BITCODE_DIR}/opencl.amdgcn.bc")
+    set(BITCODE_ARGS "-nogpulib
+      -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc
+      -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc
+      -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc
+      -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_daz_opt_on.amdgcn.bc
+      -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_isa_version_${GFXIP}.amdgcn.bc
+      -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_unsafe_math_off.amdgcn.bc
+      -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_finite_only_off.amdgcn.bc")
+  else()
+    set(BITCODE_ARGS "--hip-device-lib-path=${BITCODE_DIR}")
+  endif()
+
   separate_arguments(CLANG_ARG_LIST UNIX_COMMAND
     "-O2 -x cl -cl-denorms-are-zero -cl-std=CL2.0 -target amdgcn-amd-amdhsa
     -Xclang -finclude-default-header -mcpu=${TARGET_DEV} -m${XNACK_OPT}
-    -nogpulib
-    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/opencl.amdgcn.bc
-    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ockl.amdgcn.bc
-    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/ocml.amdgcn.bc
-    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_daz_opt_on.amdgcn.bc
-    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_isa_version_${GFXIP}.amdgcn.bc
-    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_unsafe_math_off.amdgcn.bc
-    -Xclang -mlink-bitcode-file -Xclang ${BITCODE_DIR}/oclc_finite_only_off.amdgcn.bc
-    -o ${OUTPUT_FILE} ${INPUT_FILE}")
+    ${BITCODE_ARGS} -o ${OUTPUT_FILE} ${INPUT_FILE}")
 
   ## Add custom command to produce a code object file.
   ## This depends on the kernel source file & compiler.
@@ -125,7 +141,7 @@ endfunction(gen_kernel_bc)
 ## Find device code object name and forward to custom command
 ##==========================================
 function(build_kernel BLIT_NAME TARG_DEV)
-  
+
   list (FIND XNACK_DEVS ${TARG_DEV} XNACK_IDX)
   if (${XNACK_IDX} GREATER -1)
     set (XNACK_OPT "xnack")
@@ -137,7 +153,7 @@ function(build_kernel BLIT_NAME TARG_DEV)
   set (CODE_OBJECT_FILE "${BLIT_NAME}_${TARG_DEV}")
   set (CL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/imageblit_kernels.cl)
   gen_kernel_bc(${TARG_DEV} ${XNACK_OPT} ${CL_FILE} ${CODE_OBJECT_FILE})
-  
+
   ## Build a list of code object file names
   ## These will be target dependencies.
   set (HSACO_TARG_LIST ${HSACO_TARG_LIST} "${CODE_OBJECT_FILE}" PARENT_SCOPE)
@@ -159,7 +175,7 @@ function(build_kernel_for_devices BLIT_NAME)
   endforeach(dev)
 
   set(HSACO_TARG_LIST ${HSACO_TARG_LIST} PARENT_SCOPE)
-  
+
 endfunction(build_kernel_for_devices)
 
 ##==========================================

projects/rocr-runtime/runtime/hsa-runtime/image/blit_src/README.md

@@ -25,14 +25,14 @@ To add a new supported device, the following steps are required:
 In order to create the code object file, the bitcodes of the kernels are
 generated by the compiler and the following bitcode libraries are required,
 
-   opencl.amdgcn.bc
-   ocml.amdgcn.bc
-   irif.amdgcn.bc
-   oclc_correctly_rounded_sqrt_off.amdgcn.bc
-   oclc_daz_opt_on.amdgcn.bc
-   oclc_finite_only_off.amdgcn.bc
-   oclc_isa_version_<GFXIP>.amdgcn.bc
-   oclc_unsafe_math_off.amdgcn.bc
+   opencl.bc
+   ocml.bc
+   irif.bc
+   oclc_correctly_rounded_sqrt_off.bc
+   oclc_daz_opt_on.bc
+   oclc_finite_only_off.bc
+   oclc_isa_version_<GFXIP>.bc
+   oclc_unsafe_math_off.bc
 
 where <GFXIP> is the gfxip number of the GPU. The directory contains the
 bitcode libraries is specified in a CMake varaible.
@@ -43,7 +43,7 @@ object file.  All of them have default values, and defined as following:
       OPENCL_DIR - the location of installed OpenCL
                    (Default: /opt/rocm/opencl)
      BITCODE_DIR - the directory contains the bitcode library
-                   (Default: ${OPENCL_DIR}/lib/x86_64/bitcode)
+                   (Default: /opt/rocm/amdgcn/bitcode)
         LLVM_DIR - the directory contains the clang, llvm-link and llvm-dis
                    executables
                    (Default: ${PROJECT_BUILD_DIR}/../lightning/bin)