Improve python wrapper generation

- Add "BUILD_PACKAGE" option that is OFF by default (CI issues) - Optionally package wrapper if python3.7 or above is present (hack for CI) - Optionally build wrapper if clang is present (hack for CI) - Improve dependency resolution for wrapper - Use python venv - Use pyproject.toml instead of setup.py - Use CMake install path to search for libamd_smi.so - Move python-specific CMakeLists into py-interface - Search for libamd_smi.so more aggressively Change-Id: Ie7dad676b0d4a5f58ad2b887db7fecf5b1297e3b Signed-off-by: Galantsev, Dmitrii <dmitrii.galantsev@amd.com>
2023-03-10 13:45:51 -06:00
parent 3af2687f17
commit 74479187d0
7 changed files with 252 additions and 82 deletions
@@ -53,6 +53,8 @@ project(${AMD_SMI_LIBS_TARGET})
 include(GNUInstallDirs)

 option(BUILD_TESTS "Build test suite" OFF)
+# TODO: Enable once virtualenv is installed on CI machines
+option(BUILD_PACKAGE "Build python package" OFF)
 option(ENABLE_LDCONFIG "Set library links and caches using ldconfig." ON)

 # Set share path here because project name != amd_smi
@@ -128,57 +130,9 @@ if(BUILD_TESTS)
    add_subdirectory("tests/amd_smi_test")
 endif()

-# Generate python_wrapper and package targets
-find_program(PYTHON3 "python3")
-find_program(PIP3 "pip3")
-
-if(PYTHON3 AND PIP3)
-    add_custom_command(
-        OUTPUT clang_tools_extract
-        COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper
-        COMMAND pip3 download ctypeslib2==2.3.2 -d ${CMAKE_CURRENT_BINARY_DIR}/dl
-        COMMAND pip3 download clang==10.0.1 -d ${CMAKE_CURRENT_BINARY_DIR}/dl
-        COMMAND unzip -o ${CMAKE_CURRENT_BINARY_DIR}/dl/ctypeslib2-2.3.2-py3-none-any.whl -d
-                ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper
-        COMMAND unzip -o ${CMAKE_CURRENT_BINARY_DIR}/dl/clang-10.0.1-py3-none-any.whl -d
-                ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
-
-    add_custom_target(
-        python_wrapper
-        DEPENDS clang_tools_extract
-        COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper
-        COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/include/amd_smi/amdsmi.h ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper
-        COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper/amd_smi
-        COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/rocm_smi/include/rocm_smi/kfd_ioctl.h
-                ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper/amd_smi/
-        COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/tools/generator.py ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper/
-        COMMAND
-            python3 ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper/generator.py -o ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper/amdsmi_wrapper.py
-            -i ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper/amdsmi.h -l ${CMAKE_CURRENT_BINARY_DIR}/src/libamd_smi.so
-        COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper/amdsmi_wrapper.py
-                ${CMAKE_CURRENT_SOURCE_DIR}/py-interface/
-        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper)
-
-    set(PY_INTERFACE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/py-interface")
-    set(PACKAGE_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/python_package/amdsmi")
-
-    add_custom_target(
-        python_package
-        DEPENDS python_wrapper
-        COMMAND mkdir -p ${PACKAGE_OUTPUT_DIR}
-        COMMAND cp ${PY_INTERFACE_DIR}/__init__.py ${PACKAGE_OUTPUT_DIR}
-        COMMAND cp ${PY_INTERFACE_DIR}/amdsmi_exception.py ${PACKAGE_OUTPUT_DIR}
-        COMMAND cp ${PY_INTERFACE_DIR}/amdsmi_interface.py ${PACKAGE_OUTPUT_DIR}
-        COMMAND cp ${PY_INTERFACE_DIR}/amdsmi_wrapper.py ${PACKAGE_OUTPUT_DIR}
-        COMMAND cp ${PY_INTERFACE_DIR}/README.md ${PACKAGE_OUTPUT_DIR}
-        COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/src/libamd_smi.so ${PACKAGE_OUTPUT_DIR}
-        COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE ${PACKAGE_OUTPUT_DIR}
-        COMMAND echo ${PACKAGE_OUTPUT_DIR})
-
-else()
-    message("Python3 or Pip3 not found.")
-endif(PYTHON3 AND PIP3)
+if(BUILD_PACKAGE)
+    add_subdirectory("py-interface")
+endif()

 include(CMakePackageConfigHelpers)

@@ -17,9 +17,14 @@ installed to query firmware information and hardware IPs.
 ## Additional Required software for building

 In order to build the AMD SMI library, the following components are required. Note that the software versions listed are what was used in development. Earlier versions are not guaranteed to work:
-* CMake (v3.11.0)
+* CMake (v3.11.0) - `pip3 install cmake`
 * g++ (5.4.0)

+In order to build the AMD SMI python package, the following components are required:
+* clang (14.0 or above)
+* python (3.7 or above)
+* virtualenv - `pip3 install virtualenv`
+
 In order to build the latest documentation, the following are required:
 * DOxygen (1.8.11)
 * latex (pdfTeX 3.14159265-2.6-1.40.16)
@@ -147,6 +152,37 @@ int main() {
 }
 ```

+# Python wrapper
+The python wrapper (binding) is an auto-generated file `py-interface/amdsmi_wrapper.py`
+
+Wrapper should be re-generated on each C++ API change, by doing:
+
+```bash
+make python_wrapper
+```
+
+After this command, the file in `py-interface/amdsmi_wrapper.py` will be automatically updated.
+
+Note: To be able to re-generate python wrapper you need several tools installed on your system: clang-14, clang-format, libclang-dev
+
+Note: python_wrapper is automatically generated with normal `make` and `make python_package` commands.
+
+Warning: If clang version is older than clang-14 - the wrapper generation will fail with a warning.
+To mitigate this issue - we include a pre-generated version of the wrapper.
+
+# Python package
+`amdsmi` package can be generated by running:
+
+```bash
+make python_package
+```
+
+It will be placed in following location: `build/py-interface/python_package/amdsmi-0.1-py3-none-any.whl`
+
+The .whl package can be installed using `pip3 install amdsmi-0.1-py3-none-any.whl` command.
+
+Warning: You must have python3.7 or above with virtualenv installed. Otherwise packaging will fail with a warning.
+
 ## DISCLAIMER

 The information contained herein is for informational purposes only, and is subject to change without notice. In addition, any stated support is planned and is also subject to change. While every precaution has been taken in the preparation of this document, it may contain technical inaccuracies, omissions and typographical errors, and AMD is under no obligation to update or otherwise correct this information. Advanced Micro Devices, Inc. makes no representations or warranties with respect to the accuracy or completeness of the contents of this document, and assumes no liability of any kind, including the implied warranties of noninfringement, merchantability or fitness for particular purposes, with respect to the operation or use of AMD hardware, software or other products described herein.
@@ -0,0 +1,154 @@
+# Generate py-interface and package targets
+
+# match this version to your clang
+# too new won't work, too old won't work either
+set(clang_ver 14.0)
+set(ctypeslib_ver 2.3.2)
+
+set(PY_BUILD_DIR "python_package" CACHE STRING "")
+# amdsmi part of this string is the directory containing all python files
+# additionally defined in pyproject.toml
+set(PY_PACKAGE_DIR "${PY_BUILD_DIR}/amdsmi" CACHE STRING "")
+
+# if Python3 is found but the version is below 3.7 - Python3_FOUND is set to FALSE
+find_package(Python3 3.7 COMPONENTS Interpreter)
+
+# WARN: This is a HACK to pass compile on AMD rhel8 CI systems!
+#       Those still use python3.6 which is too old for this project!
+# TODO: drop python3.6 support
+if(NOT Python3_FOUND)
+    message(AUTHOR_WARNING "Python3 DOESN'T EXIST OR VERSION IS TOO OLD!: ${Python3_VERSION}")
+    message(AUTHOR_WARNING "The wrapper will not be created and the project will not be packaged!")
+    # WARN: EXIT CURRENT CMAKE FILE
+    return()
+endif()
+
+# check if virtualenv is installed
+execute_process(COMMAND "${Python3_EXECUTABLE}" -m pip show virtualenv
+    ERROR_VARIABLE VIRTUALENV_NOT_FOUND)
+if(VIRTUALENV_NOT_FOUND)
+    message(FATAL_ERROR "Python virtualenv is not installed!
+${VIRTUALENV_NOT_FOUND}
+Please run:
+${Python3_EXECUTABLE} -m pip install virtualenv")
+endif()
+
+
+# set up the Python environment
+execute_process(COMMAND "${Python3_EXECUTABLE}" -m venv "${CMAKE_CURRENT_BINARY_DIR}/venv")
+
+# venv trick borrowed from:
+# https://discourse.cmake.org/t/possible-to-create-a-python-virtual-env-from-cmake-and-then-find-it-with-findpython3/1132
+# update the environment with VIRTUAL_ENV variable(mimic the activate script)
+set(ENV{VIRTUAL_ENV} "${CMAKE_CURRENT_BINARY_DIR}/venv")
+# change the context of the search
+set(Python3_FIND_VIRTUALENV FIRST)
+# unset Python3_EXECUTABLE because it is also an input variable(see documentation, Artifacts Specification section)
+unset(Python3_EXECUTABLE)
+# launch a new search
+find_package(Python3 3.7 COMPONENTS Interpreter Development REQUIRED)
+
+add_custom_target(
+    python_pre_reqs
+    COMMAND ${Python3_EXECUTABLE} -m pip install clang==${clang_ver} ctypeslib2==${ctypeslib_ver})
+
+# TODO: Figure out how python-clang and clang are related
+#       Currently only a very specific combination works
+
+# try to find clang of the right version
+set(GOOD_CLANG_FOUND FALSE)
+find_program(clang NAMES clang)
+if(clang STREQUAL "clang-NOTFOUND")
+    message(AUTHOR_WARNING "NO CLANG FOUND!")
+else()
+    # extract clang version manually because find_package(clang) doesn't work
+    execute_process(COMMAND ${clang} --version OUTPUT_VARIABLE clang_full_version_string)
+    string (REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" CLANG_VERSION_STRING ${clang_full_version_string})
+    if(CLANG_VERSION_STRING VERSION_EQUAL 14.0)
+        message("GOOD CLANG VERSION: ${CLANG_VERSION_STRING}")
+        set(GOOD_CLANG_FOUND TRUE)
+    else()
+        message(AUTHOR_WARNING "CLANG VERSION TOO OLD!: ${CLANG_VERSION_STRING}")
+    endif()
+endif()
+if(NOT GOOD_CLANG_FOUND)
+    # keep old wrapper because no clang found
+    message(AUTHOR_WARNING "A wrapper will not be generated! Using old wrapper instead.")
+    add_custom_command(
+        OUTPUT amdsmi_wrapper.py
+               ${PY_PACKAGE_DIR}/amdsmi_wrapper.py
+        DEPENDS ${AMD_SMI}
+                ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py
+        COMMAND cp -f ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py ${CMAKE_CURRENT_BINARY_DIR}/
+        # hacky alternative to configure_file that will run at MAKE compile instead of CMake configure
+        COMMAND sed -i
+                s:"@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@":"${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}":g
+                ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper.py
+        COMMAND mkdir -p ${PY_PACKAGE_DIR}
+        COMMAND ln -sf ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper.py ${PY_PACKAGE_DIR}/)
+else()
+    # generate new wrapper
+    configure_file(${PROJECT_SOURCE_DIR}/tools/generator.py generator.py @ONLY COPYONLY)
+    add_custom_command(
+        OUTPUT amdsmi.h
+               ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py
+               amdsmi_wrapper.py
+               ${PY_PACKAGE_DIR}/amdsmi_wrapper.py
+        DEPENDS ${AMD_SMI}
+                python_pre_reqs
+                generator.py
+                ${PROJECT_SOURCE_DIR}/include/amd_smi/amdsmi.h
+        COMMAND cp ${PROJECT_SOURCE_DIR}/include/amd_smi/amdsmi.h ./
+        COMMAND ${Python3_EXECUTABLE} generator.py -i amdsmi.h -l ${PROJECT_BINARY_DIR}/src/libamd_smi.so -o ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py
+        COMMAND cp -f ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_wrapper.py ${CMAKE_CURRENT_BINARY_DIR}/
+        # hacky alternative to configure_file that will run at MAKE compile instead of CMake configure
+        COMMAND sed -i
+                s:"@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@":"${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}":g
+                ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper.py
+        COMMAND mkdir -p ${PY_PACKAGE_DIR}
+        COMMAND ln -sf ${CMAKE_CURRENT_BINARY_DIR}/amdsmi_wrapper.py ${PY_PACKAGE_DIR}/)
+endif()
+
+add_custom_target(
+    python_wrapper
+    DEPENDS amdsmi_wrapper.py)
+
+# symlinking instead of copying avoids unnecessarry regeneration of packaged files
+add_custom_command(
+    OUTPUT ${PY_BUILD_DIR}/pyproject.toml
+           ${PY_PACKAGE_DIR}/__init__.py
+           ${PY_PACKAGE_DIR}/amdsmi_exception.py
+           ${PY_PACKAGE_DIR}/amdsmi_interface.py
+           ${PY_PACKAGE_DIR}/README.md
+           ${PY_PACKAGE_DIR}/LICENSE
+    DEPENDS python_wrapper
+    COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml ${PY_BUILD_DIR}/
+    COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py ${PY_PACKAGE_DIR}/
+    COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_exception.py ${PY_PACKAGE_DIR}/
+    COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/amdsmi_interface.py ${PY_PACKAGE_DIR}/
+    COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/README.md ${PY_PACKAGE_DIR}/
+    COMMAND ln -sf ${PROJECT_SOURCE_DIR}/LICENSE ${PY_PACKAGE_DIR}/)
+
+# NOTE: changing this does not change the generated file!
+# it WILL break
+set(PY_PACKAGE "${PY_BUILD_DIR}/amdsmi-0.1-py3-none-any.whl")
+
+add_custom_command(
+    OUTPUT ${PY_PACKAGE}
+    DEPENDS ${PY_BUILD_DIR}/pyproject.toml
+            ${PY_PACKAGE_DIR}/__init__.py
+            ${PY_PACKAGE_DIR}/amdsmi_exception.py
+            ${PY_PACKAGE_DIR}/amdsmi_interface.py
+            ${PY_PACKAGE_DIR}/README.md
+            ${PY_PACKAGE_DIR}/LICENSE
+    COMMAND ${Python3_EXECUTABLE} -m pip install wheel
+    COMMAND ${Python3_EXECUTABLE} -m pip wheel ./${PY_BUILD_DIR} --wheel-dir=${PY_BUILD_DIR})
+
+add_custom_target(
+    python_package ALL
+    DEPENDS ${PY_PACKAGE})
+
+install(
+    PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${PY_PACKAGE}
+    DESTINATION ${SHARE_INSTALL_PREFIX})
+
@@ -23,7 +23,7 @@
 import os
 # -*- coding: utf-8 -*-
 #
-# TARGET arch is: ['-I/usr/lib/llvm-6.0/lib/clang/6.0.0/include']
+# TARGET arch is: ['-I/usr/lib/llvm-14/lib/clang/14.0.0/include']
 # WORD_SIZE is: 8
 # POINTER_SIZE is: 8
 # LONGDOUBLE_SIZE is: 16
@@ -167,7 +167,17 @@ def char_pointer_cast(string, encoding='utf-8'):


 _libraries = {}
-_libraries['libamd_smi.so'] = ctypes.CDLL(os.path.join(os.path.dirname(__file__), 'libamd_smi.so'))
+
+if os.path.isfile('@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/libamd_smi.so'):
+    # try to find library in install directory provided by CMake
+    _libraries['libamd_smi.so'] = ctypes.CDLL(os.path.join('@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@', 'libamd_smi.so'))
+elif os.path.isfile('/opt/rocm/lib/libamd_smi.so'):
+    # try /opt/rocm/lib as a fallback
+    _libraries['libamd_smi.so'] = ctypes.CDLL(os.path.join('/opt/rocm/lib', 'libamd_smi.so'))
+else:
+    # lastly - search in current directory
+    _libraries['libamd_smi.so'] = ctypes.CDLL(os.path.join(os.path.dirname(__file__), 'libamd_smi.so'))
+



@@ -697,6 +707,16 @@ amdsmi_process_handle = ctypes.c_uint32
 class struct_c__SA_amdsmi_proc_info_t(Structure):
    pass

+class struct_c__SA_amdsmi_proc_info_t_1(Structure):
+    pass
+
+struct_c__SA_amdsmi_proc_info_t_1._pack_ = 1 # source:False
+struct_c__SA_amdsmi_proc_info_t_1._fields_ = [
+    ('gtt_mem', ctypes.c_uint64),
+    ('cpu_mem', ctypes.c_uint64),
+    ('vram_mem', ctypes.c_uint64),
+]
+
 class struct_c__SA_amdsmi_proc_info_t_0(Structure):
    pass

@@ -709,16 +729,6 @@ struct_c__SA_amdsmi_proc_info_t_0._fields_ = [
    ('dec', ctypes.c_uint64),
 ]

-class struct_c__SA_amdsmi_proc_info_t_1(Structure):
-    pass
-
-struct_c__SA_amdsmi_proc_info_t_1._pack_ = 1 # source:False
-struct_c__SA_amdsmi_proc_info_t_1._fields_ = [
-    ('gtt_mem', ctypes.c_uint64),
-    ('cpu_mem', ctypes.c_uint64),
-    ('vram_mem', ctypes.c_uint64),
-]
-
 struct_c__SA_amdsmi_proc_info_t._pack_ = 1 # source:False
 struct_c__SA_amdsmi_proc_info_t._fields_ = [
    ('name', ctypes.c_char * 32),
@@ -0,0 +1,23 @@
+# for details see:
+# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "amdsmi"
+authors = [
+    {name = "AMD", email = "amd-smi.support@amd.com"},
+]
+version = '0.1'
+license = {file = "amdsmi/LICENSE"}
+readme = {file = "amdsmi/README.md", content-type = "text/markdown"}
+description = "SMI LIB - AMD GPU Monitoring Library"
+requires-python = ">=3.7"
+
+[project.urls]
+"Homepage" = "https://github.com/RadeonOpenCompute/amdsmi"
+
+[tool.setuptools]
+packages = ["amdsmi"]
@@ -1,16 +0,0 @@
-from setuptools import setup
-
-with open("amdsmi/README.md", "r", encoding="utf-8") as fh:
-    long_description = fh.read()
-
-setup(
-    name='amdsmi',
-    version='0.1',
-    description="SMI LIB - AMD GPU Monitoring Library",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    packages=['amdsmi'],
-    package_data={'': ['LICENSE']},
-    include_package_data=True,
-    python_requires=">=3.6",
-)
@@ -106,8 +106,17 @@ def main():
        arguments = [input_file, "-o", output_file, "-l", library]
        library_path = os.path.join(os.path.dirname(__file__), library)
        line_to_replace = "_libraries['{}'] = ctypes.CDLL('{}')".format(library_name, library_path)
-        new_line = "_libraries['{}'] = ctypes.CDLL(os.path.join(os.path.dirname(__file__), '{}'))" \
-            .format(library_name, library_name)
+        new_line = """
+if os.path.isfile('@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/{0}'):
+    # try to find library in install directory provided by CMake
+    _libraries['{0}'] = ctypes.CDLL(os.path.join('@CPACK_PACKAGING_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@', '{0}'))
+elif os.path.isfile('/opt/rocm/lib/{0}'):
+    # try /opt/rocm/lib as a fallback
+    _libraries['{0}'] = ctypes.CDLL(os.path.join('/opt/rocm/lib', '{0}'))
+else:
+    # lastly - search in current directory
+    _libraries['{0}'] = ctypes.CDLL(os.path.join(os.path.dirname(__file__), '{0}'))
+""".format(library_name)
    else:
        print("Unknown operating system. It is only supporing Linux and Windows.")
        return