From a49185987509ca0cde58f7eb7a66518e48aae86d Mon Sep 17 00:00:00 2001 From: Franck Nijhof Date: Sat, 22 Apr 2023 17:23:02 +0200 Subject: [PATCH] Several changes to Wheels building for the cp311 ABI (#91850) --- .github/workflows/wheels.yml | 129 +++++++++++++++++++++++++++++++++-- 1 file changed, 125 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index e18ca933d68..a4511c369aa 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -72,7 +72,7 @@ jobs: path: ./requirements_diff.txt core: - name: Build musllinux wheels with musllinux_1_2 / ${{ matrix.abi }} at ${{ matrix.arch }} for core + name: Build Core wheels ${{ matrix.abi }} for ${{ matrix.arch }} (musllinux_1_2) if: github.repository_owner == 'home-assistant' needs: init runs-on: ubuntu-latest @@ -109,15 +109,15 @@ jobs: requirements-diff: "requirements_diff.txt" requirements: "requirements.txt" - integrations: - name: Build musllinux wheels with musllinux_1_2 / ${{ matrix.abi }} at ${{ matrix.arch }} for integrations + integrations_cp310: + name: Build wheels ${{ matrix.abi }} for ${{ matrix.arch }} if: github.repository_owner == 'home-assistant' needs: init runs-on: ubuntu-latest strategy: fail-fast: false matrix: - abi: ["cp310", "cp311"] + abi: ["cp310"] arch: ${{ fromJson(needs.init.outputs.architectures) }} steps: - name: Checkout the repository @@ -201,3 +201,124 @@ jobs: constraints: "homeassistant/package_constraints.txt" requirements-diff: "requirements_diff.txt" requirements: "requirements_all.txtab" + + # Wheels building for the cp311 ABI is currently split + # This is mainly until we have figured out to get all wheels built. + # Without harming our current workflow. + integrations_cp311: + name: Build wheels ${{ matrix.abi }} for ${{ matrix.arch }} + if: github.repository_owner == 'home-assistant' + needs: init + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + abi: ["cp311"] + arch: ${{ fromJson(needs.init.outputs.architectures) }} + steps: + - name: Checkout the repository + uses: actions/checkout@v3.5.2 + + - name: Write alternative env-file for cp311 + run: | + ( + echo "GRPC_BUILD_WITH_BORING_SSL_ASM=false" + echo "GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=true" + echo "GRPC_PYTHON_BUILD_WITH_CYTHON=true" + echo "GRPC_PYTHON_DISABLE_LIBC_COMPATIBILITY=true" + + # GRPC on armv7 needed -lexecinfo (issue #56669) since home assistant installed + # execinfo-dev when building wheels. However, this package is no longer available + # Alpine 3.17, which we use for the cp311 ABI, so the flag should no longer be needed. + echo "GRPC_PYTHON_LDFLAGS=-lpthread -Wl,-wrap,memcpy -static-libgcc" # -lexecinfo + + # Fix out of memory issues with rust + echo "CARGO_NET_GIT_FETCH_WITH_CLI=true" + + # OpenCV headless installation + echo "CI_BUILD=1" + echo "ENABLE_HEADLESS=1" + + # Use C-Extension for sqlalchemy + echo "REQUIRE_SQLALCHEMY_CEXT=1" + ) > .env_file + + - name: Download requirements_diff + uses: actions/download-artifact@v3 + with: + name: requirements_diff + + - name: Uncomment packages + run: | + requirement_files="requirements_all.txt requirements_diff.txt" + for requirement_file in ${requirement_files}; do + + # PyBluez no longer compiles. Commented it out for now. + # It need further cleanup down the line, as all machine images + # try to install it. + # sed -i "s|# pybluez|pybluez|g" ${requirement_file} + + sed -i "s|# beacontools|beacontools|g" ${requirement_file} + sed -i "s|# fritzconnection|fritzconnection|g" ${requirement_file} + sed -i "s|# pyuserinput|pyuserinput|g" ${requirement_file} + sed -i "s|# evdev|evdev|g" ${requirement_file} + sed -i "s|# pycups|pycups|g" ${requirement_file} + sed -i "s|# homekit|homekit|g" ${requirement_file} + sed -i "s|# decora_wifi|decora_wifi|g" ${requirement_file} + sed -i "s|# python-gammu|python-gammu|g" ${requirement_file} + sed -i "s|# opencv-python-headless|opencv-python-headless|g" ${requirement_file} + done + + - name: Split requirements all + run: | + # We split requirements all into two different files. + # This is to prevent the build from running out of memory when + # resolving packages on 32-bits systems (like armhf, armv7). + + split -l $(expr $(expr $(cat requirements_all.txt | wc -l) + 1) / 2) requirements_all.txt requirements_all.txt + + - name: Adjust build env + run: | + if [ "${{ matrix.arch }}" = "i386" ]; then + echo "NPY_DISABLE_SVML=1" >> .env_file + fi + + # Probably not an issue anymore. Removing for now. + # ( + # # cmake > 3.22.2 have issue on arm + # # Tested until 3.22.5 + # echo "cmake==3.22.2" + # ) >> homeassistant/package_constraints.txt + + # Do not pin numpy in wheels building + sed -i "/numpy/d" homeassistant/package_constraints.txt + + - name: Build wheels (part 1) + uses: home-assistant/wheels@2023.04.0 + with: + abi: ${{ matrix.abi }} + tag: musllinux_1_2 + arch: ${{ matrix.arch }} + wheels-key: ${{ secrets.WHEELS_KEY }} + env-file: true + apk: "bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;cups-dev;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev;gammu-dev;yaml-dev;openblas-dev;fftw-dev;lapack-dev;gfortran;blas-dev;eigen-dev;freetype-dev;glew-dev;harfbuzz-dev;hdf5-dev;libdc1394-dev;libtbb-dev;mesa-dev;openexr-dev;openjpeg-dev;openssl1.1-compat-dev;uchardet-dev" + skip-binary: aiohttp;grpcio;sqlalchemy;protobuf + legacy: true + constraints: "homeassistant/package_constraints.txt" + requirements-diff: "requirements_diff.txt" + requirements: "requirements_all.txtaa" + + - name: Build wheels (part 2) + uses: home-assistant/wheels@2023.04.0 + with: + abi: ${{ matrix.abi }} + tag: musllinux_1_2 + arch: ${{ matrix.arch }} + wheels-key: ${{ secrets.WHEELS_KEY }} + env-file: true + apk: "bluez-dev;libffi-dev;openssl-dev;glib-dev;eudev-dev;libxml2-dev;libxslt-dev;libpng-dev;libjpeg-turbo-dev;tiff-dev;cups-dev;gmp-dev;mpfr-dev;mpc1-dev;ffmpeg-dev;gammu-dev;yaml-dev;openblas-dev;fftw-dev;lapack-dev;gfortran;blas-dev;eigen-dev;freetype-dev;glew-dev;harfbuzz-dev;hdf5-dev;libdc1394-dev;libtbb-dev;mesa-dev;openexr-dev;openjpeg-dev;openssl1.1-compat-dev;uchardet-dev" + skip-binary: aiohttp;grpcio;sqlalchemy;protobuf + legacy: true + constraints: "homeassistant/package_constraints.txt" + requirements-diff: "requirements_diff.txt" + requirements: "requirements_all.txtab"