From 333b3b7ac45099f7b86ab35ac89152293ac7451d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 2 Apr 2023 17:33:56 +0200 Subject: [PATCH] ci: use cibuildwheel to build linux wheel packages --- .github/workflows/packages.yml | 71 +++++++++++++---------- scripts/build/build_libpq.sh | 4 +- scripts/build/build_manylinux2014.sh | 76 ------------------------- scripts/build/build_musllinux_1_1.sh | 68 ---------------------- scripts/build/print_so_versions.sh | 37 ++++++++++++ scripts/build/strip_wheel.sh | 34 ++++++----- scripts/build/wheel_linux_before_all.sh | 53 +++++++++++++++++ 7 files changed, 153 insertions(+), 190 deletions(-) delete mode 100755 scripts/build/build_manylinux2014.sh delete mode 100755 scripts/build/build_musllinux_1_1.sh create mode 100755 scripts/build/print_so_versions.sh create mode 100755 scripts/build/wheel_linux_before_all.sh diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 07fe2151..a7fde4da 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -52,20 +52,14 @@ jobs: --health-retries 5 - build-manylinux: + build-linux: if: true strategy: fail-fast: false matrix: - include: - - {tag: manylinux2014, arch: x86_64} - - {tag: manylinux2014, arch: i686} - - {tag: manylinux2014, arch: aarch64} - - {tag: manylinux2014, arch: ppc64le} - - {tag: musllinux_1_1, arch: x86_64} - - {tag: musllinux_1_1, arch: i686} - - {tag: musllinux_1_1, arch: aarch64} - - {tag: musllinux_1_1, arch: ppc64le} + platform: [manylinux, musllinux] + arch: [x86_64, i686, aarch64, ppc64le] + pyver: [cp37, cp38, cp39, cp310, cp311] runs-on: ubuntu-latest steps: @@ -75,31 +69,46 @@ jobs: - name: Set up QEMU for multi-arch build uses: docker/setup-qemu-action@v2 - - name: Build packages - run: >- - docker run --rm - -e PLAT=${{ matrix.tag }}_${{ matrix.arch }} - -e PACKAGE_NAME=psycopg2-binary - -e PYVERS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311" - -e PSYCOPG2_TESTDB=postgres - -e PSYCOPG2_TESTDB_HOST=172.17.0.1 - -e PSYCOPG2_TESTDB_USER=postgres - -e PSYCOPG2_TESTDB_PASSWORD=password - -e PSYCOPG2_TEST_FAST=1 - -v `pwd`:/src - --workdir /src - quay.io/pypa/${{ matrix.tag }}_${{ matrix.arch }} - ./scripts/build/build_${{ matrix.tag }}.sh - - - name: Upload artifacts - uses: actions/upload-artifact@v3 + - name: Cache libpq build + uses: actions/cache@v3 with: - path: | - dist/*.whl + path: /tmp/libpq.build + key: libpq-${{ env.LIBPQ_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }} + + - name: Build wheels + uses: pypa/cibuildwheel@v2.12.0 + env: + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_I686_IMAGE: manylinux2014 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 + CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014 + CIBW_BUILD: ${{matrix.pyver}}-${{matrix.platform}}_${{matrix.arch}} + CIBW_ARCHS_LINUX: auto aarch64 ppc64le + CIBW_BEFORE_ALL_LINUX: ./scripts/build/wheel_linux_before_all.sh + CIBW_REPAIR_WHEEL_COMMAND: >- + ./scripts/build/strip_wheel.sh {wheel} + && auditwheel repair -w {dest_dir} {wheel} + CIBW_TEST_COMMAND: >- + export PYTHONPATH={project} && + python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" + CIBW_ENVIRONMENT: >- + PACKAGE_NAME=psycopg2-binary + LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build + PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH" + LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64" + PSYCOPG2_TESTDB=postgres + PSYCOPG2_TESTDB_HOST=172.17.0.1 + PSYCOPG2_TESTDB_USER=postgres + PSYCOPG2_TESTDB_PASSWORD=password + PSYCOPG2_TEST_FAST=1 + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl services: postgresql: - image: postgres:13 + image: postgres:14 env: POSTGRES_PASSWORD: password ports: diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 78822fff..0a413c42 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -18,7 +18,7 @@ ldap_version="2.6.3" # last release: https://github.com/cyrusimap/cyrus-sasl/releases sasl_version="2.1.28" -export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/usr/local} +export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build} if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then echo "libpq already available: build skipped" >&2 @@ -152,7 +152,7 @@ if [ ! -d "${postgres_dir}" ]; then src/include/pg_config_manual.h # Often needed, but currently set by the workflow - export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib;${LIBPQ_BUILD_PREFIX}/lib64" + # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib" ./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \ --without-readline --with-gssapi --with-openssl --with-pam --with-ldap \ diff --git a/scripts/build/build_manylinux2014.sh b/scripts/build/build_manylinux2014.sh deleted file mode 100755 index 49a2d337..00000000 --- a/scripts/build/build_manylinux2014.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Create manylinux2014 wheels for psycopg2 -# -# manylinux2014 is built on CentOS 7, which packages an old version of the -# libssl, (1.0, which has concurrency problems with the Python libssl). So we -# need to build these libraries from source. -# -# Look at the .github/workflows/packages.yml file for hints about how to use it. - -set -euo pipefail -set -x - -dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -prjdir="$( cd "${dir}/../.." && pwd )" - -# Build all the available versions, or just the ones specified in PYVERS -if [ ! "${PYVERS:-}" ]; then - PYVERS="$(ls /opt/python/)" -fi - -# Find psycopg version -version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/") -# A gratuitous comment to fix broken vim syntax file: '") -distdir="${prjdir}/dist" - -# Replace the package name -if [[ "${PACKAGE_NAME:-}" ]]; then - sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ - "${prjdir}/setup.py" -fi - -# Build depending libraries -"${dir}/build_libpq.sh" > /dev/null - -# Create the wheel packages -for pyver in $PYVERS; do - pybin="/opt/python/${pyver}/bin" - "${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/" -done - -# Bundle external shared libraries into the wheels -for whl in "${prjdir}"/dist/*.whl; do - "${dir}/strip_wheel.sh" "$whl" - auditwheel repair "$whl" -w "$distdir" -done - -# Make sure the libpq is not in the system -for f in $(find /usr/local/lib -name libpq\*) ; do - mkdir -pv "/libpqbak/$(dirname $f)" - mv -v "$f" "/libpqbak/$(dirname $f)" -done - -# Install packages and test -cd "${prjdir}" -for pyver in $PYVERS; do - pybin="/opt/python/${pyver}/bin" - "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" - - # Print psycopg and libpq versions - "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)" - "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)" - "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())" - - # Fail if we are not using the expected libpq library - if [[ "${WANT_LIBPQ:-}" ]]; then - "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" - fi - - "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" -done - -# Restore the libpq packages -for f in $(cd /libpqbak/ && find . -not -type d); do - mv -v "/libpqbak/$f" "/$f" -done diff --git a/scripts/build/build_musllinux_1_1.sh b/scripts/build/build_musllinux_1_1.sh deleted file mode 100755 index 62860dcb..00000000 --- a/scripts/build/build_musllinux_1_1.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -# Create musllinux_1_1 wheels for psycopg2 -# -# Look at the .github/workflows/packages.yml file for hints about how to use it. - -set -euo pipefail -set -x - -dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -prjdir="$( cd "${dir}/../.." && pwd )" - -# Build all the available versions, or just the ones specified in PYVERS -if [ ! "${PYVERS:-}" ]; then - PYVERS="$(ls /opt/python/)" -fi - -# Find psycopg version -version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/") -# A gratuitous comment to fix broken vim syntax file: '") -distdir="${prjdir}/dist/psycopg2-$version" - -# Replace the package name -if [[ "${PACKAGE_NAME:-}" ]]; then - sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ - "${prjdir}/setup.py" -fi - -# Install prerequisite libraries -apk update -apk add postgresql-dev -# Add findutils because the Busybox version lacks the `-ls` flag, used by the -# `strip_wheel.sh` script. -apk add findutils - -# Create the wheel packages -for pyver in $PYVERS; do - pybin="/opt/python/${pyver}/bin" - "${pybin}/python" -m build -w -o "${prjdir}/dist/" "${prjdir}" -done - -# Bundle external shared libraries into the wheels -for whl in "${prjdir}"/dist/*.whl; do - "${dir}/strip_wheel.sh" "$whl" - auditwheel repair "$whl" -w "$distdir" -done - -# Make sure the postgresql-dev is not in the system -apk del postgresql-dev - -# Install packages and test -cd "${prjdir}" -for pyver in $PYVERS; do - pybin="/opt/python/${pyver}/bin" - "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" - - # Print psycopg and libpq versions - "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)" - "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)" - "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())" - - # Fail if we are not using the expected libpq library - if [[ "${WANT_LIBPQ:-}" ]]; then - "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" - fi - - "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" -done diff --git a/scripts/build/print_so_versions.sh b/scripts/build/print_so_versions.sh new file mode 100755 index 00000000..a3c4ecdc --- /dev/null +++ b/scripts/build/print_so_versions.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Take a .so file as input and print the Debian packages and versions of the +# libraries it links. + +set -euo pipefail +# set -x + +source /etc/os-release + +sofile="$1" + +case "$ID" in + alpine) + depfiles=$( (ldd "$sofile" 2>/dev/null || true) | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/') + (for depfile in $depfiles; do + echo "$(basename "$depfile") => $(apk info --who-owns "${depfile}" | awk '{print $(NF)}')" + done) | sort | uniq + ;; + + debian) + depfiles=$(ldd "$sofile" | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/') + (for depfile in $depfiles; do + pkgname=$(dpkg -S "${depfile}" | sed 's/\(\): .*/\1/') + dpkg -l "${pkgname}" | grep '^ii' | awk '{print $2 " => " $3}' + done) | sort | uniq + ;; + + centos) + echo "TODO!" + ;; + + *) + echo "$0: unexpected Linux distribution: '$ID'" >&2 + exit 1 + ;; +esac diff --git a/scripts/build/strip_wheel.sh b/scripts/build/strip_wheel.sh index 0747f8c6..83e94172 100755 --- a/scripts/build/strip_wheel.sh +++ b/scripts/build/strip_wheel.sh @@ -14,28 +14,36 @@ # This script is designed to run on a wheel archive before auditwheel. set -euo pipefail -set -x +# set -x + +source /etc/os-release +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" wheel=$(realpath "$1") shift -# python or python3? -if which python > /dev/null; then - py=python -else - py=python3 -fi - tmpdir=$(mktemp -d) trap "rm -r ${tmpdir}" EXIT cd "${tmpdir}" -$py -m zipfile -e "${wheel}" . +python -m zipfile -e "${wheel}" . -find . -name *.so -ls -exec strip "$@" {} \; -# Display the size after strip -find . -name *.so -ls +echo " +Libs before:" +# Busybox doesn't have "find -ls" +find . -name \*.so | xargs ls -l -$py -m zipfile -c "${wheel}" * +# On Debian, print the package versions libraries come from +echo " +Dependencies versions of '_psycopg.so' library:" +"${dir}/print_so_versions.sh" "$(find . -name \*_psycopg\*.so)" + +find . -name \*.so -exec strip "$@" {} \; + +echo " +Libs after:" +find . -name \*.so | xargs ls -l + +python -m zipfile -c ${wheel} * cd - diff --git a/scripts/build/wheel_linux_before_all.sh b/scripts/build/wheel_linux_before_all.sh new file mode 100755 index 00000000..6be565ec --- /dev/null +++ b/scripts/build/wheel_linux_before_all.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Configure the libraries needed to build wheel packages on linux. +# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_LINUX + +set -euo pipefail +set -x + +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +prjdir="$( cd "${dir}/../.." && pwd )" + +source /etc/os-release + +# Install PostgreSQL development files. +case "$ID" in + alpine) + "${dir}/build_libpq.sh" > /dev/null + ;; + + debian) + # Note that the pgdg doesn't have an aarch64 repository so wheels are + # build with the libpq packaged with Debian 9, which is 9.6. + if [ "$AUDITWHEEL_ARCH" != 'aarch64' ]; then + echo "deb http://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main" \ + > /etc/apt/sources.list.d/pgdg.list + # TODO: On 2021-11-09 curl fails on 'ppc64le' with: + # curl: (60) SSL certificate problem: certificate has expired + # Test again later if -k can be removed. + curl -skf https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + > /etc/apt/trusted.gpg.d/postgresql.asc + fi + + apt-get update + apt-get -y upgrade + apt-get -y install libpq-dev + ;; + + centos) + "${dir}/build_libpq.sh" > /dev/null + ;; + + *) + echo "$0: unexpected Linux distribution: '$ID'" >&2 + exit 1 + ;; +esac + +# Replace the package name +if [[ "${PACKAGE_NAME:-}" ]]; then + sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ + "${prjdir}/setup.py" +fi +