ci: use cibuildwheel to build linux wheel packages

.github/workflows/packages.yml

@@ -52,20 +52,14 @@ jobs:
           --health-retries 5
 
 
-  build-manylinux:
+  build-linux:
     if: true
     strategy:
       fail-fast: false
       matrix:
-        include:
+        platform: [manylinux, musllinux]
-          - {tag: manylinux2014, arch: x86_64}
+        arch: [x86_64, i686, aarch64, ppc64le]
-          - {tag: manylinux2014, arch: i686}
+        pyver: [cp37, cp38, cp39, cp310, cp311]
-          - {tag: manylinux2014, arch: aarch64}
-          - {tag: manylinux2014, arch: ppc64le}
-          - {tag: musllinux_1_1, arch: x86_64}
-          - {tag: musllinux_1_1, arch: i686}
-          - {tag: musllinux_1_1, arch: aarch64}
-          - {tag: musllinux_1_1, arch: ppc64le}
 
     runs-on: ubuntu-latest
     steps:
@@ -75,31 +69,46 @@ jobs:
       - name: Set up QEMU for multi-arch build
         uses: docker/setup-qemu-action@v2
 
-      - name: Build packages
+      - name: Cache libpq build
-        run: >-
+        uses: actions/cache@v3
-          docker run --rm
-          -e PLAT=${{ matrix.tag }}_${{ matrix.arch }}
-          -e PACKAGE_NAME=psycopg2-binary
-          -e PYVERS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311"
-          -e PSYCOPG2_TESTDB=postgres
-          -e PSYCOPG2_TESTDB_HOST=172.17.0.1
-          -e PSYCOPG2_TESTDB_USER=postgres
-          -e PSYCOPG2_TESTDB_PASSWORD=password
-          -e PSYCOPG2_TEST_FAST=1
-          -v `pwd`:/src
-          --workdir /src
-          quay.io/pypa/${{ matrix.tag }}_${{ matrix.arch }}
-          ./scripts/build/build_${{ matrix.tag }}.sh
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v3
         with:
-          path: |
+          path: /tmp/libpq.build
-            dist/*.whl
+          key: libpq-${{ env.LIBPQ_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.12.0
+        env:
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
+          CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014
+          CIBW_BUILD: ${{matrix.pyver}}-${{matrix.platform}}_${{matrix.arch}}
+          CIBW_ARCHS_LINUX: auto aarch64 ppc64le
+          CIBW_BEFORE_ALL_LINUX: ./scripts/build/wheel_linux_before_all.sh
+          CIBW_REPAIR_WHEEL_COMMAND: >-
+            ./scripts/build/strip_wheel.sh {wheel}
+            && auditwheel repair -w {dest_dir} {wheel}
+          CIBW_TEST_COMMAND: >-
+            export PYTHONPATH={project} &&
+            python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
+          CIBW_ENVIRONMENT: >-
+            PACKAGE_NAME=psycopg2-binary
+            LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build
+            PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
+            LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64"
+            PSYCOPG2_TESTDB=postgres
+            PSYCOPG2_TESTDB_HOST=172.17.0.1
+            PSYCOPG2_TESTDB_USER=postgres
+            PSYCOPG2_TESTDB_PASSWORD=password
+            PSYCOPG2_TEST_FAST=1
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
 
     services:
       postgresql:
-        image: postgres:13
+        image: postgres:14
         env:
           POSTGRES_PASSWORD: password
         ports:

scripts/build/build_libpq.sh

@@ -18,7 +18,7 @@ ldap_version="2.6.3"
 # last release: https://github.com/cyrusimap/cyrus-sasl/releases
 sasl_version="2.1.28"
 
-export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/usr/local}
+export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build}
 
 if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then
     echo "libpq already available: build skipped" >&2
@@ -152,7 +152,7 @@ if [ ! -d "${postgres_dir}" ]; then
         src/include/pg_config_manual.h
 
     # Often needed, but currently set by the workflow
-    export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib;${LIBPQ_BUILD_PREFIX}/lib64"
+    # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib"
 
     ./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \
         --without-readline --with-gssapi --with-openssl --with-pam --with-ldap \

scripts/build/build_manylinux2014.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-
-# Create manylinux2014 wheels for psycopg2
-#
-# manylinux2014 is built on CentOS 7, which packages an old version of the
-# libssl, (1.0, which has concurrency problems with the Python libssl). So we
-# need to build these libraries from source.
-#
-# Look at the .github/workflows/packages.yml file for hints about how to use it.
-
-set -euo pipefail
-set -x
-
-dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-prjdir="$( cd "${dir}/../.." && pwd )"
-
-# Build all the available versions, or just the ones specified in PYVERS
-if [ ! "${PYVERS:-}" ]; then
-    PYVERS="$(ls /opt/python/)"
-fi
-
-# Find psycopg version
-version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/")
-# A gratuitous comment to fix broken vim syntax file: '")
-distdir="${prjdir}/dist"
-
-# Replace the package name
-if [[ "${PACKAGE_NAME:-}" ]]; then
-    sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
-        "${prjdir}/setup.py"
-fi
-
-# Build depending libraries
-"${dir}/build_libpq.sh" > /dev/null
-
-# Create the wheel packages
-for pyver in $PYVERS; do
-    pybin="/opt/python/${pyver}/bin"
-    "${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/"
-done
-
-# Bundle external shared libraries into the wheels
-for whl in "${prjdir}"/dist/*.whl; do
-    "${dir}/strip_wheel.sh" "$whl"
-    auditwheel repair "$whl" -w "$distdir"
-done
-
-# Make sure the libpq is not in the system
-for f in $(find /usr/local/lib -name libpq\*) ; do
-    mkdir -pv "/libpqbak/$(dirname $f)"
-    mv -v "$f" "/libpqbak/$(dirname $f)"
-done
-
-# Install packages and test
-cd "${prjdir}"
-for pyver in $PYVERS; do
-    pybin="/opt/python/${pyver}/bin"
-    "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
-
-    # Print psycopg and libpq versions
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)"
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
-
-    # Fail if we are not using the expected libpq library
-    if [[ "${WANT_LIBPQ:-}" ]]; then
-        "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
-    fi
-
-    "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
-done
-
-# Restore the libpq packages
-for f in $(cd /libpqbak/ && find . -not -type d); do
-    mv -v "/libpqbak/$f" "/$f"
-done

scripts/build/build_musllinux_1_1.sh

@@ -1,68 +0,0 @@
-#!/bin/bash
-
-# Create musllinux_1_1 wheels for psycopg2
-#
-# Look at the .github/workflows/packages.yml file for hints about how to use it.
-
-set -euo pipefail
-set -x
-
-dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-prjdir="$( cd "${dir}/../.." && pwd )"
-
-# Build all the available versions, or just the ones specified in PYVERS
-if [ ! "${PYVERS:-}" ]; then
-    PYVERS="$(ls /opt/python/)"
-fi
-
-# Find psycopg version
-version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/")
-# A gratuitous comment to fix broken vim syntax file: '")
-distdir="${prjdir}/dist/psycopg2-$version"
-
-# Replace the package name
-if [[ "${PACKAGE_NAME:-}" ]]; then
-    sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
-        "${prjdir}/setup.py"
-fi
-
-# Install prerequisite libraries
-apk update
-apk add postgresql-dev
-# Add findutils because the Busybox version lacks the `-ls` flag, used by the
-# `strip_wheel.sh` script.
-apk add findutils
-
-# Create the wheel packages
-for pyver in $PYVERS; do
-    pybin="/opt/python/${pyver}/bin"
-    "${pybin}/python" -m build -w -o "${prjdir}/dist/" "${prjdir}"
-done
-
-# Bundle external shared libraries into the wheels
-for whl in "${prjdir}"/dist/*.whl; do
-    "${dir}/strip_wheel.sh" "$whl"
-    auditwheel repair "$whl" -w "$distdir"
-done
-
-# Make sure the postgresql-dev is not in the system
-apk del postgresql-dev
-
-# Install packages and test
-cd "${prjdir}"
-for pyver in $PYVERS; do
-    pybin="/opt/python/${pyver}/bin"
-    "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
-
-    # Print psycopg and libpq versions
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)"
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
-
-    # Fail if we are not using the expected libpq library
-    if [[ "${WANT_LIBPQ:-}" ]]; then
-        "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
-    fi
-
-    "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
-done

scripts/build/print_so_versions.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Take a .so file as input and print the Debian packages and versions of the
+# libraries it links.
+
+set -euo pipefail
+# set -x
+
+source /etc/os-release
+
+sofile="$1"
+
+case "$ID" in
+    alpine)
+        depfiles=$( (ldd "$sofile" 2>/dev/null || true) | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/')
+        (for depfile in $depfiles; do
+             echo "$(basename "$depfile") => $(apk info --who-owns "${depfile}" | awk '{print $(NF)}')"
+        done) | sort | uniq
+        ;;
+
+    debian)
+        depfiles=$(ldd "$sofile" | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/')
+        (for depfile in $depfiles; do
+            pkgname=$(dpkg -S "${depfile}" | sed 's/\(\): .*/\1/')
+            dpkg -l "${pkgname}" | grep '^ii' | awk '{print $2 " => " $3}'
+        done) | sort | uniq
+        ;;
+
+    centos)
+        echo "TODO!"
+        ;;
+
+    *)
+        echo "$0: unexpected Linux distribution: '$ID'" >&2
+        exit 1
+        ;;
+esac

scripts/build/strip_wheel.sh

@@ -14,28 +14,36 @@
 # This script is designed to run on a wheel archive before auditwheel.
 
 set -euo pipefail
-set -x
+# set -x
+
+source /etc/os-release
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 wheel=$(realpath "$1")
 shift
 
-# python or python3?
-if which python > /dev/null; then
-    py=python
-else
-    py=python3
-fi
-
 tmpdir=$(mktemp -d)
 trap "rm -r ${tmpdir}" EXIT
 
 cd "${tmpdir}"
-$py -m zipfile -e "${wheel}" .
+python -m zipfile -e "${wheel}" .
 
-find . -name *.so -ls -exec strip "$@" {} \;
+echo "
-# Display the size after strip
+Libs before:"
-find . -name *.so -ls
+# Busybox doesn't have "find -ls"
+find . -name \*.so | xargs ls -l
 
-$py -m zipfile -c "${wheel}" *
+# On Debian, print the package versions libraries come from
+echo "
+Dependencies versions of '_psycopg.so' library:"
+"${dir}/print_so_versions.sh" "$(find . -name \*_psycopg\*.so)"
+
+find . -name \*.so -exec strip "$@" {} \;
+
+echo "
+Libs after:"
+find . -name \*.so | xargs ls -l
+
+python -m zipfile -c ${wheel} *
 
 cd -

scripts/build/wheel_linux_before_all.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Configure the libraries needed to build wheel packages on linux.
+# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_LINUX
+
+set -euo pipefail
+set -x
+
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+prjdir="$( cd "${dir}/../.." && pwd )"
+
+source /etc/os-release
+
+# Install PostgreSQL development files.
+case "$ID" in
+    alpine)
+        "${dir}/build_libpq.sh" > /dev/null
+        ;;
+
+    debian)
+        # Note that the pgdg doesn't have an aarch64 repository so wheels are
+        # build with the libpq packaged with Debian 9, which is 9.6.
+        if [ "$AUDITWHEEL_ARCH" != 'aarch64' ]; then
+            echo "deb http://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main" \
+                > /etc/apt/sources.list.d/pgdg.list
+            # TODO: On 2021-11-09 curl fails on 'ppc64le' with:
+            #   curl: (60) SSL certificate problem: certificate has expired
+            # Test again later if -k can be removed.
+            curl -skf https://www.postgresql.org/media/keys/ACCC4CF8.asc \
+                > /etc/apt/trusted.gpg.d/postgresql.asc
+        fi
+
+        apt-get update
+        apt-get -y upgrade
+        apt-get -y install libpq-dev
+        ;;
+
+    centos)
+        "${dir}/build_libpq.sh" > /dev/null
+        ;;
+
+    *)
+        echo "$0: unexpected Linux distribution: '$ID'" >&2
+        exit 1
+        ;;
+esac
+
+# Replace the package name
+if [[ "${PACKAGE_NAME:-}" ]]; then
+    sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
+        "${prjdir}/setup.py"
+fi
+