Build manylinux2014 packages for i686/x86_64 platforms

The manylinux_2_24 tag leaves too many users without usable packages.
Using this tag requires to build libraries too or a libssh too old will
be used, with the segfault risks seen in the past.

OTOH building all the libraries on arm/ppc platforms proves very time
consuming and requires further tooling than what available in the image.
Because these packages are new it seems ok to use the manylinux_2_24
tag and use the package versions of libpq/libssl.

.github/workflows/packages.yml

@@ -52,15 +52,15 @@ jobs:
           --health-retries 5
 
 
-  build-manylinux_2_24:
+  build-manylinux:
     strategy:
       fail-fast: false
       matrix:
         include:
-          - platform: manylinux_2_24_x86_64
+          - {tag: manylinux2014, arch: x86_64}
-          - platform: manylinux_2_24_i686
+          - {tag: manylinux2014, arch: i686}
-          - platform: manylinux_2_24_aarch64
+          - {tag: manylinux_2_24, arch: aarch64}
-          - platform: manylinux_2_24_ppc64le
+          - {tag: manylinux_2_24, arch: ppc64le}
 
     runs-on: ubuntu-20.04
     steps:
@@ -73,9 +73,9 @@ jobs:
       - name: Build packages
         run: >-
           docker run --rm
-          -e PLAT=${{ matrix.platform }}
+          -e PLAT=${{ matrix.tag }}_${{ matrix.arch }}
           -e PACKAGE_NAME=psycopg2-binary
-          -e PYVERS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310"
+          -e PYVERS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39"
           -e PSYCOPG2_TESTDB=postgres
           -e PSYCOPG2_TESTDB_HOST=172.17.0.1
           -e PSYCOPG2_TESTDB_USER=postgres
@@ -83,15 +83,15 @@ jobs:
           -e PSYCOPG2_TEST_FAST=1
           -v `pwd`:/src
           --workdir /src
-          quay.io/pypa/${{ matrix.platform }}
+          quay.io/pypa/${{ matrix.tag }}_${{ matrix.arch }}
-          ./scripts/build/build_manylinux_2_24.sh
+          ./scripts/build/build_${{ matrix.tag }}.sh
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: packages_${{ matrix.platform }}
+          name: packages_${{ matrix.tag }}_${{ matrix.arch }}
           path: |
-            dist/*/*${{ matrix.platform }}.whl
+            dist/*/*${{ matrix.tag }}_${{ matrix.arch }}.whl
 
     services:
       postgresql:

NEWS

@@ -24,8 +24,11 @@ Other changes:
   `~psycopg2.tz.FixedOffsetTimezone`.
 - The `psycopg2.tz` module is deprecated and scheduled to be dropped in the
   next major release.
-- Build system for Linux/MacOS binary packages moved to GitHub action, now
+- Provide :pep:`599` wheels packages (manylinux2014 tag) for i686 and x86_64
-  providing :pep:`600`\-style wheels packages.
+  platforms.
+- Provide :pep:`600` wheels packages (manylinux_2_24 tag) for aarch64 and
+  ppc64le platforms.
+- Build system for Linux/MacOS binary packages moved to GitHub action.
 
 
 What's new in psycopg 2.8.7

scripts/build/build_libpq.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+# Build a modern version of libpq and depending libs from source on Centos 5
+
+set -euo pipefail
+set -x
+
+openssl_version="1.1.1k"
+ldap_version="2.4.59"
+sasl_version="2.1.27"
+postgres_version="13.3"
+
+yum install -y zlib-devel krb5-devel pam-devel
+
+
+# Build openssl if needed
+openssl_tag="OpenSSL_${openssl_version//./_}"
+openssl_dir="openssl-${openssl_tag}"
+if [ ! -d "${openssl_dir}" ]; then curl -sL \
+        https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \
+        | tar xzf -
+
+    cd "${openssl_dir}"
+
+    ./config --prefix=/usr/local/ --openssldir=/usr/local/ \
+        zlib -fPIC shared
+    make depend
+    make
+else
+    cd "${openssl_dir}"
+fi
+
+# Install openssl
+make install_sw
+cd ..
+
+
+# Build libsasl2 if needed
+# The system package (cyrus-sasl-devel) causes an amazing error on i686:
+# "unsupported version 0 of Verneed record"
+# https://github.com/pypa/manylinux/issues/376
+sasl_tag="cyrus-sasl-${sasl_version}"
+sasl_dir="cyrus-sasl-${sasl_tag}"
+if [ ! -d "${sasl_dir}" ]; then
+    curl -sL \
+        https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \
+        | tar xzf -
+
+    cd "${sasl_dir}"
+
+    autoreconf -i
+    ./configure
+    make
+else
+    cd "${sasl_dir}"
+fi
+
+# Install libsasl2
+# requires missing nroff to build
+touch saslauthd/saslauthd.8
+make install
+cd ..
+
+
+# Build openldap if needed
+ldap_tag="${ldap_version}"
+ldap_dir="openldap-${ldap_tag}"
+if [ ! -d "${ldap_dir}" ]; then
+    curl -sL \
+        https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \
+        | tar xzf -
+
+    cd "${ldap_dir}"
+
+    ./configure --enable-backends=no --enable-null
+    make depend
+    make -C libraries/liblutil/
+    make -C libraries/liblber/
+    make -C libraries/libldap/
+    make -C libraries/libldap_r/
+else
+    cd "${ldap_dir}"
+fi
+
+# Install openldap
+make -C libraries/liblber/ install
+make -C libraries/libldap/ install
+make -C libraries/libldap_r/ install
+make -C include/ install
+chmod +x /usr/local/lib/{libldap,liblber}*.so*
+cd ..
+
+
+# Build libpq if needed
+postgres_tag="REL_${postgres_version//./_}"
+postgres_dir="postgres-${postgres_tag}"
+if [ ! -d "${postgres_dir}" ]; then
+    curl -sL \
+        https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \
+        | tar xzf -
+
+    cd "${postgres_dir}"
+
+    # Match the default unix socket dir default with what defined on Ubuntu and
+    # Red Hat, which seems the most common location
+    sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\
+'|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \
+        src/include/pg_config_manual.h
+
+    # Without this, libpq ./configure fails on i686
+    if [[ "$(uname -m)" == "i686" ]]; then
+        export LD_LIBRARY_PATH=/usr/local/lib
+    fi
+
+    ./configure --prefix=/usr/local --without-readline \
+        --with-gssapi --with-openssl --with-pam --with-ldap
+    make -C src/interfaces/libpq
+    make -C src/bin/pg_config
+    make -C src/include
+else
+    cd "${postgres_dir}"
+fi
+
+# Install libpq
+make -C src/interfaces/libpq install
+make -C src/bin/pg_config install
+make -C src/include install
+cd ..
+
+find /usr/local/ -name \*.so.\* -type f -exec strip --strip-unneeded {} \;

scripts/build/build_macos.sh

@@ -8,8 +8,8 @@
 set -euo pipefail
 set -x
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-PRJDIR="$( cd "${DIR}/../.." && pwd )"
+prjdir="$( cd "${dir}/../.." && pwd )"
 
 brew install gnu-sed postgresql@13
 
@@ -27,38 +27,40 @@ for i in $(seq 10 -1 0); do
 done
 
 # Find psycopg version
-VERSION=$(grep -e ^PSYCOPG_VERSION "${PRJDIR}/setup.py" | gsed "s/.*'\(.*\)'/\1/")
+version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | gsed "s/.*'\(.*\)'/\1/")
 # A gratuitous comment to fix broken vim syntax file: '")
-DISTDIR="${PRJDIR}/dist/psycopg2-$VERSION"
+distdir="${prjdir}/dist/psycopg2-$version"
-mkdir -p "$DISTDIR"
+mkdir -p "$distdir"
 
 # Install required python packages
 pip install -U pip wheel delocate
 
 # Replace the package name
-gsed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
+if [[ "${PACKAGE_NAME:-}" ]]; then
-    "${PRJDIR}/setup.py"
+    gsed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
+        "${prjdir}/setup.py"
+fi
 
 # Build the wheels
-WHEELDIR="${PRJDIR}/wheels"
+wheeldir="${prjdir}/wheels"
-pip wheel -w ${WHEELDIR} .
+pip wheel -w ${wheeldir} .
-delocate-listdeps ${WHEELDIR}/*.whl
+delocate-listdeps ${wheeldir}/*.whl
 
 # Check where is the libpq. I'm gonna kill it for testing
 if [[ -z "${LIBPQ:-}" ]]; then
-    export LIBPQ=$(delocate-listdeps ${WHEELDIR}/*.whl | grep libpq)
+    export LIBPQ=$(delocate-listdeps ${wheeldir}/*.whl | grep libpq)
 fi
 
-delocate-wheel ${WHEELDIR}/*.whl
+delocate-wheel ${wheeldir}/*.whl
 # https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel
-delocate-addplat --rm-orig -x 10_9 -x 10_10 ${WHEELDIR}/*.whl
+delocate-addplat --rm-orig -x 10_9 -x 10_10 ${wheeldir}/*.whl
-cp ${WHEELDIR}/*.whl ${DISTDIR}
+cp ${wheeldir}/*.whl ${distdir}
 
 # kill the libpq to make sure tests don't depend on it
 mv "$LIBPQ" "${LIBPQ}-bye"
 
 # Install and test the built wheel
-pip install ${PACKAGE_NAME} --no-index -f "$DISTDIR"
+pip install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
 
 # Print psycopg and libpq versions
 python -c "import psycopg2; print(psycopg2.__version__)"

scripts/build/build_manylinux2014.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Create manylinux2014 wheels for psycopg2
+#
+# manylinux2014 is built on CentOS 7, which packages an old version of the
+# libssl, (1.0, which has concurrency problems with the Python libssl). So we
+# need to build these libraries from source.
+#
+# Look at the .github/workflows/packages.yml file for hints about how to use it.
+
+set -euo pipefail
+set -x
+
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+prjdir="$( cd "${dir}/../.." && pwd )"
+
+# Build all the available versions, or just the ones specified in PYVERS
+if [ ! "${PYVERS:-}" ]; then
+    PYVERS="$(ls /opt/python/)"
+fi
+
+# Find psycopg version
+version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/")
+# A gratuitous comment to fix broken vim syntax file: '")
+distdir="${prjdir}/dist/psycopg2-$version"
+
+# Replace the package name
+if [[ "${PACKAGE_NAME:-}" ]]; then
+    sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
+        "${prjdir}/setup.py"
+fi
+
+# Build depending libraries
+"${dir}/build_libpq.sh" > /dev/null
+
+# Create the wheel packages
+for pyver in $PYVERS; do
+    pybin="/opt/python/${pyver}/bin"
+    "${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/"
+done
+
+# Bundle external shared libraries into the wheels
+for whl in "${prjdir}"/dist/*.whl; do
+    auditwheel repair "$whl" -w "$distdir"
+done
+
+# Make sure the libpq is not in the system
+for f in $(find /usr/local/lib -name libpq\*) ; do
+    mkdir -pv "/libpqbak/$(dirname $f)"
+    mv -v "$f" "/libpqbak/$(dirname $f)"
+done
+
+# Install packages and test
+cd "${prjdir}"
+for pyver in $PYVERS; do
+    pybin="/opt/python/${pyver}/bin"
+    "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
+
+    # Print psycopg and libpq versions
+    "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)"
+    "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
+    "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
+
+    # Fail if we are not using the expected libpq library
+    if [[ "${WANT_LIBPQ:-}" ]]; then
+        "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
+    fi
+
+    "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
+done
+
+# Restore the libpq packages
+for f in $(cd /libpqbak/ && find . -not -type d); do
+    mv -v "/libpqbak/$f" "/$f"
+done

scripts/build/build_manylinux_2_24.sh

@@ -7,8 +7,8 @@
 set -euo pipefail
 set -x
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-PRJDIR="$( cd "${DIR}/../.." && pwd )"
+prjdir="$( cd "${dir}/../.." && pwd )"
 
 # Build all the available versions, or just the ones specified in PYVERS
 if [ ! "${PYVERS:-}" ]; then
@@ -16,14 +16,14 @@ if [ ! "${PYVERS:-}" ]; then
 fi
 
 # Find psycopg version
-VERSION=$(grep -e ^PSYCOPG_VERSION "${PRJDIR}/setup.py" | sed "s/.*'\(.*\)'/\1/")
+version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/")
 # A gratuitous comment to fix broken vim syntax file: '")
-DISTDIR="${PRJDIR}/dist/psycopg2-$VERSION"
+distdir="${prjdir}/dist/psycopg2-$version"
 
 # Replace the package name
 if [[ "${PACKAGE_NAME:-}" ]]; then
     sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
-        "${PRJDIR}/setup.py"
+        "${prjdir}/setup.py"
 fi
 
 # Install prerequisite libraries
@@ -34,14 +34,14 @@ apt-get -y update
 apt-get install -y libpq-dev
 
 # Create the wheel packages
-for PYVER in $PYVERS; do
+for pyver in $PYVERS; do
-    PYBIN="/opt/python/${PYVER}/bin"
+    pybin="/opt/python/${pyver}/bin"
-    "${PYBIN}/pip" wheel "${PRJDIR}" -w "${PRJDIR}/dist/"
+    "${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/"
 done
 
 # Bundle external shared libraries into the wheels
-for WHL in "${PRJDIR}"/dist/*.whl; do
+for whl in "${prjdir}"/dist/*.whl; do
-    auditwheel repair "$WHL" -w "$DISTDIR"
+    auditwheel repair "$whl" -w "$distdir"
 done
 
 # Make sure the libpq is not in the system
@@ -51,22 +51,22 @@ for f in $(find /usr/lib /usr/lib64 -name libpq\*) ; do
 done
 
 # Install packages and test
-cd "${PRJDIR}"
+cd "${prjdir}"
-for PYVER in $PYVERS; do
+for pyver in $PYVERS; do
-    PYBIN="/opt/python/${PYVER}/bin"
+    pybin="/opt/python/${pyver}/bin"
-    "${PYBIN}/pip" install ${PACKAGE_NAME} --no-index -f "$DISTDIR"
+    "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
 
     # Print psycopg and libpq versions
-    "${PYBIN}/python" -c "import psycopg2; print(psycopg2.__version__)"
+    "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)"
-    "${PYBIN}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
+    "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
-    "${PYBIN}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
+    "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
 
     # Fail if we are not using the expected libpq library
     if [[ "${WANT_LIBPQ:-}" ]]; then
-        "${PYBIN}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
+        "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
     fi
 
-    "${PYBIN}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
+    "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
 done
 
 # Restore the libpq packages

scripts/build/build_sdist.sh

@@ -3,24 +3,24 @@
 set -euo pipefail
 set -x
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-PRJDIR="$( cd "${DIR}/../.." && pwd )"
+prjdir="$( cd "${dir}/../.." && pwd )"
 
 # Find psycopg version
-export VERSION=$(grep -e ^PSYCOPG_VERSION setup.py | sed "s/.*'\(.*\)'/\1/")
+version=$(grep -e ^PSYCOPG_VERSION setup.py | sed "s/.*'\(.*\)'/\1/")
 # A gratuitous comment to fix broken vim syntax file: '")
-export DISTDIR="${PRJDIR}/dist/psycopg2-$VERSION"
+distdir="${prjdir}/dist/psycopg2-$version"
 
 # Replace the package name
 if [[ "${PACKAGE_NAME:-}" ]]; then
     sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
-        setup.py
+        "${prjdir}/setup.py"
 fi
 
 # Build the source package
-python setup.py sdist -d "$DISTDIR"
+python setup.py sdist -d "$distdir"
 
 # install and test
-pip install "${DISTDIR}"/*.tar.gz
+pip install "${distdir}"/*.tar.gz
 
 python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"