Merge pull request #1545 from AmirBitaraf/aarch64_manylinux2014_libpq

Move to manylinux2014 for aarch64, ppc64le builds.

.github/workflows/packages.yml

@@ -61,8 +61,8 @@ jobs:
         include:
           - {tag: manylinux2014, arch: x86_64}
           - {tag: manylinux2014, arch: i686}
-          - {tag: manylinux_2_24, arch: aarch64}
+          - {tag: manylinux2014, arch: aarch64}
-          - {tag: manylinux_2_24, arch: ppc64le}
+          - {tag: manylinux2014, arch: ppc64le}
           - {tag: musllinux_1_1, arch: x86_64}
           - {tag: musllinux_1_1, arch: i686}
           - {tag: musllinux_1_1, arch: aarch64}

scripts/build/build_libpq.sh

@@ -6,93 +6,133 @@ set -euo pipefail
 set -x
 
 # Last release: https://www.postgresql.org/ftp/source/
-postgres_version="15.0"
+# IMPORTANT! Change the cache key in packages.yml when upgrading libraries
+postgres_version="${LIBPQ_VERSION:-15.0}"
+
 # last release: https://www.openssl.org/source/
-openssl_version="1.1.1r"
+openssl_version="${OPENSSL_VERSION:-1.1.1r}"
+
 # last release: https://openldap.org/software/download/
-ldap_version="2.4.59"
+ldap_version="2.6.3"
+
 # last release: https://github.com/cyrusimap/cyrus-sasl/releases
 sasl_version="2.1.28"
 
-yum install -y zlib-devel krb5-devel pam-devel
+export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/usr/local}
 
-
+if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then
-# Build openssl if needed
+    echo "libpq already available: build skipped" >&2
-openssl_tag="OpenSSL_${openssl_version//./_}"
+    exit 0
-openssl_dir="openssl-${openssl_tag}"
-if [ ! -d "${openssl_dir}" ]; then curl -sL \
-        https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \
-        | tar xzf -
-
-    cd "${openssl_dir}"
-
-    ./config --prefix=/usr/local/ --openssldir=/usr/local/ \
-        zlib -fPIC shared
-    make depend
-    make
-else
-    cd "${openssl_dir}"
 fi
 
-# Install openssl
+source /etc/os-release
-make install_sw
-cd ..
 
+case "$ID" in
+    centos)
+        yum update -y
+        yum install -y zlib-devel krb5-devel pam-devel
+        ;;
 
-# Build libsasl2 if needed
+    alpine)
-# The system package (cyrus-sasl-devel) causes an amazing error on i686:
+        apk upgrade
-# "unsupported version 0 of Verneed record"
+        apk add --no-cache zlib-dev krb5-dev linux-pam-dev openldap-dev
-# https://github.com/pypa/manylinux/issues/376
+        ;;
-sasl_tag="cyrus-sasl-${sasl_version}"
-sasl_dir="cyrus-sasl-${sasl_tag}"
-if [ ! -d "${sasl_dir}" ]; then
-    curl -sL \
-        https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \
-        | tar xzf -
 
-    cd "${sasl_dir}"
+    *)
+        echo "$0: unexpected Linux distribution: '$ID'" >&2
+        exit 1
+        ;;
+esac
+
+if [ "$ID" == "centos" ]; then
+
+    # Build openssl if needed
+    openssl_tag="OpenSSL_${openssl_version//./_}"
+    openssl_dir="openssl-${openssl_tag}"
+    if [ ! -d "${openssl_dir}" ]; then curl -sL \
+            https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \
+            | tar xzf -
+
+        cd "${openssl_dir}"
+
+        ./config --prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \
+            zlib -fPIC shared
+        make depend
+        make
+    else
+        cd "${openssl_dir}"
+    fi
+
+    # Install openssl
+    make install_sw
+    cd ..
 
-    autoreconf -i
-    ./configure
-    make
-else
-    cd "${sasl_dir}"
 fi
 
-# Install libsasl2
-# requires missing nroff to build
-touch saslauthd/saslauthd.8
-make install
-cd ..
 
+if [ "$ID" == "centos" ]; then
 
-# Build openldap if needed
+    # Build libsasl2 if needed
-ldap_tag="${ldap_version}"
+    # The system package (cyrus-sasl-devel) causes an amazing error on i686:
-ldap_dir="openldap-${ldap_tag}"
+    # "unsupported version 0 of Verneed record"
-if [ ! -d "${ldap_dir}" ]; then
+    # https://github.com/pypa/manylinux/issues/376
-    curl -sL \
+    sasl_tag="cyrus-sasl-${sasl_version}"
-        https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \
+    sasl_dir="cyrus-sasl-${sasl_tag}"
-        | tar xzf -
+    if [ ! -d "${sasl_dir}" ]; then
+        curl -sL \
+            https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \
+            | tar xzf -
 
-    cd "${ldap_dir}"
+        cd "${sasl_dir}"
+
+        autoreconf -i
+        ./configure --prefix=${LIBPQ_BUILD_PREFIX} \
+            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+        make
+    else
+        cd "${sasl_dir}"
+    fi
+
+    # Install libsasl2
+    # requires missing nroff to build
+    touch saslauthd/saslauthd.8
+    make install
+    cd ..
 
-    ./configure --enable-backends=no --enable-null
-    make depend
-    make -C libraries/liblutil/
-    make -C libraries/liblber/
-    make -C libraries/libldap/
-    make -C libraries/libldap_r/
-else
-    cd "${ldap_dir}"
 fi
 
-# Install openldap
+
-make -C libraries/liblber/ install
+if [ "$ID" == "centos" ]; then
-make -C libraries/libldap/ install
+
-make -C libraries/libldap_r/ install
+    # Build openldap if needed
-make -C include/ install
+    ldap_tag="${ldap_version}"
-chmod +x /usr/local/lib/{libldap,liblber}*.so*
+    ldap_dir="openldap-${ldap_tag}"
-cd ..
+    if [ ! -d "${ldap_dir}" ]; then
+        curl -sL \
+            https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \
+            | tar xzf -
+
+        cd "${ldap_dir}"
+
+        ./configure --prefix=${LIBPQ_BUILD_PREFIX} --enable-backends=no --enable-null \
+            CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
+
+        make depend
+        make -C libraries/liblutil/
+        make -C libraries/liblber/
+        make -C libraries/libldap/
+    else
+        cd "${ldap_dir}"
+    fi
+
+    # Install openldap
+    make -C libraries/liblber/ install
+    make -C libraries/libldap/ install
+    make -C include/ install
+    chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.so*
+    cd ..
+
+fi
 
 
 # Build libpq if needed
@@ -111,14 +151,12 @@ if [ ! -d "${postgres_dir}" ]; then
 '|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \
         src/include/pg_config_manual.h
 
-    # Without this, libpq ./configure fails on i686
+    # Often needed, but currently set by the workflow
-    if [[ "$(uname -m)" == "i686" ]]; then
+    export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib;${LIBPQ_BUILD_PREFIX}/lib64"
-        export LD_LIBRARY_PATH=/usr/local/lib
-    fi
 
-    ./configure --prefix=/usr/local --without-readline \
+    ./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \
-        --with-gssapi --with-openssl --with-pam --with-ldap \
+        --without-readline --with-gssapi --with-openssl --with-pam --with-ldap \
-        --sysconfdir=/etc/postgresql-common
+        CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib
     make -C src/interfaces/libpq
     make -C src/bin/pg_config
     make -C src/include
@@ -132,4 +170,4 @@ make -C src/bin/pg_config install
 make -C src/include install
 cd ..
 
-find /usr/local/ -name \*.so.\* -type f -exec strip --strip-unneeded {} \;
+find ${LIBPQ_BUILD_PREFIX} -name \*.so.\* -type f -exec strip --strip-unneeded {} \;

scripts/build/build_manylinux_2_24.sh

@@ -1,76 +0,0 @@
-#!/bin/bash
-
-# Create manylinux_2_24 wheels for psycopg2
-#
-# Look at the .github/workflows/packages.yml file for hints about how to use it.
-
-set -euo pipefail
-set -x
-
-dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-prjdir="$( cd "${dir}/../.." && pwd )"
-
-# Build all the available versions, or just the ones specified in PYVERS
-if [ ! "${PYVERS:-}" ]; then
-    PYVERS="$(ls /opt/python/)"
-fi
-
-# Find psycopg version
-version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/")
-# A gratuitous comment to fix broken vim syntax file: '")
-distdir="${prjdir}/dist/psycopg2-$version"
-
-# Replace the package name
-if [[ "${PACKAGE_NAME:-}" ]]; then
-    sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
-        "${prjdir}/setup.py"
-fi
-
-# Install prerequisite libraries
-curl -k -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-echo "deb http://apt.postgresql.org/pub/repos/apt stretch-pgdg main" \
-    > /etc/apt/sources.list.d/pgdg.list
-apt-get -y update
-apt-get install -y libpq-dev
-
-# Create the wheel packages
-for pyver in $PYVERS; do
-    pybin="/opt/python/${pyver}/bin"
-    "${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/"
-done
-
-# Bundle external shared libraries into the wheels
-for whl in "${prjdir}"/dist/*.whl; do
-    "${dir}/strip_wheel.sh" "$whl"
-    auditwheel repair "$whl" -w "$distdir"
-done
-
-# Make sure the libpq is not in the system
-for f in $(find /usr/lib /usr/lib64 -name libpq\*) ; do
-    mkdir -pv "/libpqbak/$(dirname $f)"
-    mv -v "$f" "/libpqbak/$(dirname $f)"
-done
-
-# Install packages and test
-cd "${prjdir}"
-for pyver in $PYVERS; do
-    pybin="/opt/python/${pyver}/bin"
-    "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
-
-    # Print psycopg and libpq versions
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)"
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
-    "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
-
-    # Fail if we are not using the expected libpq library
-    if [[ "${WANT_LIBPQ:-}" ]]; then
-        "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
-    fi
-
-    "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
-done
-
-# Restore the libpq packages
-for f in $(cd /libpqbak/ && find . -not -type d); do
-    mv -v "/libpqbak/$f" "/$f"
-done