From cefb8181058342b9a4d0bd13b0fad920365d89d5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 29 May 2021 22:16:28 +0100 Subject: [PATCH] Build manylinux2014 packages for i686/x86_64 platforms The manylinux_2_24 tag leaves too many users without usable packages. Using this tag requires to build libraries too or a libssh too old will be used, with the segfault risks seen in the past. OTOH building all the libraries on arm/ppc platforms proves very time consuming and requires further tooling than what available in the image. Because these packages are new it seems ok to use the manylinux_2_24 tag and use the package versions of libpq/libssl. --- .github/workflows/packages.yml | 22 ++--- NEWS | 7 +- scripts/build/build_libpq.sh | 130 ++++++++++++++++++++++++++ scripts/build/build_macos.sh | 32 ++++--- scripts/build/build_manylinux2014.sh | 75 +++++++++++++++ scripts/build/build_manylinux_2_24.sh | 38 ++++---- scripts/build/build_sdist.sh | 14 +-- 7 files changed, 264 insertions(+), 54 deletions(-) create mode 100755 scripts/build/build_libpq.sh create mode 100755 scripts/build/build_manylinux2014.sh diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 95d11ff3..137e91c2 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -52,15 +52,15 @@ jobs: --health-retries 5 - build-manylinux_2_24: + build-manylinux: strategy: fail-fast: false matrix: include: - - platform: manylinux_2_24_x86_64 - - platform: manylinux_2_24_i686 - - platform: manylinux_2_24_aarch64 - - platform: manylinux_2_24_ppc64le + - {tag: manylinux2014, arch: x86_64} + - {tag: manylinux2014, arch: i686} + - {tag: manylinux_2_24, arch: aarch64} + - {tag: manylinux_2_24, arch: ppc64le} runs-on: ubuntu-20.04 steps: @@ -73,9 +73,9 @@ jobs: - name: Build packages run: >- docker run --rm - -e PLAT=${{ matrix.platform }} + -e PLAT=${{ matrix.tag }}_${{ matrix.arch }} -e PACKAGE_NAME=psycopg2-binary - -e PYVERS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310" + -e PYVERS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39" -e PSYCOPG2_TESTDB=postgres -e PSYCOPG2_TESTDB_HOST=172.17.0.1 -e PSYCOPG2_TESTDB_USER=postgres @@ -83,15 +83,15 @@ jobs: -e PSYCOPG2_TEST_FAST=1 -v `pwd`:/src --workdir /src - quay.io/pypa/${{ matrix.platform }} - ./scripts/build/build_manylinux_2_24.sh + quay.io/pypa/${{ matrix.tag }}_${{ matrix.arch }} + ./scripts/build/build_${{ matrix.tag }}.sh - name: Upload artifacts uses: actions/upload-artifact@v2 with: - name: packages_${{ matrix.platform }} + name: packages_${{ matrix.tag }}_${{ matrix.arch }} path: | - dist/*/*${{ matrix.platform }}.whl + dist/*/*${{ matrix.tag }}_${{ matrix.arch }}.whl services: postgresql: diff --git a/NEWS b/NEWS index ebb1288e..56752e31 100644 --- a/NEWS +++ b/NEWS @@ -24,8 +24,11 @@ Other changes: `~psycopg2.tz.FixedOffsetTimezone`. - The `psycopg2.tz` module is deprecated and scheduled to be dropped in the next major release. -- Build system for Linux/MacOS binary packages moved to GitHub action, now - providing :pep:`600`\-style wheels packages. +- Provide :pep:`599` wheels packages (manylinux2014 tag) for i686 and x86_64 + platforms. +- Provide :pep:`600` wheels packages (manylinux_2_24 tag) for aarch64 and + ppc64le platforms. +- Build system for Linux/MacOS binary packages moved to GitHub action. What's new in psycopg 2.8.7 diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh new file mode 100755 index 00000000..32d2222d --- /dev/null +++ b/scripts/build/build_libpq.sh @@ -0,0 +1,130 @@ +#!/bin/bash + +# Build a modern version of libpq and depending libs from source on Centos 5 + +set -euo pipefail +set -x + +openssl_version="1.1.1k" +ldap_version="2.4.59" +sasl_version="2.1.27" +postgres_version="13.3" + +yum install -y zlib-devel krb5-devel pam-devel + + +# Build openssl if needed +openssl_tag="OpenSSL_${openssl_version//./_}" +openssl_dir="openssl-${openssl_tag}" +if [ ! -d "${openssl_dir}" ]; then curl -sL \ + https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \ + | tar xzf - + + cd "${openssl_dir}" + + ./config --prefix=/usr/local/ --openssldir=/usr/local/ \ + zlib -fPIC shared + make depend + make +else + cd "${openssl_dir}" +fi + +# Install openssl +make install_sw +cd .. + + +# Build libsasl2 if needed +# The system package (cyrus-sasl-devel) causes an amazing error on i686: +# "unsupported version 0 of Verneed record" +# https://github.com/pypa/manylinux/issues/376 +sasl_tag="cyrus-sasl-${sasl_version}" +sasl_dir="cyrus-sasl-${sasl_tag}" +if [ ! -d "${sasl_dir}" ]; then + curl -sL \ + https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \ + | tar xzf - + + cd "${sasl_dir}" + + autoreconf -i + ./configure + make +else + cd "${sasl_dir}" +fi + +# Install libsasl2 +# requires missing nroff to build +touch saslauthd/saslauthd.8 +make install +cd .. + + +# Build openldap if needed +ldap_tag="${ldap_version}" +ldap_dir="openldap-${ldap_tag}" +if [ ! -d "${ldap_dir}" ]; then + curl -sL \ + https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \ + | tar xzf - + + cd "${ldap_dir}" + + ./configure --enable-backends=no --enable-null + make depend + make -C libraries/liblutil/ + make -C libraries/liblber/ + make -C libraries/libldap/ + make -C libraries/libldap_r/ +else + cd "${ldap_dir}" +fi + +# Install openldap +make -C libraries/liblber/ install +make -C libraries/libldap/ install +make -C libraries/libldap_r/ install +make -C include/ install +chmod +x /usr/local/lib/{libldap,liblber}*.so* +cd .. + + +# Build libpq if needed +postgres_tag="REL_${postgres_version//./_}" +postgres_dir="postgres-${postgres_tag}" +if [ ! -d "${postgres_dir}" ]; then + curl -sL \ + https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \ + | tar xzf - + + cd "${postgres_dir}" + + # Match the default unix socket dir default with what defined on Ubuntu and + # Red Hat, which seems the most common location + sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\ +'|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \ + src/include/pg_config_manual.h + + # Without this, libpq ./configure fails on i686 + if [[ "$(uname -m)" == "i686" ]]; then + export LD_LIBRARY_PATH=/usr/local/lib + fi + + ./configure --prefix=/usr/local --without-readline \ + --with-gssapi --with-openssl --with-pam --with-ldap + make -C src/interfaces/libpq + make -C src/bin/pg_config + make -C src/include +else + cd "${postgres_dir}" +fi + +# Install libpq +make -C src/interfaces/libpq install +make -C src/bin/pg_config install +make -C src/include install +cd .. + +find /usr/local/ -name \*.so.\* -type f -exec strip --strip-unneeded {} \; diff --git a/scripts/build/build_macos.sh b/scripts/build/build_macos.sh index 4841ee75..0fe4b56c 100755 --- a/scripts/build/build_macos.sh +++ b/scripts/build/build_macos.sh @@ -8,8 +8,8 @@ set -euo pipefail set -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -PRJDIR="$( cd "${DIR}/../.." && pwd )" +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +prjdir="$( cd "${dir}/../.." && pwd )" brew install gnu-sed postgresql@13 @@ -27,38 +27,40 @@ for i in $(seq 10 -1 0); do done # Find psycopg version -VERSION=$(grep -e ^PSYCOPG_VERSION "${PRJDIR}/setup.py" | gsed "s/.*'\(.*\)'/\1/") +version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | gsed "s/.*'\(.*\)'/\1/") # A gratuitous comment to fix broken vim syntax file: '") -DISTDIR="${PRJDIR}/dist/psycopg2-$VERSION" -mkdir -p "$DISTDIR" +distdir="${prjdir}/dist/psycopg2-$version" +mkdir -p "$distdir" # Install required python packages pip install -U pip wheel delocate # Replace the package name -gsed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ - "${PRJDIR}/setup.py" +if [[ "${PACKAGE_NAME:-}" ]]; then + gsed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ + "${prjdir}/setup.py" +fi # Build the wheels -WHEELDIR="${PRJDIR}/wheels" -pip wheel -w ${WHEELDIR} . -delocate-listdeps ${WHEELDIR}/*.whl +wheeldir="${prjdir}/wheels" +pip wheel -w ${wheeldir} . +delocate-listdeps ${wheeldir}/*.whl # Check where is the libpq. I'm gonna kill it for testing if [[ -z "${LIBPQ:-}" ]]; then - export LIBPQ=$(delocate-listdeps ${WHEELDIR}/*.whl | grep libpq) + export LIBPQ=$(delocate-listdeps ${wheeldir}/*.whl | grep libpq) fi -delocate-wheel ${WHEELDIR}/*.whl +delocate-wheel ${wheeldir}/*.whl # https://github.com/MacPython/wiki/wiki/Spinning-wheels#question-will-pip-give-me-a-broken-wheel -delocate-addplat --rm-orig -x 10_9 -x 10_10 ${WHEELDIR}/*.whl -cp ${WHEELDIR}/*.whl ${DISTDIR} +delocate-addplat --rm-orig -x 10_9 -x 10_10 ${wheeldir}/*.whl +cp ${wheeldir}/*.whl ${distdir} # kill the libpq to make sure tests don't depend on it mv "$LIBPQ" "${LIBPQ}-bye" # Install and test the built wheel -pip install ${PACKAGE_NAME} --no-index -f "$DISTDIR" +pip install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" # Print psycopg and libpq versions python -c "import psycopg2; print(psycopg2.__version__)" diff --git a/scripts/build/build_manylinux2014.sh b/scripts/build/build_manylinux2014.sh new file mode 100755 index 00000000..0e87bd54 --- /dev/null +++ b/scripts/build/build_manylinux2014.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Create manylinux2014 wheels for psycopg2 +# +# manylinux2014 is built on CentOS 7, which packages an old version of the +# libssl, (1.0, which has concurrency problems with the Python libssl). So we +# need to build these libraries from source. +# +# Look at the .github/workflows/packages.yml file for hints about how to use it. + +set -euo pipefail +set -x + +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +prjdir="$( cd "${dir}/../.." && pwd )" + +# Build all the available versions, or just the ones specified in PYVERS +if [ ! "${PYVERS:-}" ]; then + PYVERS="$(ls /opt/python/)" +fi + +# Find psycopg version +version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/") +# A gratuitous comment to fix broken vim syntax file: '") +distdir="${prjdir}/dist/psycopg2-$version" + +# Replace the package name +if [[ "${PACKAGE_NAME:-}" ]]; then + sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ + "${prjdir}/setup.py" +fi + +# Build depending libraries +"${dir}/build_libpq.sh" > /dev/null + +# Create the wheel packages +for pyver in $PYVERS; do + pybin="/opt/python/${pyver}/bin" + "${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/" +done + +# Bundle external shared libraries into the wheels +for whl in "${prjdir}"/dist/*.whl; do + auditwheel repair "$whl" -w "$distdir" +done + +# Make sure the libpq is not in the system +for f in $(find /usr/local/lib -name libpq\*) ; do + mkdir -pv "/libpqbak/$(dirname $f)" + mv -v "$f" "/libpqbak/$(dirname $f)" +done + +# Install packages and test +cd "${prjdir}" +for pyver in $PYVERS; do + pybin="/opt/python/${pyver}/bin" + "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" + + # Print psycopg and libpq versions + "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)" + "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)" + "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())" + + # Fail if we are not using the expected libpq library + if [[ "${WANT_LIBPQ:-}" ]]; then + "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" + fi + + "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" +done + +# Restore the libpq packages +for f in $(cd /libpqbak/ && find . -not -type d); do + mv -v "/libpqbak/$f" "/$f" +done diff --git a/scripts/build/build_manylinux_2_24.sh b/scripts/build/build_manylinux_2_24.sh index 2d247da8..d83c8414 100755 --- a/scripts/build/build_manylinux_2_24.sh +++ b/scripts/build/build_manylinux_2_24.sh @@ -7,8 +7,8 @@ set -euo pipefail set -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -PRJDIR="$( cd "${DIR}/../.." && pwd )" +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +prjdir="$( cd "${dir}/../.." && pwd )" # Build all the available versions, or just the ones specified in PYVERS if [ ! "${PYVERS:-}" ]; then @@ -16,14 +16,14 @@ if [ ! "${PYVERS:-}" ]; then fi # Find psycopg version -VERSION=$(grep -e ^PSYCOPG_VERSION "${PRJDIR}/setup.py" | sed "s/.*'\(.*\)'/\1/") +version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/") # A gratuitous comment to fix broken vim syntax file: '") -DISTDIR="${PRJDIR}/dist/psycopg2-$VERSION" +distdir="${prjdir}/dist/psycopg2-$version" # Replace the package name if [[ "${PACKAGE_NAME:-}" ]]; then sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ - "${PRJDIR}/setup.py" + "${prjdir}/setup.py" fi # Install prerequisite libraries @@ -34,14 +34,14 @@ apt-get -y update apt-get install -y libpq-dev # Create the wheel packages -for PYVER in $PYVERS; do - PYBIN="/opt/python/${PYVER}/bin" - "${PYBIN}/pip" wheel "${PRJDIR}" -w "${PRJDIR}/dist/" +for pyver in $PYVERS; do + pybin="/opt/python/${pyver}/bin" + "${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/" done # Bundle external shared libraries into the wheels -for WHL in "${PRJDIR}"/dist/*.whl; do - auditwheel repair "$WHL" -w "$DISTDIR" +for whl in "${prjdir}"/dist/*.whl; do + auditwheel repair "$whl" -w "$distdir" done # Make sure the libpq is not in the system @@ -51,22 +51,22 @@ for f in $(find /usr/lib /usr/lib64 -name libpq\*) ; do done # Install packages and test -cd "${PRJDIR}" -for PYVER in $PYVERS; do - PYBIN="/opt/python/${PYVER}/bin" - "${PYBIN}/pip" install ${PACKAGE_NAME} --no-index -f "$DISTDIR" +cd "${prjdir}" +for pyver in $PYVERS; do + pybin="/opt/python/${pyver}/bin" + "${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir" # Print psycopg and libpq versions - "${PYBIN}/python" -c "import psycopg2; print(psycopg2.__version__)" - "${PYBIN}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)" - "${PYBIN}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())" + "${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)" + "${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)" + "${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())" # Fail if we are not using the expected libpq library if [[ "${WANT_LIBPQ:-}" ]]; then - "${PYBIN}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" + "${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())" fi - "${PYBIN}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" + "${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" done # Restore the libpq packages diff --git a/scripts/build/build_sdist.sh b/scripts/build/build_sdist.sh index ef7ec518..6408cac9 100755 --- a/scripts/build/build_sdist.sh +++ b/scripts/build/build_sdist.sh @@ -3,24 +3,24 @@ set -euo pipefail set -x -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -PRJDIR="$( cd "${DIR}/../.." && pwd )" +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +prjdir="$( cd "${dir}/../.." && pwd )" # Find psycopg version -export VERSION=$(grep -e ^PSYCOPG_VERSION setup.py | sed "s/.*'\(.*\)'/\1/") +version=$(grep -e ^PSYCOPG_VERSION setup.py | sed "s/.*'\(.*\)'/\1/") # A gratuitous comment to fix broken vim syntax file: '") -export DISTDIR="${PRJDIR}/dist/psycopg2-$VERSION" +distdir="${prjdir}/dist/psycopg2-$version" # Replace the package name if [[ "${PACKAGE_NAME:-}" ]]; then sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \ - setup.py + "${prjdir}/setup.py" fi # Build the source package -python setup.py sdist -d "$DISTDIR" +python setup.py sdist -d "$distdir" # install and test -pip install "${DISTDIR}"/*.tar.gz +pip install "${distdir}"/*.tar.gz python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"