diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index b41c1bea..c69489cd 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -5,6 +5,8 @@ on: env: PIP_BREAK_SYSTEM_PACKAGES: "1" + LIBPQ_VERSION: "16.0" + OPENSSL_VERSION: "1.1.1w" jobs: sdist: # {{{ @@ -59,10 +61,6 @@ jobs: linux: # {{{ if: true - env: - LIBPQ_VERSION: "16.0" - OPENSSL_VERSION: "1.1.1w" - strategy: fail-fast: false matrix: @@ -134,55 +132,46 @@ jobs: # }}} macos: # {{{ - runs-on: macos-${{ matrix.macver }} + runs-on: macos-latest if: true strategy: fail-fast: false matrix: # These archs require an Apple M1 runner: [arm64, universal2] - arch: [x86_64] + arch: [x86_64, arm64] pyver: [cp39, cp310, cp311, cp312, cp313] - macver: ["13"] - include: - - arch: arm64 - pyver: cp310 - macver: "14" - - arch: arm64 - pyver: cp311 - macver: "14" - - arch: arm64 - pyver: cp312 - macver: "14" - - arch: arm64 - pyver: cp313 - macver: "14" steps: - name: Checkout repos uses: actions/checkout@v4 + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-${{ env.LIBPQ_VERSION }}-macos-${{ matrix.arch }} + - name: Build wheels uses: pypa/cibuildwheel@v2.22.0 env: CIBW_BUILD: ${{matrix.pyver}}-macosx_${{matrix.arch}} CIBW_ARCHS_MACOS: ${{matrix.arch}} + MACOSX_ARCHITECTURE: ${{matrix.arch}} CIBW_BEFORE_ALL_MACOS: ./scripts/build/wheel_macos_before_all.sh CIBW_TEST_COMMAND: >- export PYTHONPATH={project} && python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" CIBW_ENVIRONMENT: >- - MACOSX_DEPLOYMENT_TARGET=${{ matrix.macver }}.0 PG_VERSION=16 PACKAGE_NAME=psycopg2-binary PSYCOPG2_TESTDB=postgres - PSYCOPG2_TEST_FAST=1 - PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" + PATH="/tmp/libpq.build/bin:$PATH" - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: macos-${{matrix.pyver}}-macos-${{matrix.macver}}_${{matrix.arch}} + name: macos-${{matrix.pyver}}-macos-${{matrix.arch}} path: ./wheelhouse/*.whl # }}} diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 8a99fbe8..107460f8 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Build a modern version of libpq and depending libs from source on Centos 5 +# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS set -euo pipefail set -x @@ -12,21 +12,39 @@ postgres_version="${LIBPQ_VERSION}" # last release: https://www.openssl.org/source/ openssl_version="${OPENSSL_VERSION}" +# last release: https://kerberos.org/dist/ +krb5_version="1.21.3" + # last release: https://openldap.org/software/download/ -ldap_version="2.6.3" +ldap_version="2.6.8" # last release: https://github.com/cyrusimap/cyrus-sasl/releases sasl_version="2.1.28" export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build} -if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then +case "$(uname)" in + Darwin) + ID=macos + library_suffix=dylib + ;; + + Linux) + source /etc/os-release + library_suffix=so + ;; + + *) + echo "$0: unexpected Operating system: '$(uname)'" >&2 + exit 1 + ;; +esac + +if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.${library_suffix}" ]]; then echo "libpq already available: build skipped" >&2 exit 0 fi -source /etc/os-release - case "$ID" in centos) yum update -y @@ -38,39 +56,101 @@ case "$ID" in apk add --no-cache zlib-dev krb5-dev linux-pam-dev openldap-dev openssl-dev ;; + macos) + brew install automake m4 libtool + # If available, libpq seemingly insists on linking against homebrew's + # openssl no matter what so remove it. Since homebrew's curl depends on + # it, force use of system curl. + brew uninstall --force --ignore-dependencies openssl gettext curl + if [ -z "${MACOSX_ARCHITECTURE:-}" ]; then + MACOSX_ARCHITECTURE="$(uname -m)" + fi + # Set the deployment target to be <= to that of the oldest supported Python version. + # e.g. https://www.python.org/downloads/release/python-380/ + if [ "$MACOSX_ARCHITECTURE" == "x86_64" ]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 + else + export MACOSX_DEPLOYMENT_TARGET=11.0 + fi + ;; + *) echo "$0: unexpected Linux distribution: '$ID'" >&2 exit 1 ;; esac -if [ "$ID" == "centos" ]; then + +if [ "$ID" == "macos" ]; then + make_configure_standard_flags=( \ + --prefix=${LIBPQ_BUILD_PREFIX} \ + "CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ -arch $MACOSX_ARCHITECTURE" \ + "LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib -arch $MACOSX_ARCHITECTURE" \ + ) +else + make_configure_standard_flags=( \ + --prefix=${LIBPQ_BUILD_PREFIX} \ + CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ \ + LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib \ + ) +fi + + +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openssl if needed openssl_tag="OpenSSL_${openssl_version//./_}" openssl_dir="openssl-${openssl_tag}" - if [ ! -d "${openssl_dir}" ]; then curl -sL \ + if [ ! -d "${openssl_dir}" ]; then + curl -sL \ https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \ | tar xzf - - cd "${openssl_dir}" + pushd "${openssl_dir}" + + options=(--prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \ + zlib -fPIC shared) + if [ -z "${MACOSX_ARCHITECTURE:-}" ]; then + ./config $options + else + ./configure "darwin64-$MACOSX_ARCHITECTURE-cc" $options + fi - ./config --prefix=${LIBPQ_BUILD_PREFIX} --openssldir=${LIBPQ_BUILD_PREFIX} \ - zlib -fPIC shared make depend make else - cd "${openssl_dir}" + pushd "${openssl_dir}" fi # Install openssl make install_sw - cd .. + popd fi -if [ "$ID" == "centos" ]; then +if [ "$ID" == "macos" ]; then + + # Build kerberos if needed + krb5_dir="krb5-${krb5_version}/src" + if [ ! -d "${krb5_dir}" ]; then + curl -sL "https://kerberos.org/dist/krb5/${krb5_version%.*}/krb5-${krb5_version}.tar.gz" \ + | tar xzf - + + pushd "${krb5_dir}" + ./configure "${make_configure_standard_flags[@]}" + make + else + pushd "${krb5_dir}" + fi + + make install + popd + +fi + + +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build libsasl2 if needed # The system package (cyrus-sasl-devel) causes an amazing error on i686: @@ -83,26 +163,25 @@ if [ "$ID" == "centos" ]; then https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \ | tar xzf - - cd "${sasl_dir}" + pushd "${sasl_dir}" autoreconf -i - ./configure --prefix=${LIBPQ_BUILD_PREFIX} \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + ./configure "${make_configure_standard_flags[@]}" --disable-macos-framework make else - cd "${sasl_dir}" + pushd "${sasl_dir}" fi # Install libsasl2 # requires missing nroff to build touch saslauthd/saslauthd.8 make install - cd .. + popd fi -if [ "$ID" == "centos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openldap if needed ldap_tag="${ldap_version}" @@ -112,25 +191,24 @@ if [ "$ID" == "centos" ]; then https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \ | tar xzf - - cd "${ldap_dir}" + pushd "${ldap_dir}" - ./configure --prefix=${LIBPQ_BUILD_PREFIX} --enable-backends=no --enable-null \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + ./configure "${make_configure_standard_flags[@]}" --enable-backends=no --enable-null make depend make -C libraries/liblutil/ make -C libraries/liblber/ make -C libraries/libldap/ else - cd "${ldap_dir}" + pushd "${ldap_dir}" fi # Install openldap make -C libraries/liblber/ install make -C libraries/libldap/ install make -C include/ install - chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.so* - cd .. + chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.${library_suffix}* + popd fi @@ -143,32 +221,33 @@ if [ ! -d "${postgres_dir}" ]; then https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \ | tar xzf - - cd "${postgres_dir}" + pushd "${postgres_dir}" - # Match the default unix socket dir default with what defined on Ubuntu and - # Red Hat, which seems the most common location - sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\ + if [ "$ID" != "macos" ]; then + # Match the default unix socket dir default with what defined on Ubuntu and + # Red Hat, which seems the most common location + sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\ '|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \ - src/include/pg_config_manual.h + src/include/pg_config_manual.h + fi # Often needed, but currently set by the workflow # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib" - ./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \ + ./configure "${make_configure_standard_flags[@]}" --sysconfdir=/etc/postgresql-common \ --with-gssapi --with-openssl --with-pam --with-ldap \ - --without-readline --without-icu \ - CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + --without-readline --without-icu make -C src/interfaces/libpq make -C src/bin/pg_config make -C src/include else - cd "${postgres_dir}" + pushd "${postgres_dir}" fi # Install libpq make -C src/interfaces/libpq install make -C src/bin/pg_config install make -C src/include install -cd .. +popd -find ${LIBPQ_BUILD_PREFIX} -name \*.so.\* -type f -exec strip --strip-unneeded {} \; +find ${LIBPQ_BUILD_PREFIX} -name \*.${library_suffix}.\* -type f -exec strip --strip-unneeded {} \; diff --git a/scripts/build/wheel_macos_before_all.sh b/scripts/build/wheel_macos_before_all.sh index 882b887f..36e1c0b8 100755 --- a/scripts/build/wheel_macos_before_all.sh +++ b/scripts/build/wheel_macos_before_all.sh @@ -11,6 +11,12 @@ set -x dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" prjdir="$( cd "${dir}/../.." && pwd )" +# Build dependency libraries +"${prjdir}/scripts/build/build_libpq.sh" + +# Show dependency tree +otool -L /tmp/libpq.build/lib/*.dylib + brew install gnu-sed postgresql@${PG_VERSION} brew link --overwrite postgresql@${PG_VERSION} diff --git a/tests/test_async.py b/tests/test_async.py index dfc7b598..975e7f4d 100755 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -504,7 +504,7 @@ class AsyncTests(ConnectingTestCase): raise Exception("Unexpected result from poll: %r", state) polls += 1 - self.assert_(polls >= 8, polls) + self.assert_(polls >= 5, polls) def test_poll_noop(self): self.conn.poll() diff --git a/tests/test_green.py b/tests/test_green.py index e558d626..5e2ae513 100755 --- a/tests/test_green.py +++ b/tests/test_green.py @@ -152,7 +152,7 @@ class GreenTestCase(ConnectingTestCase): """) polls = stub.polls.count(POLL_READ) - self.assert_(polls > 8, polls) + self.assert_(polls > 6, polls) class CallbackErrorTestCase(ConnectingTestCase):