psycopg/psycopg2
1
1
Fork 0
mirror of https://github.com/psycopg/psycopg2.git synced 2024-11-25 02:13:44 +03:00
Code Issues Packages Projects Releases Wiki Activity

ci: use cibuildwheel to build linux wheel packages

Browse Source
This commit is contained in:
Daniele Varrazzo 2023-04-02 17:33:56 +02:00
parent 7a8f4d6222
commit 333b3b7ac4
7 changed files with 153 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
.github/workflows/packages.yml vendored
View File

@ -52,20 +52,14 @@ jobs:
--health-retries 5
build-manylinux:
build-linux:
if: true
strategy:
fail-fast: false
matrix:
include:
- {tag: manylinux2014, arch: x86_64}
- {tag: manylinux2014, arch: i686}
- {tag: manylinux2014, arch: aarch64}
- {tag: manylinux2014, arch: ppc64le}
- {tag: musllinux_1_1, arch: x86_64}
- {tag: musllinux_1_1, arch: i686}
- {tag: musllinux_1_1, arch: aarch64}
- {tag: musllinux_1_1, arch: ppc64le}
platform: [manylinux, musllinux]
arch: [x86_64, i686, aarch64, ppc64le]
pyver: [cp37, cp38, cp39, cp310, cp311]
runs-on: ubuntu-latest
steps:
@ -75,31 +69,46 @@ jobs:
- name: Set up QEMU for multi-arch build
uses: docker/setup-qemu-action@v2
- name: Build packages
run: >-
docker run --rm
-e PLAT=${{ matrix.tag }}_${{ matrix.arch }}
-e PACKAGE_NAME=psycopg2-binary
-e PYVERS="cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311"
-e PSYCOPG2_TESTDB=postgres
-e PSYCOPG2_TESTDB_HOST=172.17.0.1
-e PSYCOPG2_TESTDB_USER=postgres
-e PSYCOPG2_TESTDB_PASSWORD=password
-e PSYCOPG2_TEST_FAST=1
-v `pwd`:/src
--workdir /src
quay.io/pypa/${{ matrix.tag }}_${{ matrix.arch }}
./scripts/build/build_${{ matrix.tag }}.sh
- name: Upload artifacts
uses: actions/upload-artifact@v3
- name: Cache libpq build
uses: actions/cache@v3
with:
path: |
dist/*.whl
path: /tmp/libpq.build
key: libpq-${{ env.LIBPQ_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}
- name: Build wheels
uses: pypa/cibuildwheel@v2.12.0
env:
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
CIBW_MANYLINUX_I686_IMAGE: manylinux2014
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014
CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014
CIBW_BUILD: ${{matrix.pyver}}-${{matrix.platform}}_${{matrix.arch}}
CIBW_ARCHS_LINUX: auto aarch64 ppc64le
CIBW_BEFORE_ALL_LINUX: ./scripts/build/wheel_linux_before_all.sh
CIBW_REPAIR_WHEEL_COMMAND: >-
./scripts/build/strip_wheel.sh {wheel}
&& auditwheel repair -w {dest_dir} {wheel}
CIBW_TEST_COMMAND: >-
export PYTHONPATH={project} &&
python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
CIBW_ENVIRONMENT: >-
PACKAGE_NAME=psycopg2-binary
LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build
PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64"
PSYCOPG2_TESTDB=postgres
PSYCOPG2_TESTDB_HOST=172.17.0.1
PSYCOPG2_TESTDB_USER=postgres
PSYCOPG2_TESTDB_PASSWORD=password
PSYCOPG2_TEST_FAST=1
- uses: actions/upload-artifact@v3
with:
path: ./wheelhouse/*.whl
services:
postgresql:
image: postgres:13
image: postgres:14
env:
POSTGRES_PASSWORD: password
ports:

4
scripts/build/build_libpq.sh
View File

@ -18,7 +18,7 @@ ldap_version="2.6.3"
# last release: https://github.com/cyrusimap/cyrus-sasl/releases
sasl_version="2.1.28"
export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/usr/local}
export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build}
if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then
echo "libpq already available: build skipped" >&2
@ -152,7 +152,7 @@ if [ ! -d "${postgres_dir}" ]; then
src/include/pg_config_manual.h
# Often needed, but currently set by the workflow
export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib;${LIBPQ_BUILD_PREFIX}/lib64"
# export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib"
./configure --prefix=${LIBPQ_BUILD_PREFIX} --sysconfdir=/etc/postgresql-common \
--without-readline --with-gssapi --with-openssl --with-pam --with-ldap \

76
scripts/build/build_manylinux2014.sh
View File

@ -1,76 +0,0 @@
#!/bin/bash
# Create manylinux2014 wheels for psycopg2
#
# manylinux2014 is built on CentOS 7, which packages an old version of the
# libssl, (1.0, which has concurrency problems with the Python libssl). So we
# need to build these libraries from source.
#
# Look at the .github/workflows/packages.yml file for hints about how to use it.
set -euo pipefail
set -x
dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
prjdir="$( cd "${dir}/../.." && pwd )"
# Build all the available versions, or just the ones specified in PYVERS
if [ ! "${PYVERS:-}" ]; then
PYVERS="$(ls /opt/python/)"
fi
# Find psycopg version
version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/")
# A gratuitous comment to fix broken vim syntax file: '")
distdir="${prjdir}/dist"
# Replace the package name
if [[ "${PACKAGE_NAME:-}" ]]; then
sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
"${prjdir}/setup.py"
fi
# Build depending libraries
"${dir}/build_libpq.sh" > /dev/null
# Create the wheel packages
for pyver in $PYVERS; do
pybin="/opt/python/${pyver}/bin"
"${pybin}/pip" wheel "${prjdir}" -w "${prjdir}/dist/"
done
# Bundle external shared libraries into the wheels
for whl in "${prjdir}"/dist/*.whl; do
"${dir}/strip_wheel.sh" "$whl"
auditwheel repair "$whl" -w "$distdir"
done
# Make sure the libpq is not in the system
for f in $(find /usr/local/lib -name libpq\*) ; do
mkdir -pv "/libpqbak/$(dirname $f)"
mv -v "$f" "/libpqbak/$(dirname $f)"
done
# Install packages and test
cd "${prjdir}"
for pyver in $PYVERS; do
pybin="/opt/python/${pyver}/bin"
"${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
# Print psycopg and libpq versions
"${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)"
"${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
"${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
# Fail if we are not using the expected libpq library
if [[ "${WANT_LIBPQ:-}" ]]; then
"${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
fi
"${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
done
# Restore the libpq packages
for f in $(cd /libpqbak/ && find . -not -type d); do
mv -v "/libpqbak/$f" "/$f"
done

68
scripts/build/build_musllinux_1_1.sh
View File

@ -1,68 +0,0 @@
#!/bin/bash
# Create musllinux_1_1 wheels for psycopg2
#
# Look at the .github/workflows/packages.yml file for hints about how to use it.
set -euo pipefail
set -x
dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
prjdir="$( cd "${dir}/../.." && pwd )"
# Build all the available versions, or just the ones specified in PYVERS
if [ ! "${PYVERS:-}" ]; then
PYVERS="$(ls /opt/python/)"
fi
# Find psycopg version
version=$(grep -e ^PSYCOPG_VERSION "${prjdir}/setup.py" | sed "s/.*'\(.*\)'/\1/")
# A gratuitous comment to fix broken vim syntax file: '")
distdir="${prjdir}/dist/psycopg2-$version"
# Replace the package name
if [[ "${PACKAGE_NAME:-}" ]]; then
sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
"${prjdir}/setup.py"
fi
# Install prerequisite libraries
apk update
apk add postgresql-dev
# Add findutils because the Busybox version lacks the `-ls` flag, used by the
# `strip_wheel.sh` script.
apk add findutils
# Create the wheel packages
for pyver in $PYVERS; do
pybin="/opt/python/${pyver}/bin"
"${pybin}/python" -m build -w -o "${prjdir}/dist/" "${prjdir}"
done
# Bundle external shared libraries into the wheels
for whl in "${prjdir}"/dist/*.whl; do
"${dir}/strip_wheel.sh" "$whl"
auditwheel repair "$whl" -w "$distdir"
done
# Make sure the postgresql-dev is not in the system
apk del postgresql-dev
# Install packages and test
cd "${prjdir}"
for pyver in $PYVERS; do
pybin="/opt/python/${pyver}/bin"
"${pybin}/pip" install ${PACKAGE_NAME:-psycopg2} --no-index -f "$distdir"
# Print psycopg and libpq versions
"${pybin}/python" -c "import psycopg2; print(psycopg2.__version__)"
"${pybin}/python" -c "import psycopg2; print(psycopg2.__libpq_version__)"
"${pybin}/python" -c "import psycopg2; print(psycopg2.extensions.libpq_version())"
# Fail if we are not using the expected libpq library
if [[ "${WANT_LIBPQ:-}" ]]; then
"${pybin}/python" -c "import psycopg2, sys; sys.exit(${WANT_LIBPQ} != psycopg2.extensions.libpq_version())"
fi
"${pybin}/python" -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')"
done

37
scripts/build/print_so_versions.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
# Take a .so file as input and print the Debian packages and versions of the
# libraries it links.
set -euo pipefail
# set -x
source /etc/os-release
sofile="$1"
case "$ID" in
alpine)
depfiles=$( (ldd "$sofile" 2>/dev/null || true) | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/')
(for depfile in $depfiles; do
echo "$(basename "$depfile") => $(apk info --who-owns "${depfile}" | awk '{print $(NF)}')"
done) | sort | uniq
;;
debian)
depfiles=$(ldd "$sofile" | grep '=>' | sed 's/.*=> \(.*\) (.*)/\1/')
(for depfile in $depfiles; do
pkgname=$(dpkg -S "${depfile}" | sed 's/\(\): .*/\1/')
dpkg -l "${pkgname}" | grep '^ii' | awk '{print $2 " => " $3}'
done) | sort | uniq
;;
centos)
echo "TODO!"
;;
*)
echo "$0: unexpected Linux distribution: '$ID'" >&2
exit 1
;;
esac

34
scripts/build/strip_wheel.sh
View File

@ -14,28 +14,36 @@
# This script is designed to run on a wheel archive before auditwheel.
set -euo pipefail
set -x
# set -x
source /etc/os-release
dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
wheel=$(realpath "$1")
shift
# python or python3?
if which python > /dev/null; then
py=python
else
py=python3
fi
tmpdir=$(mktemp -d)
trap "rm -r ${tmpdir}" EXIT
cd "${tmpdir}"
$py -m zipfile -e "${wheel}" .
python -m zipfile -e "${wheel}" .
find . -name *.so -ls -exec strip "$@" {} \;
# Display the size after strip
find . -name *.so -ls
echo "
Libs before:"
# Busybox doesn't have "find -ls"
find . -name \*.so | xargs ls -l
$py -m zipfile -c "${wheel}" *
# On Debian, print the package versions libraries come from
echo "
Dependencies versions of '_psycopg.so' library:"
"${dir}/print_so_versions.sh" "$(find . -name \*_psycopg\*.so)"
find . -name \*.so -exec strip "$@" {} \;
echo "
Libs after:"
find . -name \*.so | xargs ls -l
python -m zipfile -c ${wheel} *
cd -

53
scripts/build/wheel_linux_before_all.sh Executable file
View File

@ -0,0 +1,53 @@
#!/bin/bash
# Configure the libraries needed to build wheel packages on linux.
# This script is designed to be used by cibuildwheel as CIBW_BEFORE_ALL_LINUX
set -euo pipefail
set -x
dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
prjdir="$( cd "${dir}/../.." && pwd )"
source /etc/os-release
# Install PostgreSQL development files.
case "$ID" in
alpine)
"${dir}/build_libpq.sh" > /dev/null
;;
debian)
# Note that the pgdg doesn't have an aarch64 repository so wheels are
# build with the libpq packaged with Debian 9, which is 9.6.
if [ "$AUDITWHEEL_ARCH" != 'aarch64' ]; then
echo "deb http://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main" \
> /etc/apt/sources.list.d/pgdg.list
# TODO: On 2021-11-09 curl fails on 'ppc64le' with:
# curl: (60) SSL certificate problem: certificate has expired
# Test again later if -k can be removed.
curl -skf https://www.postgresql.org/media/keys/ACCC4CF8.asc \
> /etc/apt/trusted.gpg.d/postgresql.asc
fi
apt-get update
apt-get -y upgrade
apt-get -y install libpq-dev
;;
centos)
"${dir}/build_libpq.sh" > /dev/null
;;
*)
echo "$0: unexpected Linux distribution: '$ID'" >&2
exit 1
;;
esac
# Replace the package name
if [[ "${PACKAGE_NAME:-}" ]]; then
sed -i "s/^setup(name=\"psycopg2\"/setup(name=\"${PACKAGE_NAME}\"/" \
"${prjdir}/setup.py"
fi