From 8341792c5bb8d47408f72afc5ae128c53e17fbf4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 4 Jan 2017 04:39:53 +0100 Subject: [PATCH 1/5] Added script to create manylinux1 wheels See issue #425 --- .gitignore | 1 + scripts/wheels-build.sh | 47 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100755 scripts/wheels-build.sh diff --git a/.gitignore b/.gitignore index a017eb3e..409bb3a7 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ scripts/pypi_docs_upload.py env .tox /rel +/wheels diff --git a/scripts/wheels-build.sh b/scripts/wheels-build.sh new file mode 100755 index 00000000..ede3024b --- /dev/null +++ b/scripts/wheels-build.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Create manylinux1 wheels for psycopg2 +# +# Run this script with something like: +# +# docker run -t --rm -v `pwd`:/psycopg quay.io/pypa/manylinux1_x86_64 /psycopg/scripts/wheels-build.sh +# docker run -t --rm -v `pwd`:/psycopg quay.io/pypa/manylinux1_i686 linux32 /psycopg/scripts/wheels-build.sh +# +# (Note: -t is requrired for sudo) + +set -e -x + +# Install postgres packages for build and testing +# This doesn't work: +# rpm -Uvh "http://yum.postgresql.org/9.5/redhat/rhel-5-x86_64/pgdg-redhat95-9.5-3.noarch.rpm" +wget -O "/tmp/pgdg.rpm" "https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-5-x86_64/pgdg-centos95-9.5-3.noarch.rpm" +rpm -Uvh "/tmp/pgdg.rpm" +yum install -y postgresql95-devel postgresql95-server sudo + +# Make pg_config available +export PGPATH=/usr/pgsql-9.5/bin/ +export PATH="$PGPATH:$PATH" + +# Create the wheel packages +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/pip" wheel /psycopg/ -w wheels/ +done + +# Bundle external shared libraries into the wheels +for WHL in wheels/*.whl; do + auditwheel repair "$WHL" -w /psycopg/wheels +done + +# Create a test cluster +/usr/bin/sudo -u postgres "$PGPATH/initdb" -D /var/lib/pgsql/9.5/data/ +/usr/bin/sudo -u postgres "$PGPATH/pg_ctl" -D /var/lib/pgsql/9.5/data/ start +sleep 5 # wait server started +/usr/bin/sudo -u postgres "$PGPATH/createdb" psycopg2_test + +export PSYCOPG2_TESTDB_USER=postgres + +# Install packages and test +for PYBIN in /opt/python/*/bin; do + "${PYBIN}/pip" install psycopg2 --no-index -f /psycopg/wheels + "${PYBIN}/python" -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" +done From 815869375b6346add3b76d74dab2e8e177c0e0d0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 1 Feb 2017 17:05:47 +0000 Subject: [PATCH 2/5] Merge back manylinux build script from the psycopg2-wheels project --- .../{wheels-build.sh => build-manylinux.sh} | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) rename scripts/{wheels-build.sh => build-manylinux.sh} (66%) diff --git a/scripts/wheels-build.sh b/scripts/build-manylinux.sh similarity index 66% rename from scripts/wheels-build.sh rename to scripts/build-manylinux.sh index ede3024b..4ceb1330 100755 --- a/scripts/wheels-build.sh +++ b/scripts/build-manylinux.sh @@ -4,8 +4,8 @@ # # Run this script with something like: # -# docker run -t --rm -v `pwd`:/psycopg quay.io/pypa/manylinux1_x86_64 /psycopg/scripts/wheels-build.sh -# docker run -t --rm -v `pwd`:/psycopg quay.io/pypa/manylinux1_i686 linux32 /psycopg/scripts/wheels-build.sh +# docker run -t --rm -v `pwd`:/psycopg2 quay.io/pypa/manylinux1_x86_64 /psycopg2/scripts/build-manylinux.sh +# docker run -t --rm -v `pwd`:/psycopg2 quay.io/pypa/manylinux1_i686 linux32 /psycopg2/scripts/build-manylinux.sh # # (Note: -t is requrired for sudo) @@ -15,21 +15,25 @@ set -e -x # This doesn't work: # rpm -Uvh "http://yum.postgresql.org/9.5/redhat/rhel-5-x86_64/pgdg-redhat95-9.5-3.noarch.rpm" wget -O "/tmp/pgdg.rpm" "https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-5-x86_64/pgdg-centos95-9.5-3.noarch.rpm" -rpm -Uvh "/tmp/pgdg.rpm" +rpm -Uv "/tmp/pgdg.rpm" yum install -y postgresql95-devel postgresql95-server sudo # Make pg_config available export PGPATH=/usr/pgsql-9.5/bin/ export PATH="$PGPATH:$PATH" +# Find psycopg version +export VERSION=$(grep -e ^PSYCOPG_VERSION /psycopg2/setup.py | sed "s/.*'\(.*\)'/\1/") +export WHEELSDIR="/psycopg2/wheels/psycopg2-$VERSION" + # Create the wheel packages for PYBIN in /opt/python/*/bin; do - "${PYBIN}/pip" wheel /psycopg/ -w wheels/ + "${PYBIN}/pip" wheel /psycopg2/ -w "$WHEELSDIR" done # Bundle external shared libraries into the wheels -for WHL in wheels/*.whl; do - auditwheel repair "$WHL" -w /psycopg/wheels +for WHL in "$WHEELSDIR"/*.whl; do + auditwheel repair "$WHL" -w "$WHEELSDIR" done # Create a test cluster @@ -42,6 +46,6 @@ export PSYCOPG2_TESTDB_USER=postgres # Install packages and test for PYBIN in /opt/python/*/bin; do - "${PYBIN}/pip" install psycopg2 --no-index -f /psycopg/wheels + "${PYBIN}/pip" install psycopg2 --no-index -f "$WHEELSDIR" "${PYBIN}/python" -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" done From 9ca51e0ed921270b2c641959ffeeb2ca9f58809b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 1 Feb 2017 17:33:12 +0000 Subject: [PATCH 3/5] Use the server on the host to test --- scripts/build-manylinux.sh | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scripts/build-manylinux.sh b/scripts/build-manylinux.sh index 4ceb1330..e1bdd917 100755 --- a/scripts/build-manylinux.sh +++ b/scripts/build-manylinux.sh @@ -4,10 +4,11 @@ # # Run this script with something like: # -# docker run -t --rm -v `pwd`:/psycopg2 quay.io/pypa/manylinux1_x86_64 /psycopg2/scripts/build-manylinux.sh -# docker run -t --rm -v `pwd`:/psycopg2 quay.io/pypa/manylinux1_i686 linux32 /psycopg2/scripts/build-manylinux.sh +# docker run --rm -v `pwd`:/psycopg2 quay.io/pypa/manylinux1_x86_64 /psycopg2/scripts/build-manylinux.sh +# docker run --rm -v `pwd`:/psycopg2 quay.io/pypa/manylinux1_i686 linux32 /psycopg2/scripts/build-manylinux.sh # -# (Note: -t is requrired for sudo) +# Tests run against a postgres on the host. Use -e PSYCOPG_TESTDB_USER=... etc +# to configure tests run. set -e -x @@ -16,7 +17,7 @@ set -e -x # rpm -Uvh "http://yum.postgresql.org/9.5/redhat/rhel-5-x86_64/pgdg-redhat95-9.5-3.noarch.rpm" wget -O "/tmp/pgdg.rpm" "https://download.postgresql.org/pub/repos/yum/9.5/redhat/rhel-5-x86_64/pgdg-centos95-9.5-3.noarch.rpm" rpm -Uv "/tmp/pgdg.rpm" -yum install -y postgresql95-devel postgresql95-server sudo +yum install -y postgresql95-devel # Make pg_config available export PGPATH=/usr/pgsql-9.5/bin/ @@ -36,13 +37,11 @@ for WHL in "$WHEELSDIR"/*.whl; do auditwheel repair "$WHL" -w "$WHEELSDIR" done -# Create a test cluster -/usr/bin/sudo -u postgres "$PGPATH/initdb" -D /var/lib/pgsql/9.5/data/ -/usr/bin/sudo -u postgres "$PGPATH/pg_ctl" -D /var/lib/pgsql/9.5/data/ start -sleep 5 # wait server started -/usr/bin/sudo -u postgres "$PGPATH/createdb" psycopg2_test +# Make sure libpq is not in the system +yum remove -y postgresql95-devel -export PSYCOPG2_TESTDB_USER=postgres +# Connect to the host to test. Use 'docker -e' to pass other variables +export PSYCOPG2_TESTDB_HOST=$(ip route show | awk '/default/ {print $3}') # Install packages and test for PYBIN in /opt/python/*/bin; do From f24de0357ff4472aa8bb283980ea27b3ddd722e4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 2 Feb 2017 01:53:50 +0000 Subject: [PATCH 4/5] Allow skipping the slow test It's not so much about tests being slow: some just get stuck and timeout travis. Skipped all tests taking about more than 0.2s to run on my laptop. Fast testing takes about 8s instead of 24. --- tests/test_async.py | 3 ++- tests/test_cancel.py | 3 ++- tests/test_connection.py | 6 +++++- tests/test_copy.py | 7 +++++-- tests/test_cursor.py | 5 +++-- tests/test_errcodes.py | 3 ++- tests/test_green.py | 3 ++- tests/test_notify.py | 9 ++++++++- tests/test_transaction.py | 4 +++- tests/test_types_extras.py | 6 +++--- tests/testutils.py | 11 ++++++++++- 11 files changed, 45 insertions(+), 15 deletions(-) diff --git a/tests/test_async.py b/tests/test_async.py index 6f8fed58..b379baea 100755 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -23,7 +23,7 @@ # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. -from testutils import unittest, skip_before_postgres +from testutils import unittest, skip_before_postgres, slow import psycopg2 from psycopg2 import extensions @@ -318,6 +318,7 @@ class AsyncTests(ConnectingTestCase): self.assert_(conn.async) conn.close() + @slow def test_flush_on_write(self): # a very large query requires a flush loop to be sent to the backend curs = self.conn.cursor() diff --git a/tests/test_cancel.py b/tests/test_cancel.py index a8eb7506..cb08e5f8 100755 --- a/tests/test_cancel.py +++ b/tests/test_cancel.py @@ -30,7 +30,7 @@ import psycopg2.extensions from psycopg2 import extras from testconfig import dsn -from testutils import unittest, ConnectingTestCase, skip_before_postgres +from testutils import unittest, ConnectingTestCase, skip_before_postgres, slow class CancelTests(ConnectingTestCase): @@ -48,6 +48,7 @@ class CancelTests(ConnectingTestCase): def test_empty_cancel(self): self.conn.cancel() + @slow @skip_before_postgres(8, 2) def test_cancel(self): errors = [] diff --git a/tests/test_connection.py b/tests/test_connection.py index 833751b9..a5b5c418 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -35,7 +35,7 @@ from psycopg2 import extensions as ext from testutils import ( unittest, decorate_all_tests, skip_if_no_superuser, skip_before_postgres, skip_after_postgres, skip_before_libpq, - ConnectingTestCase, skip_if_tpc_disabled, skip_if_windows) + ConnectingTestCase, skip_if_tpc_disabled, skip_if_windows, slow) from testconfig import dsn, dbname @@ -196,6 +196,7 @@ class ConnectionTests(ConnectingTestCase): self.assertRaises(psycopg2.NotSupportedError, cnn.xid, 42, "foo", "bar") + @slow @skip_before_postgres(8, 2) def test_concurrent_execution(self): def slave(): @@ -246,6 +247,7 @@ class ConnectionTests(ConnectingTestCase): gc.collect() self.assert_(w() is None) + @slow def test_commit_concurrency(self): # The problem is the one reported in ticket #103. Because of bad # status check, we commit even when a commit is already on its way. @@ -899,6 +901,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): (dbname,)) self.assertEqual('42_Z3RyaWQ=_YnF1YWw=', cur.fetchone()[0]) + @slow def test_xid_roundtrip(self): for fid, gtrid, bqual in [ (0, "", ""), @@ -921,6 +924,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn.tpc_rollback(xid) + @slow def test_unparsed_roundtrip(self): for tid in [ '', diff --git a/tests/test_copy.py b/tests/test_copy.py index ac42c980..8cbe4eaa 100755 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -24,8 +24,8 @@ import sys import string -from testutils import unittest, ConnectingTestCase, decorate_all_tests -from testutils import skip_if_no_iobase, skip_before_postgres +from testutils import (unittest, ConnectingTestCase, decorate_all_tests, + skip_if_no_iobase, skip_before_postgres, slow) from cStringIO import StringIO from itertools import cycle, izip from subprocess import Popen, PIPE @@ -77,6 +77,7 @@ class CopyTests(ConnectingTestCase): data text )''') + @slow def test_copy_from(self): curs = self.conn.cursor() try: @@ -84,6 +85,7 @@ class CopyTests(ConnectingTestCase): finally: curs.close() + @slow def test_copy_from_insane_size(self): # Trying to trigger a "would block" error curs = self.conn.cursor() @@ -120,6 +122,7 @@ class CopyTests(ConnectingTestCase): self.assertRaises(ZeroDivisionError, curs.copy_from, MinimalRead(f), "tcopy", columns=cols()) + @slow def test_copy_to(self): curs = self.conn.cursor() try: diff --git a/tests/test_cursor.py b/tests/test_cursor.py index fc924c4b..a8fedccb 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -26,8 +26,8 @@ import time import pickle import psycopg2 import psycopg2.extensions -from testutils import unittest, ConnectingTestCase, skip_before_postgres -from testutils import skip_if_no_namedtuple, skip_if_no_getrefcount +from testutils import (unittest, ConnectingTestCase, skip_before_postgres, + skip_if_no_namedtuple, skip_if_no_getrefcount, slow) class CursorTests(ConnectingTestCase): @@ -331,6 +331,7 @@ class CursorTests(ConnectingTestCase): curs.scroll(2) self.assertRaises(psycopg2.OperationalError, curs.scroll, -1) + @slow @skip_before_postgres(8, 2) def test_iter_named_cursor_efficient(self): curs = self.conn.cursor('tmp') diff --git a/tests/test_errcodes.py b/tests/test_errcodes.py index 6865194f..accee565 100755 --- a/tests/test_errcodes.py +++ b/tests/test_errcodes.py @@ -22,7 +22,7 @@ # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. -from testutils import unittest, ConnectingTestCase +from testutils import unittest, ConnectingTestCase, slow try: reload @@ -34,6 +34,7 @@ from psycopg2 import errorcodes class ErrocodeTests(ConnectingTestCase): + @slow def test_lookup_threadsafe(self): # Increase if it does not fail with KeyError diff --git a/tests/test_green.py b/tests/test_green.py index 0424a2cc..6d1571d4 100755 --- a/tests/test_green.py +++ b/tests/test_green.py @@ -27,7 +27,7 @@ import psycopg2 import psycopg2.extensions import psycopg2.extras -from testutils import ConnectingTestCase +from testutils import ConnectingTestCase, slow class ConnectionStub(object): @@ -61,6 +61,7 @@ class GreenTestCase(ConnectingTestCase): lambda conn: psycopg2.extras.wait_select(stub)) return stub + @slow def test_flush_on_write(self): # a very large query requires a flush loop to be sent to the backend conn = self.conn diff --git a/tests/test_notify.py b/tests/test_notify.py index 1a0ac457..4e99a3e9 100755 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -26,7 +26,7 @@ from testutils import unittest import psycopg2 from psycopg2 import extensions -from testutils import ConnectingTestCase, script_to_py3 +from testutils import ConnectingTestCase, script_to_py3, slow from testconfig import dsn import sys @@ -72,6 +72,7 @@ conn.close() return Popen([sys.executable, '-c', script_to_py3(script)], stdout=PIPE) + @slow def test_notifies_received_on_poll(self): self.autocommit(self.conn) self.listen('foo') @@ -90,6 +91,7 @@ conn.close() self.assertEqual(pid, self.conn.notifies[0][0]) self.assertEqual('foo', self.conn.notifies[0][1]) + @slow def test_many_notifies(self): self.autocommit(self.conn) for name in ['foo', 'bar', 'baz']: @@ -119,6 +121,7 @@ conn.close() self.assertEqual(pid, self.conn.notifies[0][0]) self.assertEqual('foo', self.conn.notifies[0][1]) + @slow def test_notify_object(self): self.autocommit(self.conn) self.listen('foo') @@ -128,6 +131,7 @@ conn.close() notify = self.conn.notifies[0] self.assert_(isinstance(notify, psycopg2.extensions.Notify)) + @slow def test_notify_attributes(self): self.autocommit(self.conn) self.listen('foo') @@ -140,6 +144,7 @@ conn.close() self.assertEqual('foo', notify.channel) self.assertEqual('', notify.payload) + @slow def test_notify_payload(self): if self.conn.server_version < 90000: return self.skipTest("server version %s doesn't support notify payload" @@ -155,6 +160,7 @@ conn.close() self.assertEqual('foo', notify.channel) self.assertEqual('Hello, world!', notify.payload) + @slow def test_notify_deque(self): from collections import deque self.autocommit(self.conn) @@ -167,6 +173,7 @@ conn.close() self.assert_(isinstance(notify, psycopg2.extensions.Notify)) self.assertEqual(len(self.conn.notifies), 0) + @slow def test_notify_noappend(self): self.autocommit(self.conn) self.conn.notifies = None diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 2dc44ec5..36947dee 100755 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -23,7 +23,7 @@ # License for more details. import threading -from testutils import unittest, ConnectingTestCase, skip_before_postgres +from testutils import unittest, ConnectingTestCase, skip_before_postgres, slow import psycopg2 from psycopg2.extensions import ( @@ -131,6 +131,7 @@ class DeadlockSerializationTests(ConnectingTestCase): ConnectingTestCase.tearDown(self) + @slow def test_deadlock(self): self.thread1_error = self.thread2_error = None step1 = threading.Event() @@ -178,6 +179,7 @@ class DeadlockSerializationTests(ConnectingTestCase): self.assertTrue(isinstance( error, psycopg2.extensions.TransactionRollbackError)) + @slow def test_serialisation_failure(self): self.thread1_error = self.thread2_error = None step1 = threading.Event() diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 8e615616..f28c5c21 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -22,9 +22,8 @@ from datetime import date, datetime from functools import wraps from pickle import dumps, loads -from testutils import unittest, skip_if_no_uuid, skip_before_postgres -from testutils import ConnectingTestCase, decorate_all_tests -from testutils import py3_raises_typeerror +from testutils import (unittest, skip_if_no_uuid, skip_before_postgres, + ConnectingTestCase, decorate_all_tests, py3_raises_typeerror, slow) import psycopg2 import psycopg2.extras @@ -708,6 +707,7 @@ class AdaptTypeTestCase(ConnectingTestCase): curs.execute("select (1,2)::type_ii") self.assertRaises(psycopg2.DataError, curs.fetchone) + @slow @skip_if_no_composite @skip_before_postgres(8, 4) def test_from_tables(self): diff --git a/tests/testutils.py b/tests/testutils.py index 93477357..b34a5a87 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -447,7 +447,6 @@ def script_to_py3(script): class py3_raises_typeerror(object): - def __enter__(self): pass @@ -455,3 +454,13 @@ class py3_raises_typeerror(object): if sys.version_info[0] >= 3: assert type is TypeError return True + + +def slow(f): + """Decorator to mark slow tests we may want to skip""" + @wraps(f) + def slow_(self): + if os.environ.get('PSYCOPG2_TEST_FAST'): + return self.skipTest("slow test") + return f(self) + return slow_ From 88a21689cee0e23ea4b2a6e3b423aa0c6c29c36a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 2 Feb 2017 02:09:59 +0000 Subject: [PATCH 5/5] Added note about finding slow tests --- tests/testutils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/testutils.py b/tests/testutils.py index b34a5a87..c36f6749 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -457,7 +457,12 @@ class py3_raises_typeerror(object): def slow(f): - """Decorator to mark slow tests we may want to skip""" + """Decorator to mark slow tests we may want to skip + + Note: in order to find slow tests you can run: + + make check 2>&1 | ts -i "%.s" | sort -n + """ @wraps(f) def slow_(self): if os.environ.get('PSYCOPG2_TEST_FAST'):