From cfb0937605c20b6c2004625542031ebaba0f1101 Mon Sep 17 00:00:00 2001 From: Ashesh Vashi Date: Mon, 17 Jul 2017 10:32:56 +0530 Subject: [PATCH 01/55] Added support for preparing the encrypted password of a PostgreSQL password using the libpq functions - 'PQencryptPasswordConn', and 'PQencryptPassword'. --- doc/src/connection.rst | 27 +++++++++++++++ psycopg/connection_type.c | 69 +++++++++++++++++++++++++++++++++++++++ tests/test_connection.py | 56 ++++++++++++++++++++++++++++++- 3 files changed, 151 insertions(+), 1 deletion(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index fbbc53e1..0e6b0b13 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -681,6 +681,33 @@ The ``connection`` class .. __: http://www.postgresql.org/docs/current/static/libpq-status.html#LIBPQ-PQTRANSACTIONSTATUS + .. method:: encrypt_password(password, user, [algorithm]) + + Returns the encrypted form of a PostgreSQL password based on the + current password encryption algorithm. + + Raises `~psycopg2.NotSupportedError` if the ``psycopg2`` module was + compiled with a ``libpq`` version lesser than 10 (which can be detected + by the `~psycopg2.__libpq_version__` constant), when encryption + algorithm other than 'md5' is specified for the server version greater + than, or equal to 10. + + Ignores the encrytion algorithm for servers version less than 10, and + always uses 'md5' as encryption algorithm. + + .. seealso:: libpq docs for `PQencryptPasswordConn()`__ for details. + + .. __: https://www.postgresql.org/docs/devel/static/libpq-misc.html#libpq-pqencryptpasswordconn + + .. seealso:: libpq docs for `PQencryptPassword()`__ for details. + + .. __: https://www.postgresql.org/docs/devel/static/libpq-misc.html#libpq-pqencryptpassword + + .. seealso:: libpq docs for `password_encryption`__ for details. + + .. __: https://www.postgresql.org/docs/devel/static/runtime-config-connection.html#guc-password-encryption + + .. index:: pair: Protocol; Version diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 8c5085b5..7d365f3f 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -547,6 +547,73 @@ do { \ EXC_IF_TPC_PREPARED(self, what); \ } while(0) +/* encrypt_password - Prepare the encrypted password form */ +#define psyco_encrypt_password_doc \ +"encrypt_password('password', 'user', ...) -- Prepares the encrypted form of a PostgreSQL password.\n\n" \ +"Accepted arguments are 'algorithm'." + +static PyObject * +psyco_encrypt_password(connectionObject *self, PyObject *args, PyObject *kwargs) +{ + const char *password = NULL, + *user = NULL, + *algorithm = NULL; + char *encrypted = NULL; + + PyObject *res = Py_None; + + static char *kwlist[] = {"password", "user", "algorithm", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ss|s", kwlist, &password, &user, &algorithm)) { + return NULL; + } + + if (self->server_version < 100000 || + (algorithm && strcmp(algorithm, "md5") == 0) + ) { + encrypted = PQencryptPassword(password, user); + + if (encrypted != NULL) + { + res = PyString_FromString(encrypted); + free(encrypted); + } + return res; + } + else + { +#if PG_VERSION_NUM >= 100000 + encrypted = PQencryptPasswordConn(self->pgconn, password, user, algorithm); + + if (!encrypted) + { + const char *msg; + msg = PQerrorMessage(self->pgconn); + if (msg && *msg) { + PyErr_Format( + ProgrammingError, + "Error encrypting the password!\n%s", + msg + ); + return NULL; + } + } + else + { + res = PyString_FromString(encrypted); + } + + return res; +#else + PyErr_SetString( + NotSupportedError, + "Password encryption (other than 'md5' algorithm) is not supported for the server version >= 10 in libpq < 10" + ); +#endif + } + return NULL; +} + /* set_session - set default transaction characteristics */ #define psyco_conn_set_session_doc \ @@ -1176,6 +1243,8 @@ static struct PyMethodDef connectionObject_methods[] = { METH_NOARGS, psyco_conn_isexecuting_doc}, {"cancel", (PyCFunction)psyco_conn_cancel, METH_NOARGS, psyco_conn_cancel_doc}, + {"encrypt_password", (PyCFunction)psyco_encrypt_password, + METH_VARARGS|METH_KEYWORDS, psyco_encrypt_password_doc}, {NULL} }; diff --git a/tests/test_connection.py b/tests/test_connection.py index 42a406ce..b9944ae8 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -37,7 +37,9 @@ from psycopg2 import extensions as ext from testutils import ( script_to_py3, unittest, decorate_all_tests, skip_if_no_superuser, skip_before_postgres, skip_after_postgres, skip_before_libpq, - ConnectingTestCase, skip_if_tpc_disabled, skip_if_windows, slow) + ConnectingTestCase, skip_if_tpc_disabled, skip_if_windows, slow, + libpq_version +) from testconfig import dsn, dbname @@ -1382,6 +1384,58 @@ class TransactionControlTests(ConnectingTestCase): cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'off') + @skip_before_postgres(10) + def test_encrypt_password_post_9_6(self): + cur = self.conn.cursor() + cur.execute("SHOW password_encryption;") + server_encryption_algorithm = cur.fetchone()[0] + + # MD5 algorithm + self.assertEqual( + self.conn.encrypt_password('psycopg2', 'ashesh', 'md5'), + 'md594839d658c28a357126f105b9cb14cfc' + ) + + if libpq_version() < 100000: + if server_encryption_algorithm == 'md5': + self.assertEqual( + self.conn.encrypt_password('psycopg2', 'ashesh'), + 'md594839d658c28a357126f105b9cb14cfc' + ) + else: + self.assertRaises( + psycopg2.ProgrammingError, + self.conn.encrypt_password, 'psycopg2', 'ashesh' + ) + else: + enc_password = self.conn.encrypt_password('psycopg2', 'ashesh') + if server_encryption_algorithm == 'md5': + self.assertEqual( + enc_password, 'md594839d658c28a357126f105b9cb14cfc' + ) + elif server_encryption_algorithm == 'scram-sha-256': + self.assertEqual(enc_password[:14], 'SCRAM-SHA-256$') + + self.assertEqual( + self.conn.encrypt_password( + 'psycopg2', 'ashesh', 'scram-sha-256' + )[:14], 'SCRAM-SHA-256$' + ) + + @skip_after_postgres(10) + def test_encrypt_password_pre_10(self): + self.assertEqual( + self.conn.encrypt_password('psycopg2', 'ashesh'), + 'md594839d658c28a357126f105b9cb14cfc' + ) + + # Encryption algorithm will be ignored for postgres version < 10, it + # will always use MD5. + self.assertEqual( + self.conn.encrypt_password('psycopg2', 'ashesh', 'abc'), + 'md594839d658c28a357126f105b9cb14cfc' + ) + class AutocommitTests(ConnectingTestCase): def test_closed(self): From 78eb80d0cfb305d7eb467f7f25fdea50d379dd0b Mon Sep 17 00:00:00 2001 From: Ashesh Vashi Date: Mon, 17 Jul 2017 10:54:50 +0530 Subject: [PATCH 02/55] Using 'Text_FromUTF8' macro for transforming the encrypted C string to Python string to make it Python 3 compatible. --- psycopg/connection_type.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 7d365f3f..52c1a9f8 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -575,8 +575,8 @@ psyco_encrypt_password(connectionObject *self, PyObject *args, PyObject *kwargs) if (encrypted != NULL) { - res = PyString_FromString(encrypted); - free(encrypted); + res = Text_FromUTF8(encrypted); + PQfreemem(encrypted); } return res; } @@ -600,10 +600,11 @@ psyco_encrypt_password(connectionObject *self, PyObject *args, PyObject *kwargs) } else { - res = PyString_FromString(encrypted); + res = Text_FromUTF8(encrypted); + PQfreemem(encrypted); } - return res; + #else PyErr_SetString( NotSupportedError, From 2c1966a7f61e6623798395c02a8bce5027e258f7 Mon Sep 17 00:00:00 2001 From: Ashesh Vashi Date: Mon, 17 Jul 2017 11:06:55 +0530 Subject: [PATCH 03/55] When compiled with libpq version < 10, it raises 'psycopg2.NotSupportedError' (not, psycopg2.ProgrammingError). --- tests/test_connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index b9944ae8..5894f1d2 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1404,7 +1404,7 @@ class TransactionControlTests(ConnectingTestCase): ) else: self.assertRaises( - psycopg2.ProgrammingError, + psycopg2.NotSupportedError, self.conn.encrypt_password, 'psycopg2', 'ashesh' ) else: From e089d94c8858d75db523d48738efa409d0ac4167 Mon Sep 17 00:00:00 2001 From: Ashesh Vashi Date: Mon, 17 Jul 2017 11:46:44 +0530 Subject: [PATCH 04/55] 'encrypt_password' raises 'psycopg2.NotSupportedErorr' exception for server version >= 10, when compiled using libpq version < 10, when no algorithm is specified. --- tests/test_connection.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 5894f1d2..4e1d5ccc 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1397,16 +1397,10 @@ class TransactionControlTests(ConnectingTestCase): ) if libpq_version() < 100000: - if server_encryption_algorithm == 'md5': - self.assertEqual( - self.conn.encrypt_password('psycopg2', 'ashesh'), - 'md594839d658c28a357126f105b9cb14cfc' - ) - else: - self.assertRaises( - psycopg2.NotSupportedError, - self.conn.encrypt_password, 'psycopg2', 'ashesh' - ) + self.assertRaises( + psycopg2.NotSupportedError, + self.conn.encrypt_password, 'psycopg2', 'ashesh' + ) else: enc_password = self.conn.encrypt_password('psycopg2', 'ashesh') if server_encryption_algorithm == 'md5': From 84d405894c4236c1d2bb54c673a71e4a8e78a2ec Mon Sep 17 00:00:00 2001 From: Ashesh Vashi Date: Thu, 14 Sep 2017 23:42:37 +0530 Subject: [PATCH 05/55] Moving the encrypt_password method from the connection class to the psycopgmodule, and exported it from psycopg2.extensions as per review comments. --- lib/extensions.py | 2 +- psycopg/connection_type.c | 70 ------------------------- psycopg/psycopgmodule.c | 105 ++++++++++++++++++++++++++++++++++++++ tests/test_connection.py | 17 +++--- 4 files changed, 116 insertions(+), 78 deletions(-) diff --git a/lib/extensions.py b/lib/extensions.py index 91b81331..47155bb7 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -63,7 +63,7 @@ from psycopg2._psycopg import ( # noqa string_types, binary_types, new_type, new_array_type, register_type, ISQLQuote, Notify, Diagnostics, Column, QueryCanceledError, TransactionRollbackError, - set_wait_callback, get_wait_callback, ) + set_wait_callback, get_wait_callback, encrypt_password, ) """Isolation level values.""" diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 52c1a9f8..8c5085b5 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -547,74 +547,6 @@ do { \ EXC_IF_TPC_PREPARED(self, what); \ } while(0) -/* encrypt_password - Prepare the encrypted password form */ -#define psyco_encrypt_password_doc \ -"encrypt_password('password', 'user', ...) -- Prepares the encrypted form of a PostgreSQL password.\n\n" \ -"Accepted arguments are 'algorithm'." - -static PyObject * -psyco_encrypt_password(connectionObject *self, PyObject *args, PyObject *kwargs) -{ - const char *password = NULL, - *user = NULL, - *algorithm = NULL; - char *encrypted = NULL; - - PyObject *res = Py_None; - - static char *kwlist[] = {"password", "user", "algorithm", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "ss|s", kwlist, &password, &user, &algorithm)) { - return NULL; - } - - if (self->server_version < 100000 || - (algorithm && strcmp(algorithm, "md5") == 0) - ) { - encrypted = PQencryptPassword(password, user); - - if (encrypted != NULL) - { - res = Text_FromUTF8(encrypted); - PQfreemem(encrypted); - } - return res; - } - else - { -#if PG_VERSION_NUM >= 100000 - encrypted = PQencryptPasswordConn(self->pgconn, password, user, algorithm); - - if (!encrypted) - { - const char *msg; - msg = PQerrorMessage(self->pgconn); - if (msg && *msg) { - PyErr_Format( - ProgrammingError, - "Error encrypting the password!\n%s", - msg - ); - return NULL; - } - } - else - { - res = Text_FromUTF8(encrypted); - PQfreemem(encrypted); - } - return res; - -#else - PyErr_SetString( - NotSupportedError, - "Password encryption (other than 'md5' algorithm) is not supported for the server version >= 10 in libpq < 10" - ); -#endif - } - return NULL; -} - /* set_session - set default transaction characteristics */ #define psyco_conn_set_session_doc \ @@ -1244,8 +1176,6 @@ static struct PyMethodDef connectionObject_methods[] = { METH_NOARGS, psyco_conn_isexecuting_doc}, {"cancel", (PyCFunction)psyco_conn_cancel, METH_NOARGS, psyco_conn_cancel_doc}, - {"encrypt_password", (PyCFunction)psyco_encrypt_password, - METH_VARARGS|METH_KEYWORDS, psyco_encrypt_password_doc}, {NULL} }; diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 6c95bd69..a6f7240e 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -403,6 +403,109 @@ psyco_libpq_version(PyObject *self) #endif } +/* encrypt_password - Prepare the encrypted password form */ +#define psyco_encrypt_password_doc \ +"encrypt_password(password, user, [conn_or_curs], [algorithm]) -- Prepares the encrypted form of a PostgreSQL password.\n\n" + +static PyObject * +psyco_encrypt_password(PyObject *self, PyObject *args) +{ + char *encrypted = NULL; + + PyObject *obj = NULL, + *res = Py_None, + *password = NULL, + *user = NULL, + *algorithm = NULL; + + connectionObject *conn = NULL; + + static char *kwlist[] = {"password", "user", "scope", "algorithm", NULL}; + + if (!PyArg_ParseTuple(args, "OO|OO", + &password, &user, &obj, &algorithm)) { + return NULL; + } + + if (obj != NULL && obj != Py_None) { + if (PyObject_TypeCheck(obj, &cursorType)) { + conn = ((cursorObject*)obj)->conn; + } + else if (PyObject_TypeCheck(obj, &connectionType)) { + conn = (connectionObject*)obj; + } + else { + PyErr_SetString(PyExc_TypeError, + "argument 3 must be a connection or a cursor"); + return NULL; + } + } + + /* for ensure_bytes */ + Py_INCREF(user); + Py_INCREF(password); + if (algorithm) { + Py_INCREF(algorithm); + } + + if (!(user = psycopg_ensure_bytes(user))) { goto exit; } + if (!(password = psycopg_ensure_bytes(password))) { goto exit; } + if (algorithm && !(algorithm = psycopg_ensure_bytes(algorithm))) { + goto exit; + } + + /* Use the libpq API 'PQencryptPassword', when no connection object is + available, or the algorithm is set to as 'md5', or the database server + version < 10 */ + if (conn == NULL || conn->server_version < 100000 || + (algorithm != NULL && algorithm != Py_None && + strcmp(Bytes_AS_STRING(algorithm), "md5") == 0)) { + encrypted = PQencryptPassword(Bytes_AS_STRING(password), + Bytes_AS_STRING(user)); + + if (encrypted != NULL) { + res = Text_FromUTF8(encrypted); + PQfreemem(encrypted); + } + goto exit; + } + +#if PG_VERSION_NUM >= 100000 + encrypted = PQencryptPasswordConn(conn->pgconn, Bytes_AS_STRING(password), + Bytes_AS_STRING(user), + algorithm ? Bytes_AS_STRING(algorithm) : NULL); + + if (!encrypted) { + const char *msg = PQerrorMessage(conn->pgconn); + if (msg && *msg) { + PyErr_Format(ProgrammingError, msg); + res = NULL; + goto exit; + } + } + else { + res = Text_FromUTF8(encrypted); + PQfreemem(encrypted); + } +#else + PyErr_SetString( + NotSupportedError, + "Password encryption (other than 'md5' algorithm) is not supported for the server version >= 10 in libpq < 10" + ); + res = NULL; +#endif + +exit: + Py_XDECREF(user); + Py_XDECREF(password); + if (algorithm) { + Py_XDECREF(algorithm); + } + + return res; +} + + /* psyco_encodings_fill Fill the module's postgresql<->python encoding table */ @@ -852,6 +955,8 @@ static PyMethodDef psycopgMethods[] = { METH_O, psyco_set_wait_callback_doc}, {"get_wait_callback", (PyCFunction)psyco_get_wait_callback, METH_NOARGS, psyco_get_wait_callback_doc}, + {"encrypt_password", (PyCFunction)psyco_encrypt_password, + METH_VARARGS, psyco_encrypt_password_doc}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/tests/test_connection.py b/tests/test_connection.py index cc9330b2..8cc2db5f 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1392,17 +1392,20 @@ class TransactionControlTests(ConnectingTestCase): # MD5 algorithm self.assertEqual( - self.conn.encrypt_password('psycopg2', 'ashesh', 'md5'), + ext.encrypt_password('psycopg2', 'ashesh', self.conn, 'md5'), 'md594839d658c28a357126f105b9cb14cfc' ) if libpq_version() < 100000: self.assertRaises( psycopg2.NotSupportedError, - self.conn.encrypt_password, 'psycopg2', 'ashesh' + ext.encrypt_password, 'psycopg2', 'ashesh', self.conn, + 'scram-sha-256' ) else: - enc_password = self.conn.encrypt_password('psycopg2', 'ashesh') + enc_password = ext.encrypt_password( + 'psycopg2', 'ashesh', self.conn + ) if server_encryption_algorithm == 'md5': self.assertEqual( enc_password, 'md594839d658c28a357126f105b9cb14cfc' @@ -1411,22 +1414,22 @@ class TransactionControlTests(ConnectingTestCase): self.assertEqual(enc_password[:14], 'SCRAM-SHA-256$') self.assertEqual( - self.conn.encrypt_password( - 'psycopg2', 'ashesh', 'scram-sha-256' + ext.encrypt_password( + 'psycopg2', 'ashesh', self.conn, 'scram-sha-256' )[:14], 'SCRAM-SHA-256$' ) @skip_after_postgres(10) def test_encrypt_password_pre_10(self): self.assertEqual( - self.conn.encrypt_password('psycopg2', 'ashesh'), + ext.encrypt_password('psycopg2', 'ashesh', self.conn), 'md594839d658c28a357126f105b9cb14cfc' ) # Encryption algorithm will be ignored for postgres version < 10, it # will always use MD5. self.assertEqual( - self.conn.encrypt_password('psycopg2', 'ashesh', 'abc'), + ext.encrypt_password('psycopg2', 'ashesh', self.conn, 'abc'), 'md594839d658c28a357126f105b9cb14cfc' ) From 852884e086706625a69247f2ddafd1a1465b138c Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 12 Dec 2017 03:06:18 -0800 Subject: [PATCH 06/55] use accessor macros for pypy3 compatibility --- psycopg/adapter_datetime.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 9d04df40..a78311ee 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -100,7 +100,7 @@ _pydatetime_string_delta(pydatetimeObject *self) char buffer[8]; int i; - int a = obj->microseconds; + int a = PyDateTime_DELTA_GET_MICROSECONDS(obj); for (i=0; i < 6 ; i++) { buffer[5-i] = '0' + (a % 10); @@ -109,7 +109,9 @@ _pydatetime_string_delta(pydatetimeObject *self) buffer[6] = '\0'; return Bytes_FromFormat("'%d days %d.%s seconds'::interval", - obj->days, obj->seconds, buffer); + PyDateTime_DELTA_GET_DAYS(obj), + PyDateTime_DELTA_GET_SECONDS(obj), + buffer); } static PyObject * From 3fcb0351265e3077ee4088ba6e90016e5735a2a1 Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 12 Dec 2017 03:12:54 -0800 Subject: [PATCH 07/55] define a "polyfill" inline for python 2 compatibility --- psycopg/adapter_datetime.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index a78311ee..b5135efb 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -93,6 +93,12 @@ error: return rv; } +#ifndef PyDateTime_DELTA_GET_DAYS +#define PyDateTime_DELTA_GET_DAYS(o) (o->days) +#define PyDateTime_DELTA_GET_SECONDS(o) (o->seconds) +#define PyDateTime_DELTA_GET_MICROSECONDS(o) (o->microseconds) +#endif + static PyObject * _pydatetime_string_delta(pydatetimeObject *self) { From b8f2f718190ce5ab73ee8523a18df1ccbe68ad0c Mon Sep 17 00:00:00 2001 From: Ronan Amicel Date: Thu, 14 Dec 2017 14:00:36 +0100 Subject: [PATCH 08/55] Fix typo in comment in utils.c --- psycopg/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psycopg/utils.c b/psycopg/utils.c index 7073504f..a197fe22 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -282,7 +282,7 @@ exit: /* Make a connection string out of a string and a dictionary of arguments. * - * Helper to call psycopg2.extensions.make_dns() + * Helper to call psycopg2.extensions.make_dsn() */ PyObject * psycopg_make_dsn(PyObject *dsn, PyObject *kwargs) From 77c703395befce2cc87a18a5ec706f5500f04f04 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Jan 2018 22:24:05 +0000 Subject: [PATCH 09/55] Moved datatime compatibility macros with others --- psycopg/adapter_datetime.c | 6 ------ psycopg/python.h | 5 +++++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index b5135efb..a78311ee 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -93,12 +93,6 @@ error: return rv; } -#ifndef PyDateTime_DELTA_GET_DAYS -#define PyDateTime_DELTA_GET_DAYS(o) (o->days) -#define PyDateTime_DELTA_GET_SECONDS(o) (o->seconds) -#define PyDateTime_DELTA_GET_MICROSECONDS(o) (o->microseconds) -#endif - static PyObject * _pydatetime_string_delta(pydatetimeObject *self) { diff --git a/psycopg/python.h b/psycopg/python.h index fc8c2fed..fa894bf3 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -87,6 +87,7 @@ typedef unsigned long Py_uhash_t; #ifndef PyNumber_Int #define PyNumber_Int PyNumber_Long #endif + #endif /* PY_MAJOR_VERSION > 2 */ #if PY_MAJOR_VERSION < 3 @@ -104,6 +105,10 @@ typedef unsigned long Py_uhash_t; #define Bytes_ConcatAndDel PyString_ConcatAndDel #define _Bytes_Resize _PyString_Resize +#define PyDateTime_DELTA_GET_DAYS(o) (((PyDateTime_Delta*)o)->days) +#define PyDateTime_DELTA_GET_SECONDS(o) (((PyDateTime_Delta*)o)->seconds) +#define PyDateTime_DELTA_GET_MICROSECONDS(o) (((PyDateTime_Delta*)o)->microseconds) + #else #define Bytes_Type PyBytes_Type From b3a70e09e960fd8d66e0f6a43962f7bf5cab2ba0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Jan 2018 22:44:37 +0000 Subject: [PATCH 10/55] pypi3 fix noted in the news file --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index a1085252..1f1d7c85 100644 --- a/NEWS +++ b/NEWS @@ -24,6 +24,7 @@ What's new in psycopg 2.7.4 (:ticket:`632`). - Fixed `~cursor.rowcount` after `~cursor.executemany()` with :sql:`RETURNING` statements (:ticket:`633`). +- Fixed compatibility problem with pypy3 (:ticket:`#649`). - Wheel packages compiled against PostgreSQL 10.1 libpq and OpenSSL 1.0.2m. From 04f1f06b9f1f627436c1202aaa45434334c8747d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Jan 2018 22:49:00 +0000 Subject: [PATCH 11/55] Fixed stitch_text on Python 2 --- doc/src/tools/stitch_text.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/src/tools/stitch_text.py b/doc/src/tools/stitch_text.py index dca745bd..e026622f 100755 --- a/doc/src/tools/stitch_text.py +++ b/doc/src/tools/stitch_text.py @@ -5,6 +5,7 @@ import os import sys + def main(): if len(sys.argv) != 3: sys.stderr.write("usage: %s index.rst text-dir\n") @@ -17,20 +18,20 @@ def main(): return 0 + def iter_file_base(fn): f = open(fn) - have_line = iter(f).__next__ - while not have_line().startswith('.. toctree'): + while not next(f).startswith('.. toctree'): pass - while have_line().strip().startswith(':'): + while next(f).strip().startswith(':'): pass yield os.path.splitext(os.path.basename(fn))[0] n = 0 while True: - line = have_line() + line = next(f) if line.isspace(): continue if line.startswith(".."): @@ -44,6 +45,7 @@ def iter_file_base(fn): # maybe format changed? raise Exception("Not enough files found. Format change in index.rst?") + def emit(basename, txt_dir): f = open(os.path.join(txt_dir, basename + ".txt")) for line in f: From e0226fc46ad21f10ca612b32659bab442d2db871 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Jan 2018 23:06:31 +0000 Subject: [PATCH 12/55] 'cursor.mogrify()' can be called on closed cursors Fix #579. --- NEWS | 1 + psycopg/cursor_type.c | 2 -- tests/test_cursor.py | 6 ++++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 1f1d7c85..3bc13053 100644 --- a/NEWS +++ b/NEWS @@ -18,6 +18,7 @@ What's new in psycopg 2.7.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Fixed Solaris 10 support (:ticket:`#532`). +- `cursor.mogrify()` can be called on closed cursors (:ticket:`#579`). - Fixed `~psycopg2.extras.MinTimeLoggingCursor` on Python 3 (:ticket:`#609`). - Fixed parsing of array of points as floats (:ticket:`#613`). - Fixed `~psycopg2.__libpq_version__` building with libpq >= 10.1 diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index a70e9d34..b7fd1870 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -592,8 +592,6 @@ psyco_curs_mogrify(cursorObject *self, PyObject *args, PyObject *kwargs) return NULL; } - EXC_IF_CURS_CLOSED(self); - return _psyco_curs_mogrify(self, operation, vars); } diff --git a/tests/test_cursor.py b/tests/test_cursor.py index ec76918b..cc8db0f4 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -118,6 +118,12 @@ class CursorTests(ConnectingTestCase): nref2 = sys.getrefcount(foo) self.assertEqual(nref1, nref2) + def test_modify_closed(self): + cur = self.conn.cursor() + cur.close() + sql = cur.mogrify("select %s", (10,)) + self.assertEqual(sql, b"select 10") + def test_bad_placeholder(self): cur = self.conn.cursor() self.assertRaises(psycopg2.ProgrammingError, From a78ac3c1253717f1f0f8f87f0c868e54f42af440 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Jan 2018 08:52:03 +0900 Subject: [PATCH 13/55] Fixed test in asian time zones Fix #652 --- tests/test_dates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_dates.py b/tests/test_dates.py index 47ef41cb..74dfc9ab 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -639,7 +639,8 @@ class FromTicksTestCase(unittest.TestCase): def test_date_value_error_sec_59_99(self): from datetime import date s = psycopg2.DateFromTicks(1273173119.99992) - self.assertEqual(s.adapted, date(2010, 5, 6)) + # The returned date is local + self.assert_(s.adapted in [date(2010, 5, 6), date(2010, 5, 7)]) def test_time_value_error_sec_59_99(self): from datetime import time From ddef2e30cd080c21f4a52a757c642069ceb935c5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Jan 2018 00:14:36 +0000 Subject: [PATCH 14/55] 'key' docs in getconn() improved Fix #569. --- doc/src/pool.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/src/pool.rst b/doc/src/pool.rst index 7624dc88..38cd08fa 100644 --- a/doc/src/pool.rst +++ b/doc/src/pool.rst @@ -24,13 +24,18 @@ directly in the client application. .. method:: getconn(key=None) - Get a free connection and assign it to *key* if not `!None`. + Get a free connection from the pool. + + The *key* parameter is optional: if used, the connection will be + associated to the key and calling `!getconn()` with the same key again + will return the same connection. .. method:: putconn(conn, key=None, close=False) Put away a connection. If *close* is `!True`, discard the connection from the pool. + *key* should be used consistently with `getconn()`. .. method:: closeall From 74d2c4bef934b855726961472ba662141d2e88c9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Jan 2018 01:59:49 +0000 Subject: [PATCH 15/55] Fixed idempotence check changing connection characteristics --- NEWS | 2 ++ psycopg/connection_int.c | 31 ++++++++++++++++++++++--------- psycopg/connection_type.c | 27 ++++++++++++++------------- tests/test_connection.py | 10 ++++++++++ 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/NEWS b/NEWS index 3bc13053..0f108627 100644 --- a/NEWS +++ b/NEWS @@ -19,6 +19,8 @@ What's new in psycopg 2.7.4 - Fixed Solaris 10 support (:ticket:`#532`). - `cursor.mogrify()` can be called on closed cursors (:ticket:`#579`). +- Fixed setting session characteristics in corner cases on autocommit + connections (:ticket:`#580`). - Fixed `~psycopg2.extras.MinTimeLoggingCursor` on Python 3 (:ticket:`#609`). - Fixed parsing of array of points as floats (:ticket:`#613`). - Fixed `~psycopg2.__libpq_version__` building with libpq >= 10.1 diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index e8081b9e..3ea5ca32 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -67,6 +67,9 @@ const char *srv_state_guc[] = { }; +const int SRV_STATE_UNCHANGED = -1; + + /* Return a new "string" from a char* from the database. * * On Py2 just get a string, on Py3 decode it in the connection codec. @@ -1188,6 +1191,8 @@ conn_set_session(connectionObject *self, int autocommit, int rv = -1; PGresult *pgres = NULL; char *error = NULL; + int want_autocommit = autocommit == SRV_STATE_UNCHANGED ? + self->autocommit : autocommit; if (deferrable != self->deferrable && self->server_version < 90100) { PyErr_SetString(ProgrammingError, @@ -1209,24 +1214,24 @@ conn_set_session(connectionObject *self, int autocommit, Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); - if (autocommit) { - /* we are in autocommit state, so no BEGIN will be issued: + if (want_autocommit) { + /* we are or are going in autocommit state, so no BEGIN will be issued: * configure the session with the characteristics requested */ - if (isolevel != self->isolevel) { + if (isolevel != SRV_STATE_UNCHANGED) { if (0 > pq_set_guc_locked(self, "default_transaction_isolation", srv_isolevels[isolevel], &pgres, &error, &_save)) { goto endlock; } } - if (readonly != self->readonly) { + if (readonly != SRV_STATE_UNCHANGED) { if (0 > pq_set_guc_locked(self, "default_transaction_read_only", srv_state_guc[readonly], &pgres, &error, &_save)) { goto endlock; } } - if (deferrable != self->deferrable) { + if (deferrable != SRV_STATE_UNCHANGED) { if (0 > pq_set_guc_locked(self, "default_transaction_deferrable", srv_state_guc[deferrable], &pgres, &error, &_save)) { @@ -1260,10 +1265,18 @@ conn_set_session(connectionObject *self, int autocommit, } } - self->autocommit = autocommit; - self->isolevel = isolevel; - self->readonly = readonly; - self->deferrable = deferrable; + if (autocommit != SRV_STATE_UNCHANGED) { + self->autocommit = autocommit; + } + if (isolevel != SRV_STATE_UNCHANGED) { + self->isolevel = isolevel; + } + if (readonly != SRV_STATE_UNCHANGED) { + self->readonly = readonly; + } + if (deferrable != SRV_STATE_UNCHANGED) { + self->deferrable = deferrable; + } rv = 0; endlock: diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 8c5085b5..6a66d48d 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -39,6 +39,7 @@ extern HIDDEN const char *srv_isolevels[]; extern HIDDEN const char *srv_readonly[]; extern HIDDEN const char *srv_deferrable[]; +extern HIDDEN const int SRV_STATE_UNCHANGED; /** DBAPI methods **/ @@ -561,10 +562,10 @@ psyco_conn_set_session(connectionObject *self, PyObject *args, PyObject *kwargs) PyObject *deferrable = Py_None; PyObject *autocommit = Py_None; - int c_isolevel = self->isolevel; - int c_readonly = self->readonly; - int c_deferrable = self->deferrable; - int c_autocommit = self->autocommit; + int c_isolevel = SRV_STATE_UNCHANGED; + int c_readonly = SRV_STATE_UNCHANGED; + int c_deferrable = SRV_STATE_UNCHANGED; + int c_autocommit = SRV_STATE_UNCHANGED; static char *kwlist[] = {"isolation_level", "readonly", "deferrable", "autocommit", NULL}; @@ -637,7 +638,7 @@ psyco_conn_autocommit_set(connectionObject *self, PyObject *pyvalue) if (!_psyco_set_session_check_setter_wrapper(self)) { return -1; } if (-1 == (value = PyObject_IsTrue(pyvalue))) { return -1; } if (0 > conn_set_session(self, value, - self->isolevel, self->readonly, self->deferrable)) { + SRV_STATE_UNCHANGED, SRV_STATE_UNCHANGED, SRV_STATE_UNCHANGED)) { return -1; } @@ -668,8 +669,8 @@ psyco_conn_isolation_level_set(connectionObject *self, PyObject *pyvalue) if (!_psyco_set_session_check_setter_wrapper(self)) { return -1; } if (0 > (value = _psyco_conn_parse_isolevel(pyvalue))) { return -1; } - if (0 > conn_set_session(self, self->autocommit, - value, self->readonly, self->deferrable)) { + if (0 > conn_set_session(self, SRV_STATE_UNCHANGED, + value, SRV_STATE_UNCHANGED, SRV_STATE_UNCHANGED)) { return -1; } @@ -715,13 +716,13 @@ psyco_conn_set_isolation_level(connectionObject *self, PyObject *args) if (level == 0) { if (0 > conn_set_session(self, 1, - self->isolevel, self->readonly, self->deferrable)) { + SRV_STATE_UNCHANGED, SRV_STATE_UNCHANGED, SRV_STATE_UNCHANGED)) { return NULL; } } else { if (0 > conn_set_session(self, 0, - level, self->readonly, self->deferrable)) { + level, SRV_STATE_UNCHANGED, SRV_STATE_UNCHANGED)) { return NULL; } } @@ -767,8 +768,8 @@ psyco_conn_readonly_set(connectionObject *self, PyObject *pyvalue) if (!_psyco_set_session_check_setter_wrapper(self)) { return -1; } if (0 > (value = _psyco_conn_parse_onoff(pyvalue))) { return -1; } - if (0 > conn_set_session(self, self->autocommit, - self->isolevel, value, self->deferrable)) { + if (0 > conn_set_session(self, SRV_STATE_UNCHANGED, + SRV_STATE_UNCHANGED, value, SRV_STATE_UNCHANGED)) { return -1; } @@ -813,8 +814,8 @@ psyco_conn_deferrable_set(connectionObject *self, PyObject *pyvalue) if (!_psyco_set_session_check_setter_wrapper(self)) { return -1; } if (0 > (value = _psyco_conn_parse_onoff(pyvalue))) { return -1; } - if (0 > conn_set_session(self, self->autocommit, - self->isolevel, self->readonly, value)) { + if (0 > conn_set_session(self, SRV_STATE_UNCHANGED, + SRV_STATE_UNCHANGED, SRV_STATE_UNCHANGED, value)) { return -1; } diff --git a/tests/test_connection.py b/tests/test_connection.py index 02db2645..fa9bfc01 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1381,6 +1381,16 @@ class TransactionControlTests(ConnectingTestCase): cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'off') + def test_idempotence_check(self): + self.conn.autocommit = False + self.conn.readonly = True + self.conn.autocommit = True + self.conn.readonly = True + + cur = self.conn.cursor() + cur.execute("SHOW transaction_read_only") + self.assertEqual(cur.fetchone()[0], 'on') + class AutocommitTests(ConnectingTestCase): def test_closed(self): From 8decf34ad7bc535b063184aec5ba530fd1bd34f0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 25 Jan 2018 21:58:05 +0000 Subject: [PATCH 16/55] Dropped warning about unsafe cursor names It was long made secure --- doc/src/connection.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 6f13ecd3..2910f301 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -41,11 +41,6 @@ The ``connection`` class previously only valid PostgreSQL identifiers were accepted as cursor name. - .. warning:: - It is unsafe to expose the *name* to an untrusted source, for - instance you shouldn't allow *name* to be read from a HTML form. - Consider it as part of the query, not as a query parameter. - The *cursor_factory* argument can be used to create non-standard cursors. The class returned must be a subclass of `psycopg2.extensions.cursor`. See :ref:`subclassing-cursor` for From aa2c1727066cc31d1b4431bc2e80abb2edcfb381 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 13 Jan 2018 20:11:18 +0000 Subject: [PATCH 17/55] Print info about the binary package on build failed The idea is to release a package 'psycopg2-binary' to allow installing binary, and leave the psycopg2 package to be source only, to avoid pushing the unreliability of the wheel pacakge by default (see issue #543). Version number bumped to test with new packages. --- setup.py | 46 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 0e214629..ec2b5ae0 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ except ImportError: from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc from distutils.ccompiler import get_default_compiler +from distutils.errors import CompileError from distutils.util import get_platform try: @@ -104,15 +105,23 @@ class PostgresConfig: if not self.pg_config_exe: self.pg_config_exe = self.autodetect_pg_config_path() if self.pg_config_exe is None: - sys.stderr.write("""\ + sys.stderr.write(""" Error: pg_config executable not found. -Please add the directory containing pg_config to the PATH -or specify the full executable path with the option: +pg_config is required to build psycopg2 from source. Please add the directory +containing pg_config to the $PATH or specify the full executable path with the +option: python setup.py build_ext --pg-config /path/to/pg_config build ... or with the pg_config option in 'setup.cfg'. + +If you prefer to avoid building psycopg2 from source, please install the PyPI +'psycopg2-binary' package instead. + +For further information please check the 'doc/src/install.rst' file (also at +). + """) sys.exit(1) @@ -286,8 +295,37 @@ class psycopg_build_ext(build_ext): else: return build_ext.get_export_symbols(self, extension) + built_files = 0 + def build_extension(self, extension): - build_ext.build_extension(self, extension) + # Count files compiled to print the binary blurb only if the first fails + compile_orig = getattr(self.compiler, '_compile', None) + if compile_orig is not None: + def _compile(*args, **kwargs): + rv = compile_orig(*args, **kwargs) + psycopg_build_ext.built_files += 1 + return rv + + self.compiler._compile = _compile + + try: + build_ext.build_extension(self, extension) + psycopg_build_ext.built_files += 1 + except CompileError: + if self.built_files == 0: + sys.stderr.write(""" +It appears you are missing some prerequisite to build the package from source. + +You may install a binary package by installing 'psycopg2-binary' from PyPI. +If you want to install psycopg2 from source, please install the packages +required for the build and try again. + +For further information please check the 'doc/src/install.rst' file (also at +). + +""") + raise + sysVer = sys.version_info[:2] # For Python versions that use MSVC compiler 2008, re-insert the From 5b69adf797ecb7241d2536140ac173a2404905a6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 16 Jan 2018 18:28:27 +0000 Subject: [PATCH 18/55] Document the psycopg2-binary package --- README.rst | 32 ++++--- doc/src/install.rst | 198 ++++++++++++++++++++++---------------------- 2 files changed, 121 insertions(+), 109 deletions(-) diff --git a/README.rst b/README.rst index 69b1d83d..9b88239a 100644 --- a/README.rst +++ b/README.rst @@ -25,29 +25,39 @@ Documentation is included in the ``doc`` directory and is `available online`__. .. __: http://initd.org/psycopg/docs/ +For any other resource (source code repository, bug tracker, mailing list) +please check the `project homepage`__. + Installation ------------ -If your ``pip`` version supports wheel_ packages it should be possible to -install a binary version of Psycopg including all the dependencies from PyPI_. -Just run:: +Building Psycopg requires a few prerequisites (a C compiler, some development +packages): please check the install_ and the faq_ documents in the ``doc`` dir +or online for the details. + +If prerequisites are met, you can install psycopg like any other Python +package, using ``pip`` to download it from PyPI_:: - $ pip install -U pip # make sure your pip is up-to-date $ pip install psycopg2 -If you want to build Psycopg from source you will need some prerequisites (a C -compiler, development packages): please check the install_ and the faq_ -documents in the ``doc`` dir for the details. +or using ``setup.py`` if you have downloaded the source package locally:: + + $ python setup.py build + $ sudo python setup.py install + +You can also obtain a stand-alone package, not requiring a compiler or +external libraries, by installing the ``psycopg2-binary`` package from PyPI:: + + $ pip install psycopg2-binary + +The binary package is a practical choice for development and testing but in +production it is advised to use the package built from sources. -.. _wheel: http://pythonwheels.com/ .. _PyPI: https://pypi.python.org/pypi/psycopg2 .. _install: http://initd.org/psycopg/docs/install.html#install-from-source .. _faq: http://initd.org/psycopg/docs/faq.html#faq-compile -For any other resource (source code repository, bug tracker, mailing list) -please check the `project homepage`__. - .. __: http://initd.org/psycopg/ diff --git a/doc/src/install.rst b/doc/src/install.rst index 6d30e343..3fcea867 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -12,16 +12,6 @@ to use Psycopg on a different Python implementation (PyPy, Jython, IronPython) there is an experimental `porting of Psycopg for Ctypes`__, but it is not as mature as the C implementation yet. -The current `!psycopg2` implementation supports: - -.. - NOTE: keep consistent with setup.py and the /features/ page. - -- Python version 2.7 -- Python 3 versions from 3.4 to 3.6 -- PostgreSQL server versions from 7.4 to 10 -- PostgreSQL client library version from 9.1 - .. _PostgreSQL: http://www.postgresql.org/ .. _Python: http://www.python.org/ .. _libpq: http://www.postgresql.org/docs/current/static/libpq.html @@ -32,94 +22,20 @@ The current `!psycopg2` implementation supports: .. index:: - single: Install; from PyPI - single: Install; wheel - single: Wheel + single: Prerequisites -Binary install from PyPI ------------------------- +Prerequisites +------------- -`!psycopg2` is `available on PyPI`__ in the form of wheel_ packages for the -most common platform (Linux, OSX, Windows): this should make you able to -install a binary version of the module including all the dependencies simply -using: +The current `!psycopg2` implementation supports: -.. code-block:: console +.. + NOTE: keep consistent with setup.py and the /features/ page. - $ pip install psycopg2 - -Make sure to use an up-to-date version of :program:`pip` (you can upgrade it -using something like ``pip install -U pip``) - -.. __: PyPI_ -.. _PyPI: https://pypi.python.org/pypi/psycopg2/ -.. _wheel: http://pythonwheels.com/ - -.. note:: - - The binary packages come with their own versions of a few C libraries, - among which ``libpq`` and ``libssl``, which will be used regardless of other - libraries available on the client: upgrading the system libraries will not - upgrade the libraries used by `!psycopg2`. Please build `!psycopg2` from - source if you want to maintain binary upgradeability. - -.. warning:: - - Because the `!psycopg` wheel package uses its own ``libssl`` binary, it is - incompatible with other extension modules binding with ``libssl`` as well, - for instance the Python `ssl` module: the result will likely be a - segfault. If you need using both `!psycopg2` and other libraries using - ``libssl`` please :ref:`disable the use of wheel packages for Psycopg - `. - - - -.. index:: - single: Install; disable wheel - single: Wheel; disable - -.. _disable-wheel: - -Disabling wheel packages -^^^^^^^^^^^^^^^^^^^^^^^^ - -If you want to disable the use of wheel binary packages and use the system -system libraries available on your client you can use the :command:`pip` -|--no-binary option|__: - -.. code-block:: console - - $ pip install --no-binary psycopg2 - -.. |--no-binary option| replace:: ``--no-binary`` option -.. __: https://pip.pypa.io/en/stable/reference/pip_install/#install-no-binary - -which can be specified in your :file:`requirements.txt` files too, e.g. use: - -.. code-block:: none - - psycopg2>=2.7,<2.8 --no-binary psycopg2 - -to use the last bugfix release of the `!psycopg2` 2.7 package, specifying to -always compile it from source. Of course in this case you will have to meet -the :ref:`build prerequisites `. - - - -.. index:: - single: Install; from source - -.. _install-from-source: - -Install from source -------------------- - -.. _source-package: - -You can download a copy of Psycopg source files from the `Psycopg download -page`__ or from PyPI_. - -.. __: http://initd.org/psycopg/download/ +- Python version 2.7 +- Python 3 versions from 3.4 to 3.6 +- PostgreSQL server versions from 7.4 to 10 +- PostgreSQL client library version from 9.1 @@ -128,8 +44,8 @@ page`__ or from PyPI_. Build prerequisites ^^^^^^^^^^^^^^^^^^^ -These notes illustrate how to compile Psycopg on Linux. If you want to compile -Psycopg on other platforms you may have to adjust some details accordingly. +The build prerequisites are to be met in order to install Psycopg from source +code, either from a source distribution package or from PyPI. Psycopg is a C wrapper around the libpq_ PostgreSQL client library. To install it from sources you will need: @@ -161,6 +77,12 @@ it from sources you will need: Once everything is in place it's just a matter of running the standard: +.. code-block:: console + + $ pip install psycopg2 + +or, from the directory containing the source code: + .. code-block:: console $ python setup.py build @@ -197,12 +119,92 @@ which is OS-dependent (for instance setting a suitable +.. index:: + single: Install; from PyPI + single: Install; wheel + single: Wheel + +Binary install from PyPI +------------------------ + +`!psycopg2` is also `available on PyPI`__ in the form of wheel_ packages for +the most common platform (Linux, OSX, Windows): this should make you able to +install a binary version of the module, not requiring the above build or +runtime prerequisites, simply using: + +.. code-block:: console + + $ pip install psycopg2-binary + +Make sure to use an up-to-date version of :program:`pip` (you can upgrade it +using something like ``pip install -U pip``) + +.. __: PyPI-binary_ +.. _PyPI-binary: https://pypi.python.org/pypi/psycopg2-binary/ +.. _wheel: http://pythonwheels.com/ + +.. note:: + + The binary packages come with their own versions of a few C libraries, + among which ``libpq`` and ``libssl``, which will be used regardless of other + libraries available on the client: upgrading the system libraries will not + upgrade the libraries used by `!psycopg2`. Please build `!psycopg2` from + source if you want to maintain binary upgradeability. + +.. warning:: + + The `!psycopg2` wheel package comes packaged, among the others, with its + own ``libssl`` binary. This may create conflicts with other extension + modules binding with ``libssl`` as well, for instance with the Python + `ssl` module: in some cases, under concurrency, the interaction between + the two libraries may result in a segfault. In case of doubts you are + advised to use a package built from source. + + + +.. index:: + single: Install; disable wheel + single: Wheel; disable + +.. _disable-wheel: + +Disabling wheel packages for Psycopg 2.7 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In version 2.7.x, `pip install psycopg2` would have tried to install the wheel +binary package of Psycopg. Because of the problems the wheel package have +displayed, `psycopg2-binary` has become a separate package, and from 2.8 it +has become the only way to install the binary package. + +If you are using psycopg 2.7 and you want to disable the use of wheel binary +packages, relying on the system system libraries available on your client, you +can use the :command:`pip` |--no-binary option|__: + +.. code-block:: console + + $ pip install --no-binary psycopg2 + +.. |--no-binary option| replace:: ``--no-binary`` option +.. __: https://pip.pypa.io/en/stable/reference/pip_install/#install-no-binary + +which can be specified in your :file:`requirements.txt` files too, e.g. use: + +.. code-block:: none + + psycopg2>=2.7,<2.8 --no-binary psycopg2 + +to use the last bugfix release of the `!psycopg2` 2.7 package, specifying to +always compile it from source. Of course in this case you will have to meet +the :ref:`build prerequisites `. + + + .. index:: single: setup.py single: setup.cfg Non-standard builds -^^^^^^^^^^^^^^^^^^^ +------------------- If you have less standard requirements such as: @@ -242,7 +244,7 @@ order to create a debug package: - Edit the ``setup.cfg`` file adding the ``PSYCOPG_DEBUG`` flag to the ``define`` option. -- :ref:`Compile and install ` the package. +- :ref:`Compile and install ` the package. - Set the :envvar:`PSYCOPG_DEBUG` environment variable: From 76f3e196d31a3e2006b9c0b18c400fb9f89b9117 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 00:45:40 +0000 Subject: [PATCH 19/55] Silence warning on import failing a test --- tests/test_connection.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index fa9bfc01..a82bd499 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1549,9 +1549,13 @@ import os import sys import time import signal +import warnings import threading -import psycopg2 +# ignore wheel deprecation warning +with warnings.catch_warnings(): + warnings.simplefilter('ignore') + import psycopg2 def handle_sigabort(sig, frame): sys.exit(1) From f976c428d22a1b53cb523e5f01f6f9117834c87e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 01:33:37 +0000 Subject: [PATCH 20/55] Mention new wheel packages in news file Close #543. --- NEWS | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 0f108627..5095dc66 100644 --- a/NEWS +++ b/NEWS @@ -17,6 +17,9 @@ Other changes: What's new in psycopg 2.7.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Moving away from installing the wheel package by default. + Packages installed from wheel raise a warning on import. Added package + ``psycopg2-binary`` to install from wheel instead (:ticket:`#543`). - Fixed Solaris 10 support (:ticket:`#532`). - `cursor.mogrify()` can be called on closed cursors (:ticket:`#579`). - Fixed setting session characteristics in corner cases on autocommit @@ -25,8 +28,8 @@ What's new in psycopg 2.7.4 - Fixed parsing of array of points as floats (:ticket:`#613`). - Fixed `~psycopg2.__libpq_version__` building with libpq >= 10.1 (:ticket:`632`). -- Fixed `~cursor.rowcount` after `~cursor.executemany()` with :sql:`RETURNING` statements - (:ticket:`633`). +- Fixed `~cursor.rowcount` after `~cursor.executemany()` with :sql:`RETURNING` + statements (:ticket:`633`). - Fixed compatibility problem with pypy3 (:ticket:`#649`). - Wheel packages compiled against PostgreSQL 10.1 libpq and OpenSSL 1.0.2m. From 4845393c15692fa582a1110b2c97a80640ad1187 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 01:56:10 +0000 Subject: [PATCH 21/55] Build env vars on windows less verbose Copied from psycopg2-wheels --- .appveyor.yml | 62 ++++++++++++++++++++------------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index a49c802f..754d7656 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,43 +13,14 @@ environment: matrix: # For Python versions available on Appveyor, see # http://www.appveyor.com/docs/installed-software#python - - # Py 2.7 = VS Ver. 9.0 (VS 2008) - # Py 3.4 = VS Ver. 10.0 (VS 2010) - # Py 3.5, 3.6 = VS Ver. 14.0 (VS 2015) - - - PYTHON: C:\Python27-x64 - PYTHON_ARCH: 64 - VS_VER: 9.0 - - - PYTHON: C:\Python27 - PYTHON_ARCH: 32 - VS_VER: 9.0 - - - PYTHON: C:\Python36-x64 - PYTHON_ARCH: 64 - VS_VER: 14.0 - - - PYTHON: C:\Python36 - PYTHON_ARCH: 32 - VS_VER: 14.0 - - - PYTHON: C:\Python35-x64 - PYTHON_ARCH: 64 - VS_VER: 14.0 - - - PYTHON: C:\Python35 - PYTHON_ARCH: 32 - VS_VER: 14.0 - - - PYTHON: C:\Python34-x64 - DISTUTILS_USE_SDK: '1' - PYTHON_ARCH: 64 - VS_VER: 10.0 - - - PYTHON: C:\Python34 - PYTHON_ARCH: 32 - VS_VER: 10.0 + - {PYVER: "27", PYTHON_ARCH: "32"} + - {PYVER: "27", PYTHON_ARCH: "64"} + - {PYVER: "34", PYTHON_ARCH: "32"} + - {PYVER: "34", PYTHON_ARCH: "64"} + - {PYVER: "35", PYTHON_ARCH: "32"} + - {PYVER: "35", PYTHON_ARCH: "64"} + - {PYVER: "36", PYTHON_ARCH: "32"} + - {PYVER: "36", PYTHON_ARCH: "64"} PSYCOPG2_TESTDB: psycopg2_test PSYCOPG2_TESTDB_USER: postgres @@ -77,7 +48,22 @@ cache: init: # Uncomment next line to get RDP access during the build. #- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - # + + # Set env variable according to the build environment + - SET PYTHON=C:\Python%PYVER% + - IF "%PYTHON_ARCH%"=="64" SET PYTHON=%PYTHON%-x64 + + # Py 2.7 = VS Ver. 9.0 (VS 2008) + # Py 3.3, 3.4 = VS Ver. 10.0 (VS 2010) + # Py 3.5, 3.6 = VS Ver. 14.0 (VS 2015) + - IF "%PYVER%"=="27" SET VS_VER=9.0 + - IF "%PYVER%"=="33" SET VS_VER=10.0 + - IF "%PYVER%"=="34" SET VS_VER=10.0 + - IF "%PYVER%"=="35" SET VS_VER=14.0 + - IF "%PYVER%"=="36" SET VS_VER=14.0 + + - IF "%VS_VER%"=="10.0" IF "%PYTHON_ARCH%"=="64" SET DISTUTILS_USE_SDK=1 + # Set Python to the path - SET PATH=%PYTHON%;%PYTHON%\Scripts;C:\Program Files\Git\mingw64\bin;%PATH% From 8cc0d06e65ae9581b6317a18cf695eb853589c14 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 01:57:17 +0000 Subject: [PATCH 22/55] Define openssl and libpq versions in vars in appveyor build --- .appveyor.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 754d7656..3efac773 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,6 +22,9 @@ environment: - {PYVER: "36", PYTHON_ARCH: "32"} - {PYVER: "36", PYTHON_ARCH: "64"} + OPENSSL_VERSION: "1_0_2m" + POSTGRES_VERSION: "10_1" + PSYCOPG2_TESTDB: psycopg2_test PSYCOPG2_TESTDB_USER: postgres PSYCOPG2_TESTDB_PASSWORD: Password12! @@ -139,8 +142,8 @@ install: } # Download OpenSSL source - CD C:\Others - - IF NOT EXIST OpenSSL_1_0_2m.zip ( - curl -fsSL -o OpenSSL_1_0_2m.zip https://github.com/openssl/openssl/archive/OpenSSL_1_0_2m.zip + - IF NOT EXIST OpenSSL_%OPENSSL_VERSION%.zip ( + curl -fsSL -o OpenSSL_%OPENSSL_VERSION%.zip https://github.com/openssl/openssl/archive/OpenSSL_%OPENSSL_VERSION%.zip ) # To use OpenSSL >= 1.1.0, both libpq and psycopg build environments have @@ -152,15 +155,15 @@ install: # - nmake build_libs install_dev - IF NOT EXIST %OPENSSLTOP%\lib\ssleay32.lib ( CD %BUILD_DIR% && - 7z x C:\Others\OpenSSL_1_0_2m.zip && - CD openssl-OpenSSL_1_0_2m && + 7z x C:\Others\OpenSSL_%OPENSSL_VERSION%.zip && + CD openssl-OpenSSL_%OPENSSL_VERSION% && perl Configure %TARGET% no-asm no-shared no-zlib --prefix=%OPENSSLTOP% --openssldir=%OPENSSLTOP% && CALL ms\%DO% && nmake -f ms\nt.mak init headers lib && COPY inc32\openssl\*.h %OPENSSLTOP%\include\openssl && COPY out32\*.lib %OPENSSLTOP%\lib && CD %BASE_DIR% && - RMDIR /S /Q %BUILD_DIR%\openssl-OpenSSL_1_0_2m + RMDIR /S /Q %BUILD_DIR%\openssl-OpenSSL_%OPENSSL_VERSION% ) # Setup directories for building PostgreSQL librarires @@ -174,8 +177,8 @@ install: # Download PostgreSQL source - CD C:\Others - - IF NOT EXIST postgres-REL_10_1.zip ( - curl -fsSL -o postgres-REL_10_1.zip https://github.com/postgres/postgres/archive/REL_10_1.zip + - IF NOT EXIST postgres-REL_%POSTGRES_VERSION%.zip ( + curl -fsSL -o postgres-REL_%POSTGRES_VERSION%.zip https://github.com/postgres/postgres/archive/REL_%POSTGRES_VERSION%.zip ) # Setup build config file (config.pl) @@ -186,11 +189,11 @@ install: # Prepare local include directory for building from # Build pg_config in place # NOTE: Cannot set and use the same variable inside an IF - - SET PGBUILD=%BUILD_DIR%\postgres-REL_10_1 + - SET PGBUILD=%BUILD_DIR%\postgres-REL_%POSTGRES_VERSION% - IF NOT EXIST %PGTOP%\lib\libpq.lib ( CD %BUILD_DIR% && - 7z x C:\Others\postgres-REL_10_1.zip && - CD postgres-REL_10_1\src\tools\msvc && + 7z x C:\Others\postgres-REL_%POSTGRES_VERSION%.zip && + CD postgres-REL_%POSTGRES_VERSION%\src\tools\msvc && ECHO $config-^>{ldap} = 0; > config.pl && ECHO $config-^>{openssl} = "%OPENSSLTOP:\=\\%"; >> config.pl && ECHO.>> config.pl && From 957fd79a272d718be8f23824e04409aa1fb4ee1e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 01:58:37 +0000 Subject: [PATCH 23/55] Build using OpenSSL 1.0.2n --- .appveyor.yml | 2 +- NEWS | 2 +- scripts/appveyor.cache_rebuild | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 3efac773..1761e99f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -22,7 +22,7 @@ environment: - {PYVER: "36", PYTHON_ARCH: "32"} - {PYVER: "36", PYTHON_ARCH: "64"} - OPENSSL_VERSION: "1_0_2m" + OPENSSL_VERSION: "1_0_2n" POSTGRES_VERSION: "10_1" PSYCOPG2_TESTDB: psycopg2_test diff --git a/NEWS b/NEWS index 5095dc66..c4d65011 100644 --- a/NEWS +++ b/NEWS @@ -31,7 +31,7 @@ What's new in psycopg 2.7.4 - Fixed `~cursor.rowcount` after `~cursor.executemany()` with :sql:`RETURNING` statements (:ticket:`633`). - Fixed compatibility problem with pypy3 (:ticket:`#649`). -- Wheel packages compiled against PostgreSQL 10.1 libpq and OpenSSL 1.0.2m. +- Wheel packages compiled against PostgreSQL 10.1 libpq and OpenSSL 1.0.2n. What's new in psycopg 2.7.3.2 diff --git a/scripts/appveyor.cache_rebuild b/scripts/appveyor.cache_rebuild index da1b2be5..04825149 100644 --- a/scripts/appveyor.cache_rebuild +++ b/scripts/appveyor.cache_rebuild @@ -9,7 +9,7 @@ To invalidate the cache, update this file and check it into git. Currently used modules built in the cache: OpenSSL - Version: 1.0.2m + Version: 1.0.2n PostgreSQL Version: 10.1 From ddb87b7727793ef1d49f3915ad4b2399aeb6ef78 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 02:41:44 +0000 Subject: [PATCH 24/55] Convert fields names into valid Python identifiers in NamedTupleCursor Close #211. --- NEWS | 2 ++ lib/extras.py | 12 +++++++++++- tests/test_extras_dictcursor.py | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index c4d65011..efdb4e52 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,8 @@ What's new in psycopg 2.7.4 - Moving away from installing the wheel package by default. Packages installed from wheel raise a warning on import. Added package ``psycopg2-binary`` to install from wheel instead (:ticket:`#543`). +- Convert fields names into valid Python identifiers in + `~psycopg2.extras.NamedTupleCursor` (:ticket:`#211`). - Fixed Solaris 10 support (:ticket:`#532`). - `cursor.mogrify()` can be called on closed cursors (:ticket:`#579`). - Fixed setting session characteristics in corner cases on autocommit diff --git a/lib/extras.py b/lib/extras.py index 68df344c..1f85d532 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -363,7 +363,17 @@ class NamedTupleCursor(_cursor): return def _make_nt(self): - return namedtuple("Record", [d[0] for d in self.description or ()]) + def f(s): + # NOTE: Python 3 actually allows unicode chars in fields + s = _re.sub('[^a-zA-Z0-9_]', '_', s) + # Python identifier cannot start with numbers, namedtuple fields + # cannot start with underscore. So... + if _re.match('^[0-9_]', s): + s = 'f' + s + + return s + + return namedtuple("Record", [f(d[0]) for d in self.description or ()]) class LoggingConnection(_connection): diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index 75c22773..99bdeee6 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -349,6 +349,14 @@ class NamedTupleCursorTest(ConnectingTestCase): curs.execute("update nttest set s = s") self.assertRaises(psycopg2.ProgrammingError, curs.fetchall) + def test_bad_col_names(self): + curs = self.conn.cursor() + curs.execute('select 1 as "foo.bar_baz", 2 as "?column?", 3 as "3"') + rv = curs.fetchone() + self.assertEqual(rv.foo_bar_baz, 1) + self.assertEqual(rv.f_column_, 2) + self.assertEqual(rv.f3, 3) + def test_minimal_generation(self): # Instrument the class to verify it gets called the minimum number of times. from psycopg2.extras import NamedTupleCursor From 1436ee308cef6650e522f6bb2a30f7fe1c6f96d8 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 03:09:27 +0000 Subject: [PATCH 25/55] Link psycopg2-binary package to PyPI in readme Note: the package doesn't exist yet... --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9b88239a..b5b048d4 100644 --- a/README.rst +++ b/README.rst @@ -47,7 +47,7 @@ or using ``setup.py`` if you have downloaded the source package locally:: $ sudo python setup.py install You can also obtain a stand-alone package, not requiring a compiler or -external libraries, by installing the ``psycopg2-binary`` package from PyPI:: +external libraries, by installing the `psycopg2-binary`_ package from PyPI:: $ pip install psycopg2-binary @@ -55,6 +55,7 @@ The binary package is a practical choice for development and testing but in production it is advised to use the package built from sources. .. _PyPI: https://pypi.python.org/pypi/psycopg2 +.. _psycopg2-binary: https://pypi.python.org/pypi/psycopg2-binary .. _install: http://initd.org/psycopg/docs/install.html#install-from-source .. _faq: http://initd.org/psycopg/docs/faq.html#faq-compile From c52e49a9daa95ae069e7e804e41f6534d5849ba1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 29 Jan 2018 12:55:41 +0000 Subject: [PATCH 26/55] Report Python 2.6 wheels no more available [skip ci] --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index efdb4e52..c400f249 100644 --- a/NEWS +++ b/NEWS @@ -34,6 +34,8 @@ What's new in psycopg 2.7.4 statements (:ticket:`633`). - Fixed compatibility problem with pypy3 (:ticket:`#649`). - Wheel packages compiled against PostgreSQL 10.1 libpq and OpenSSL 1.0.2n. +- Wheel packages for Python 2.6 no more available (support dropped from + wheel building infrastructure). What's new in psycopg 2.7.3.2 From 5309da117dd790aaf9dd74b6083638123a383bfa Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 6 Feb 2018 00:40:20 +0000 Subject: [PATCH 27/55] Autocommit shouldn't change deferrable on servers not supporting it Regression on unsupported Postgres versions after fixing bug #580 --- psycopg/connection_int.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 3ea5ca32..a60c4a9b 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -1194,7 +1194,7 @@ conn_set_session(connectionObject *self, int autocommit, int want_autocommit = autocommit == SRV_STATE_UNCHANGED ? self->autocommit : autocommit; - if (deferrable != self->deferrable && self->server_version < 90100) { + if (deferrable != SRV_STATE_UNCHANGED && self->server_version < 90100) { PyErr_SetString(ProgrammingError, "the 'deferrable' setting is only available" " from PostgreSQL 9.1"); @@ -1256,7 +1256,7 @@ conn_set_session(connectionObject *self, int autocommit, goto endlock; } } - if (self->deferrable != STATE_DEFAULT) { + if (self->server_version >= 90100 && self->deferrable != STATE_DEFAULT) { if (0 > pq_set_guc_locked(self, "default_transaction_deferrable", "default", &pgres, &error, &_save)) { From df952c149da2a3a7574c54a8a62c791211223270 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 9 Feb 2018 16:10:03 +0000 Subject: [PATCH 28/55] Fixed pip invocation example to skip binary packages Close #673 --- doc/src/install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/install.rst b/doc/src/install.rst index 3fcea867..a858cbe3 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -178,11 +178,11 @@ has become the only way to install the binary package. If you are using psycopg 2.7 and you want to disable the use of wheel binary packages, relying on the system system libraries available on your client, you -can use the :command:`pip` |--no-binary option|__: +can use the :command:`pip` |--no-binary option|__, e.g.: .. code-block:: console - $ pip install --no-binary psycopg2 + $ pip install --no-binary :all: psycopg2 .. |--no-binary option| replace:: ``--no-binary`` option .. __: https://pip.pypa.io/en/stable/reference/pip_install/#install-no-binary From f766d90704b03cbe277e3fd4b4153d892960f135 Mon Sep 17 00:00:00 2001 From: Mike Gerdts Date: Fri, 16 Feb 2018 22:34:41 +0000 Subject: [PATCH 29/55] Fixed building on SmartOS timeradd is missing on Solaris 10, but is present as a macro in on SmartOS, illumos, and likely Solaris 11. --- psycopg/solaris_support.c | 5 ++++- psycopg/solaris_support.h | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/psycopg/solaris_support.c b/psycopg/solaris_support.c index cf82e2ed..e5f8edf6 100644 --- a/psycopg/solaris_support.c +++ b/psycopg/solaris_support.c @@ -1,6 +1,7 @@ /* solaris_support.c - emulate functions missing on Solaris * * Copyright (C) 2017 My Karlsson + * Copyright (c) 2018, Joyent, Inc. * * This file is part of psycopg. * @@ -28,7 +29,8 @@ #include "psycopg/solaris_support.h" #if defined(__sun) && defined(__SVR4) -/* timeradd is missing on Solaris */ +/* timeradd is missing on Solaris 10 */ +#ifndef timeradd void timeradd(struct timeval *a, struct timeval *b, struct timeval *c) { @@ -51,4 +53,5 @@ timersub(struct timeval *a, struct timeval *b, struct timeval *c) c->tv_sec -= 1; } } +#endif /* timeradd */ #endif /* defined(__sun) && defined(__SVR4) */ diff --git a/psycopg/solaris_support.h b/psycopg/solaris_support.h index 33c2f2b8..880e9f18 100644 --- a/psycopg/solaris_support.h +++ b/psycopg/solaris_support.h @@ -1,6 +1,7 @@ /* solaris_support.h - definitions for solaris_support.c * * Copyright (C) 2017 My Karlsson + * Copyright (c) 2018, Joyent, Inc. * * This file is part of psycopg. * @@ -30,8 +31,10 @@ #if defined(__sun) && defined(__SVR4) #include +#ifndef timeradd extern HIDDEN void timeradd(struct timeval *a, struct timeval *b, struct timeval *c); extern HIDDEN void timersub(struct timeval *a, struct timeval *b, struct timeval *c); #endif +#endif #endif /* !defined(PSYCOPG_SOLARIS_SUPPORT_H) */ From 8670287928501edf0d3868cf468a6796ad841b50 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 19 Feb 2018 11:43:37 +0000 Subject: [PATCH 30/55] Added Solaris 11 fix to news file --- NEWS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS b/NEWS index c400f249..dfee39c9 100644 --- a/NEWS +++ b/NEWS @@ -14,6 +14,13 @@ Other changes: without using 2to3. +What's new in psycopg 2.7.5 +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Fixed building on Solaris 11 and derivatives such as SmartOS and illumos + (:ticket:`#677`) + + What's new in psycopg 2.7.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 32f5a9fc1d24be502cc71d10a2cd396348891710 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 19 Feb 2018 11:28:13 +0000 Subject: [PATCH 31/55] Avoid quoting the string in the psycopg version macro Use a macro trick to add the quotes. This seems more portable than passing the quotes to the command line (see #658). https://gcc.gnu.org/onlinedocs/cpp/Stringizing.html --- psycopg/psycopgmodule.c | 8 ++++++-- setup.py | 5 +---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 6c95bd69..5deaa163 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -72,6 +72,10 @@ HIDDEN PyObject *psyco_null = NULL; /* The type of the cursor.description items */ HIDDEN PyObject *psyco_DescriptionType = NULL; +/* macro trick to stringify a macro expansion */ +#define xstr(s) str(s) +#define str(s) #s + /** connect module-level function **/ #define psyco_connect_doc \ "_connect(dsn, [connection_factory], [async]) -- New database connection.\n\n" @@ -885,7 +889,7 @@ INIT_MODULE(_psycopg)(void) psycopg_debug_enabled = 1; #endif - Dprintf("initpsycopg: initializing psycopg %s", PSYCOPG_VERSION); + Dprintf("initpsycopg: initializing psycopg %s", xstr(PSYCOPG_VERSION)); /* initialize all the new types and then the module */ Py_TYPE(&connectionType) = &PyType_Type; @@ -1017,7 +1021,7 @@ INIT_MODULE(_psycopg)(void) if (!(psyco_DescriptionType = psyco_make_description_type())) { goto exit; } /* set some module's parameters */ - PyModule_AddStringConstant(module, "__version__", PSYCOPG_VERSION); + PyModule_AddStringConstant(module, "__version__", xstr(PSYCOPG_VERSION)); PyModule_AddStringConstant(module, "__doc__", "psycopg PostgreSQL driver"); PyModule_AddIntConstant(module, "__libpq_version__", PG_VERSION_NUM); PyModule_AddIntMacro(module, REPLICATION_PHYSICAL); diff --git a/setup.py b/setup.py index ec2b5ae0..2d608c86 100644 --- a/setup.py +++ b/setup.py @@ -596,10 +596,7 @@ if version_flags: else: PSYCOPG_VERSION_EX = PSYCOPG_VERSION -if not PLATFORM_IS_WINDOWS: - define_macros.append(('PSYCOPG_VERSION', '"' + PSYCOPG_VERSION_EX + '"')) -else: - define_macros.append(('PSYCOPG_VERSION', '\\"' + PSYCOPG_VERSION_EX + '\\"')) +define_macros.append(('PSYCOPG_VERSION', PSYCOPG_VERSION_EX)) if parser.has_option('build_ext', 'have_ssl'): have_ssl = int(parser.get('build_ext', 'have_ssl')) From ede418a009ea1875be37e5537c4b0c9640fc1685 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 19 Feb 2018 13:51:45 +0000 Subject: [PATCH 32/55] Report MSYS2 build probably fixed --- NEWS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index dfee39c9..d67ab1fa 100644 --- a/NEWS +++ b/NEWS @@ -18,7 +18,8 @@ What's new in psycopg 2.7.5 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Fixed building on Solaris 11 and derivatives such as SmartOS and illumos - (:ticket:`#677`) + (:ticket:`#677`). +- Maybe fixed building on MSYS2 (as reported in :ticket:`#658`). What's new in psycopg 2.7.4 From ea923b63a48880d3d229241de42813a822e5db1e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 20 Feb 2018 17:33:12 +0000 Subject: [PATCH 33/55] Allow strings subclasses in ensure_bytes Fix #679 --- NEWS | 1 + psycopg/utils.c | 4 ++-- tests/test_connection.py | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index d67ab1fa..09b19e5f 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,7 @@ What's new in psycopg 2.7.5 - Fixed building on Solaris 11 and derivatives such as SmartOS and illumos (:ticket:`#677`). - Maybe fixed building on MSYS2 (as reported in :ticket:`#658`). +- Allow string subclasses in connection and other places (:ticket:`#679`). What's new in psycopg 2.7.4 diff --git a/psycopg/utils.c b/psycopg/utils.c index a197fe22..261810fd 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -168,11 +168,11 @@ psycopg_ensure_bytes(PyObject *obj) PyObject *rv = NULL; if (!obj) { return NULL; } - if (PyUnicode_CheckExact(obj)) { + if (PyUnicode_Check(obj)) { rv = PyUnicode_AsUTF8String(obj); Py_DECREF(obj); } - else if (Bytes_CheckExact(obj)) { + else if (Bytes_Check(obj)) { rv = obj; } else { diff --git a/tests/test_connection.py b/tests/test_connection.py index a82bd499..4625e7e3 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -246,6 +246,13 @@ class ConnectionTests(ConnectingTestCase): else: del os.environ['PGCLIENTENCODING'] + def test_connect_no_string(self): + class MyString(str): + pass + + conn = psycopg2.connect(MyString(dsn)) + conn.close() + def test_weakref(self): from weakref import ref import gc @@ -400,6 +407,13 @@ class ParseDsnTestCase(ConnectingTestCase): self.assertRaises(TypeError, ext.parse_dsn, None) self.assertRaises(TypeError, ext.parse_dsn, 42) + def test_str_subclass(self): + class MyString(str): + pass + + res = ext.parse_dsn(MyString("dbname=test")) + self.assertEqual(res, {'dbname': 'test'}) + class MakeDsnTestCase(ConnectingTestCase): def test_empty_arguments(self): From 36f0db81d228b612a66abf6c880d1d6629da77fc Mon Sep 17 00:00:00 2001 From: Ashesh Vashi Date: Tue, 8 May 2018 15:29:16 +0530 Subject: [PATCH 34/55] Fixed the string format error reported by Travis-CI. Reference: https://travis-ci.org/psycopg/psycopg2/jobs/376288585 --- psycopg/psycopgmodule.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 8b6623af..5ed1c1c6 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -424,8 +424,6 @@ psyco_encrypt_password(PyObject *self, PyObject *args) connectionObject *conn = NULL; - static char *kwlist[] = {"password", "user", "scope", "algorithm", NULL}; - if (!PyArg_ParseTuple(args, "OO|OO", &password, &user, &obj, &algorithm)) { return NULL; @@ -482,7 +480,7 @@ psyco_encrypt_password(PyObject *self, PyObject *args) if (!encrypted) { const char *msg = PQerrorMessage(conn->pgconn); if (msg && *msg) { - PyErr_Format(ProgrammingError, msg); + PyErr_Format(ProgrammingError, "%s", msg); res = NULL; goto exit; } From b52ff101535218cbf2f6e203dac6dc86b173d4f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 18 May 2018 12:12:09 +0200 Subject: [PATCH 35/55] Travis CI: Run tests on Python 3.7 Also, switch to wheel, because eggs caused problems on 3.7: ValueError: bad marshal data (unknown type code) --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2fbf7010..51cba9a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,12 +6,14 @@ language: python python: - 2.7 + - 3.7-dev - 3.6 - 3.5 - 3.4 install: - - python setup.py install + - pip install -U pip setuptools wheel + - pip install . - rm -rf psycopg2.egg-info - sudo scripts/travis_prepare.sh From b5e4a040f8e50928aefc4f3dc9717bbcf3520cd0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 14 May 2018 02:38:44 +0100 Subject: [PATCH 36/55] Fixed adaptation of arrays of arrays of nulls Close #325, close #706. --- NEWS | 1 + psycopg/adapter_list.c | 113 +++++++++++++++++++++++++++----------- tests/test_types_basic.py | 29 +++++++--- 3 files changed, 104 insertions(+), 39 deletions(-) diff --git a/NEWS b/NEWS index 09b19e5f..3bce91c6 100644 --- a/NEWS +++ b/NEWS @@ -17,6 +17,7 @@ Other changes: What's new in psycopg 2.7.5 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Fixed adaptation of arrays of arrays of nulls (:ticket:`#325`). - Fixed building on Solaris 11 and derivatives such as SmartOS and illumos (:ticket:`#677`). - Maybe fixed building on MSYS2 (as reported in :ticket:`#658`). diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index dec17b4c..3fdff76a 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -38,13 +38,14 @@ list_quote(listObject *self) { /* adapt the list by calling adapt() recursively and then wrapping everything into "ARRAY[]" */ - PyObject *tmp = NULL, *str = NULL, *joined = NULL, *res = NULL; + PyObject *res = NULL; + PyObject **qs = NULL; + Py_ssize_t bufsize = 0; + char *buf = NULL, *ptr; /* list consisting of only NULL don't work with the ARRAY[] construct - * so we use the {NULL,...} syntax. Note however that list of lists where - * some element is a list of only null still fails: for that we should use - * the '{...}' syntax uniformly but we cannot do it in the current - * infrastructure. TODO in psycopg3 */ + * so we use the {NULL,...} syntax. The same syntax is also necessary + * to convert array of arrays containing only nulls. */ int all_nulls = 1; Py_ssize_t i, len; @@ -53,47 +54,95 @@ list_quote(listObject *self) /* empty arrays are converted to NULLs (still searching for a way to insert an empty array in postgresql */ - if (len == 0) return Bytes_FromString("'{}'"); + if (len == 0) { + res = Bytes_FromString("'{}'"); + goto exit; + } - tmp = PyTuple_New(len); + if (!(qs = PyMem_New(PyObject *, len))) { + PyErr_NoMemory(); + goto exit; + } + memset(qs, 0, len * sizeof(PyObject *)); - for (i=0; iwrapped, i); if (wrapped == Py_None) { Py_INCREF(psyco_null); - quoted = psyco_null; + qs[i] = psyco_null; } else { - quoted = microprotocol_getquoted(wrapped, - (connectionObject*)self->connection); - if (quoted == NULL) goto error; - all_nulls = 0; + if (!(qs[i] = microprotocol_getquoted( + wrapped, (connectionObject*)self->connection))) { + goto exit; + } + + /* Lists of arrays containing only nulls are also not supported + * by the ARRAY construct so we should do some special casing */ + if (!PyList_Check(wrapped) || Bytes_AS_STRING(qs[i])[0] == 'A') { + all_nulls = 0; + } } - - /* here we don't loose a refcnt: SET_ITEM does not change the - reference count and we are just transferring ownership of the tmp - object to the tuple */ - PyTuple_SET_ITEM(tmp, i, quoted); + bufsize += Bytes_GET_SIZE(qs[i]) + 1; /* this, and a comma */ } - /* now that we have a tuple of adapted objects we just need to join them - and put "ARRAY[] around the result */ - str = Bytes_FromString(", "); - joined = PyObject_CallMethod(str, "join", "(O)", tmp); - if (joined == NULL) goto error; + /* Create an array literal, usually ARRAY[...] but if the contents are + * all NULL or array of NULL we must use the '{...}' syntax + */ + if (!(ptr = buf = PyMem_Malloc(bufsize + 8))) { + PyErr_NoMemory(); + goto exit; + } - /* PG doesn't like ARRAY[NULL..] */ if (!all_nulls) { - res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined)); - } else { - res = Bytes_FromFormat("'{%s}'", Bytes_AsString(joined)); + strcpy(ptr, "ARRAY["); + ptr += 6; + for (i = 0; i < len; i++) { + Py_ssize_t sl; + sl = Bytes_GET_SIZE(qs[i]); + memcpy(ptr, Bytes_AS_STRING(qs[i]), sl); + ptr += sl; + *ptr++ = ','; + } + *(ptr - 1) = ']'; + } + else { + *ptr++ = '\''; + *ptr++ = '{'; + for (i = 0; i < len; i++) { + /* in case all the adapted things are nulls (or array of nulls), + * the quoted string is either NULL or an array of the form + * '{NULL,...}', in which case we have to strip the extra quotes */ + char *s; + Py_ssize_t sl; + s = Bytes_AS_STRING(qs[i]); + sl = Bytes_GET_SIZE(qs[i]); + if (s[0] != '\'') { + memcpy(ptr, s, sl); + ptr += sl; + } + else { + memcpy(ptr, s + 1, sl - 2); + ptr += sl - 2; + } + *ptr++ = ','; + } + *(ptr - 1) = '}'; + *ptr++ = '\''; } - error: - Py_XDECREF(tmp); - Py_XDECREF(str); - Py_XDECREF(joined); + res = Bytes_FromStringAndSize(buf, ptr - buf); + +exit: + if (qs) { + for (i = 0; i < len; i++) { + PyObject *q = qs[i]; + Py_XDECREF(q); + } + PyMem_Free(qs); + } + PyMem_Free(buf); + return res; } diff --git a/tests/test_types_basic.py b/tests/test_types_basic.py index a93265d9..76b9aa3f 100755 --- a/tests/test_types_basic.py +++ b/tests/test_types_basic.py @@ -224,16 +224,31 @@ class TypesBasicTests(ConnectingTestCase): curs.execute("insert into na (boola) values (%s)", ([True, None],)) curs.execute("insert into na (boola) values (%s)", ([None, None],)) - # TODO: array of array of nulls are not supported yet - # curs.execute("insert into na (textaa) values (%s)", ([[None]],)) + curs.execute("insert into na (textaa) values (%s)", ([[None]],)) curs.execute("insert into na (textaa) values (%s)", ([['a', None]],)) - # curs.execute("insert into na (textaa) values (%s)", ([[None, None]],)) - # curs.execute("insert into na (intaa) values (%s)", ([[None]],)) + curs.execute("insert into na (textaa) values (%s)", ([[None, None]],)) + + curs.execute("insert into na (intaa) values (%s)", ([[None]],)) curs.execute("insert into na (intaa) values (%s)", ([[42, None]],)) - # curs.execute("insert into na (intaa) values (%s)", ([[None, None]],)) - # curs.execute("insert into na (boolaa) values (%s)", ([[None]],)) + curs.execute("insert into na (intaa) values (%s)", ([[None, None]],)) + + curs.execute("insert into na (boolaa) values (%s)", ([[None]],)) curs.execute("insert into na (boolaa) values (%s)", ([[True, None]],)) - # curs.execute("insert into na (boolaa) values (%s)", ([[None, None]],)) + curs.execute("insert into na (boolaa) values (%s)", ([[None, None]],)) + + @testutils.skip_before_postgres(8, 2) + def testNestedArrays(self): + curs = self.conn.cursor() + for a in [ + [[1]], + [[None]], + [[None, None, None]], + [[None, None], [1, None]], + [[None, None], [None, None]], + [[[None, None], [None, None]]], + ]: + curs.execute("select %s::int[]", (a,)) + self.assertEqual(curs.fetchone()[0], a) @testutils.skip_from_python(3) def testTypeRoundtripBuffer(self): From 8dd00ee87469d908dfa800d50d87e09ab97ed90f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 14 May 2018 03:04:57 +0100 Subject: [PATCH 37/55] Hstore test fixed after adapting arrays dropped space after commas --- tests/test_types_extras.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 5cb13534..cda163b6 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -179,8 +179,8 @@ class HstoreTestCase(ConnectingTestCase): m = re.match(br'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)', q) self.assert_(m, repr(q)) - kk = m.group(1).split(b", ") - vv = m.group(2).split(b", ") + kk = m.group(1).split(b",") + vv = m.group(2).split(b",") ii = list(zip(kk, vv)) ii.sort() From bc84b6233eaa1e7a6302b51f8ab8950534ff1813 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 13 May 2018 23:51:21 +0100 Subject: [PATCH 38/55] Allow non-ascii chars in namedtuple fields They can be valid chars in Python 3. Or maybe not? In which case Python will throw an exception, but that's fine. Fix regression introduced fixing #211 --- NEWS | 2 ++ lib/extras.py | 9 ++++++--- tests/test_extras_dictcursor.py | 9 ++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 09b19e5f..a264c907 100644 --- a/NEWS +++ b/NEWS @@ -17,6 +17,8 @@ Other changes: What's new in psycopg 2.7.5 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Allow non-ascii chars in namedtuple fields (regression introduced fixing + :ticket':`#211`). - Fixed building on Solaris 11 and derivatives such as SmartOS and illumos (:ticket:`#677`). - Maybe fixed building on MSYS2 (as reported in :ticket:`#658`). diff --git a/lib/extras.py b/lib/extras.py index 1f85d532..9c26ccba 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -363,12 +363,15 @@ class NamedTupleCursor(_cursor): return def _make_nt(self): + # ascii except alnum and underscore + nochars = ' !"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~' + re_clean = _re.compile('[' + _re.escape(nochars) + ']') + def f(s): - # NOTE: Python 3 actually allows unicode chars in fields - s = _re.sub('[^a-zA-Z0-9_]', '_', s) + s = re_clean.sub('_', s) # Python identifier cannot start with numbers, namedtuple fields # cannot start with underscore. So... - if _re.match('^[0-9_]', s): + if s[0] == '_' or '0' <= s[0] <= '9': s = 'f' + s return s diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index 99bdeee6..2a46fbaf 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -19,7 +19,7 @@ from datetime import timedelta import psycopg2 import psycopg2.extras import unittest -from .testutils import ConnectingTestCase, skip_before_postgres +from .testutils import ConnectingTestCase, skip_before_postgres, skip_before_python class ExtrasDictCursorTests(ConnectingTestCase): @@ -357,6 +357,13 @@ class NamedTupleCursorTest(ConnectingTestCase): self.assertEqual(rv.f_column_, 2) self.assertEqual(rv.f3, 3) + @skip_before_python(3) + def test_nonascii_name(self): + curs = self.conn.cursor() + curs.execute('select 1 as \xe5h\xe9') + rv = curs.fetchone() + self.assertEqual(getattr(rv, '\xe5h\xe9'), 1) + def test_minimal_generation(self): # Instrument the class to verify it gets called the minimum number of times. from psycopg2.extras import NamedTupleCursor From eb570488a40ce67fb1df3ed548bda1c57d585084 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 14 May 2018 02:41:32 +0100 Subject: [PATCH 39/55] Test databases from newest to oldest This way we can spot when a feature was not supported yet by the first test failing. --- scripts/travis_test.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh index 0320654a..342e24cc 100755 --- a/scripts/travis_test.sh +++ b/scripts/travis_test.sh @@ -56,15 +56,15 @@ fi # Unsupported postgres versions that we still support # Images built by https://github.com/psycopg/psycopg2-wheels/tree/build-dinosaurs if [[ -n "$TEST_PAST" ]]; then - run_test 7.4 - run_test 8.0 - run_test 8.1 - run_test 8.2 - run_test 8.3 - run_test 8.4 - run_test 9.0 - run_test 9.1 run_test 9.2 + run_test 9.1 + run_test 9.0 + run_test 8.4 + run_test 8.3 + run_test 8.2 + run_test 8.1 + run_test 8.0 + run_test 7.4 fi # Postgres built from master From dd7e5c906fc37c50852619d68d6a347ab5928371 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 14 May 2018 03:11:11 +0100 Subject: [PATCH 40/55] Skipped test on db version not supporting unicode identifiers --- tests/test_extras_dictcursor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index 2a46fbaf..d43980af 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -358,6 +358,7 @@ class NamedTupleCursorTest(ConnectingTestCase): self.assertEqual(rv.f3, 3) @skip_before_python(3) + @skip_before_postgres(8) def test_nonascii_name(self): curs = self.conn.cursor() curs.execute('select 1 as \xe5h\xe9') From 3f389593f54a6ad3354140b77d9a6ce725c0516f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 13:22:38 +0100 Subject: [PATCH 41/55] Raise NotSupportedError fetching iso_8601 intervals Previously it would have failed parsing and resulted in ValueError Close #707 --- psycopg/typecast_datetime.c | 5 +++++ tests/test_dates.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/psycopg/typecast_datetime.c b/psycopg/typecast_datetime.c index f24223cb..e34117dd 100644 --- a/psycopg/typecast_datetime.c +++ b/psycopg/typecast_datetime.c @@ -406,6 +406,11 @@ typecast_PYINTERVAL_cast(const char *str, Py_ssize_t len, PyObject *curs) } break; + case 'P': + PyErr_SetString(NotSupportedError, + "iso_8601 intervalstyle currently not supported"); + return NULL; + default: break; } diff --git a/tests/test_dates.py b/tests/test_dates.py index 74dfc9ab..85216f87 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -438,6 +438,13 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): r = cur.fetchone()[0] self.assertEqual(r, v, "%s -> %s != %s" % (s, r, v)) + def test_interval_iso_8601_not_supported(self): + # We may end up supporting, but no pressure for it + cur = self.conn.cursor() + cur.execute("set local intervalstyle to iso_8601") + cur.execute("select '1 day 2 hours'::interval") + self.assertRaises(psycopg2.NotSupportedError, cur.fetchone) + # Only run the datetime tests if psycopg was compiled with support. if not hasattr(psycopg2.extensions, 'PYDATETIME'): From 49d9edce01c64dadba4fcdcd8ddeb563cd2cff63 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 13:40:57 +0100 Subject: [PATCH 42/55] Set minimal postgres version for intervalstyle test --- tests/test_dates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_dates.py b/tests/test_dates.py index 85216f87..bb5aee30 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -438,6 +438,7 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): r = cur.fetchone()[0] self.assertEqual(r, v, "%s -> %s != %s" % (s, r, v)) + @skip_before_postgres(8, 4) def test_interval_iso_8601_not_supported(self): # We may end up supporting, but no pressure for it cur = self.conn.cursor() From c4da939909ea9bef99bcdef805b68dca6131eac0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 13:56:59 +0100 Subject: [PATCH 43/55] Don't raise an exception closing an unused named cursor Close #716 --- NEWS | 1 + psycopg/cursor_type.c | 18 ++++++++++++------ tests/test_cursor.py | 5 +++++ tests/test_with.py | 7 ++++++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index 5dc4ae04..cb1b5813 100644 --- a/NEWS +++ b/NEWS @@ -24,6 +24,7 @@ What's new in psycopg 2.7.5 (:ticket:`#677`). - Maybe fixed building on MSYS2 (as reported in :ticket:`#658`). - Allow string subclasses in connection and other places (:ticket:`#679`). +- Don't raise an exception closing an unused named cursor (:ticket:`#716`). What's new in psycopg 2.7.4 diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index b7fd1870..d73bc3a4 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -59,6 +59,11 @@ psyco_curs_close(cursorObject *self) char buffer[128]; PGTransactionStatusType status; + if (!self->query) { + Dprintf("skipping named cursor close because unused"); + goto close; + } + if (self->conn) { status = PQtransactionStatus(self->conn->pgconn); } @@ -66,17 +71,18 @@ psyco_curs_close(cursorObject *self) status = PQTRANS_UNKNOWN; } - if (!(status == PQTRANS_UNKNOWN || status == PQTRANS_INERROR)) { - EXC_IF_NO_MARK(self); - PyOS_snprintf(buffer, 127, "CLOSE %s", self->qname); - if (pq_execute(self, buffer, 0, 0, 1) == -1) return NULL; - } - else { + if (status == PQTRANS_UNKNOWN || status == PQTRANS_INERROR) { Dprintf("skipping named curs close because tx status %d", (int)status); + goto close; } + + EXC_IF_NO_MARK(self); + PyOS_snprintf(buffer, 127, "CLOSE %s", self->qname); + if (pq_execute(self, buffer, 0, 0, 1) == -1) return NULL; } +close: self->closed = 1; Dprintf("psyco_curs_close: cursor at %p closed", self); diff --git a/tests/test_cursor.py b/tests/test_cursor.py index cc8db0f4..b3e03d9b 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -435,6 +435,11 @@ class CursorTests(ConnectingTestCase): self.assertEqual([(2,), (3,), (4,)], cur2.fetchmany(3)) self.assertEqual([(5,), (6,), (7,)], cur2.fetchall()) + @skip_before_postgres(8, 0) + def test_named_noop_close(self): + cur = self.conn.cursor('test') + cur.close() + @skip_before_postgres(8, 0) def test_scroll(self): cur = self.conn.cursor() diff --git a/tests/test_with.py b/tests/test_with.py index 1392d85f..f26f8f9c 100755 --- a/tests/test_with.py +++ b/tests/test_with.py @@ -26,7 +26,7 @@ import psycopg2 import psycopg2.extensions as ext import unittest -from .testutils import ConnectingTestCase +from .testutils import ConnectingTestCase, skip_before_postgres class WithTestCase(ConnectingTestCase): @@ -215,6 +215,11 @@ class WithCursorTestCase(WithTestCase): else: self.fail("where is my exception?") + @skip_before_postgres(8, 0) + def test_named_with_noop(self): + with self.conn.cursor('named') as cur: + pass + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 3f0a7f9af410ebec0ca231e31abeebf4c49b3d33 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 17:03:08 +0100 Subject: [PATCH 44/55] Added license to the docs Includes other docs improvements, such as the ones proposed in #711. --- LICENSE | 10 +++++----- NEWS | 2 +- doc/src/index.rst | 1 + doc/src/install.rst | 12 +++++++++--- doc/src/license.rst | 7 +++++++ doc/src/news.rst | 4 ++++ 6 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 doc/src/license.rst diff --git a/LICENSE b/LICENSE index 360a44f6..bdeaf9c4 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ psycopg2 and the LGPL -===================== +--------------------- psycopg2 is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published @@ -29,15 +29,15 @@ If not, see . Alternative licenses -==================== +-------------------- If you prefer you can use the Zope Database Adapter ZPsycopgDA (i.e., -every file inside the ZPsycopgDA directory) user the ZPL license as +every file inside the ZPsycopgDA directory) using the ZPL license as published on the Zope web site, http://www.zope.org/Resources/ZPL. Also, the following BSD-like license applies (at your option) to the -files following the pattern psycopg/adapter*.{h,c} and -psycopg/microprotocol*.{h,c}: +files following the pattern ``psycopg/adapter*.{h,c}`` and +``psycopg/microprotocol*.{h,c}``: Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it diff --git a/NEWS b/NEWS index cb1b5813..c44639da 100644 --- a/NEWS +++ b/NEWS @@ -18,7 +18,7 @@ What's new in psycopg 2.7.5 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Allow non-ascii chars in namedtuple fields (regression introduced fixing - :ticket':`#211`). + :ticket:`#211`). - Fixed adaptation of arrays of arrays of nulls (:ticket:`#325`). - Fixed building on Solaris 11 and derivatives such as SmartOS and illumos (:ticket:`#677`). diff --git a/doc/src/index.rst b/doc/src/index.rst index 852bbc2c..7ae073d7 100644 --- a/doc/src/index.rst +++ b/doc/src/index.rst @@ -48,6 +48,7 @@ Psycopg 2 is both Unicode and Python 3 friendly. errorcodes faq news + license .. ifconfig:: builder != 'text' diff --git a/doc/src/install.rst b/doc/src/install.rst index a858cbe3..9afb0bc3 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -45,7 +45,9 @@ Build prerequisites ^^^^^^^^^^^^^^^^^^^ The build prerequisites are to be met in order to install Psycopg from source -code, either from a source distribution package or from PyPI. +code, from a source distribution package, GitHub_ or from PyPI. + +.. _GitHub: https://github.com/psycopg/psycopg2 Psycopg is a C wrapper around the libpq_ PostgreSQL client library. To install it from sources you will need: @@ -302,10 +304,14 @@ Try the following. *In order:* - Google for `!psycopg2` *your error message*. Especially useful the week after the release of a new OS X version. -- Write to the `Mailing List`__. +- Write to the `Mailing List`_. + +- If you think that you have discovered a bug, test failure or missing feature + please raise a ticket in the `bug tracker`_. - Complain on your blog or on Twitter that `!psycopg2` is the worst package ever and about the quality time you have wasted figuring out the correct :envvar:`ARCHFLAGS`. Especially useful from the Starbucks near you. -.. __: https://lists.postgresql.org/mj/mj_wwwusr?func=lists-long-full&extra=psycopg +.. _mailing list: https://lists.postgresql.org/mj/mj_wwwusr?func=lists-long-full&extra=psycopg +.. _bug tracker: https://github.com/psycopg/psycopg2/issues diff --git a/doc/src/license.rst b/doc/src/license.rst new file mode 100644 index 00000000..53a4e724 --- /dev/null +++ b/doc/src/license.rst @@ -0,0 +1,7 @@ +.. index:: + single: License + +License +======= + +.. include:: ../../LICENSE diff --git a/doc/src/news.rst b/doc/src/news.rst index d5b11a69..053d6464 100644 --- a/doc/src/news.rst +++ b/doc/src/news.rst @@ -1,3 +1,7 @@ +.. index:: + single: Release notes + single: News + Release notes ============= From a110d7dd7056ebb8ba7bb60b868971d7fb176000 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 25 Feb 2018 18:11:17 +0000 Subject: [PATCH 45/55] Intersphinx urls to generate Python links updated Previous urls warn about a redirect, so they are probably to go. --- doc/src/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/conf.py b/doc/src/conf.py index a27d6cf4..2c52a568 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -61,8 +61,8 @@ except ImportError: release = version intersphinx_mapping = { - 'py': ('http://docs.python.org/2', None), - 'py3': ('http://docs.python.org/3', None), + 'py': ('https://docs.python.org/2', None), + 'py3': ('https://docs.python.org/3', None), } # Pattern to generate links to the bug tracker From 10caf1bd74f172fe0adb2c301894a239443a65d8 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 25 Feb 2018 18:13:29 +0000 Subject: [PATCH 46/55] Command to upload docs on pythonhosted dropped altogether --- doc/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/Makefile b/doc/Makefile index 2903b9d0..558d0a75 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -29,8 +29,6 @@ doctest: upload: # this command requires ssh configured to the proper target tar czf - -C html . | ssh psycoweb tar xzvf - -C docs/current - # this command requires a .pypirc with the right privileges - # python src/tools/pypi_docs_upload.py psycopg2 $$(pwd)/html clean: $(MAKE) $(SPHOPTS) -C src $@ From 9ceffa1cc641d4bc04a714558cd4ad6852f407e5 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Thu, 26 Apr 2018 05:52:41 -0700 Subject: [PATCH 47/55] Update all pypi.python.org URLs to pypi.org For details on the new PyPI, see the blog post: https://pythoninsider.blogspot.ca/2018/04/new-pypi-launched-legacy-pypi-shutting.html --- README.rst | 4 ++-- doc/release.rst | 2 +- doc/src/advanced.rst | 2 +- doc/src/install.rst | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index b5b048d4..a9785e3a 100644 --- a/README.rst +++ b/README.rst @@ -54,8 +54,8 @@ external libraries, by installing the `psycopg2-binary`_ package from PyPI:: The binary package is a practical choice for development and testing but in production it is advised to use the package built from sources. -.. _PyPI: https://pypi.python.org/pypi/psycopg2 -.. _psycopg2-binary: https://pypi.python.org/pypi/psycopg2-binary +.. _PyPI: https://pypi.org/project/psycopg2/ +.. _psycopg2-binary: https://pypi.org/project/psycopg2-binary/ .. _install: http://initd.org/psycopg/docs/install.html#install-from-source .. _faq: http://initd.org/psycopg/docs/faq.html#faq-compile diff --git a/doc/release.rst b/doc/release.rst index 3576cdc9..3ea4a9fa 100644 --- a/doc/release.rst +++ b/doc/release.rst @@ -100,5 +100,5 @@ Test packages may be uploaded on the `PyPI testing site`__ using:: assuming `proper configuration`__ of ``~/.pypirc``. -.. __: https://testpypi.python.org/pypi/psycopg2 +.. __: https://test.pypi.org/project/psycopg2/ .. __: https://wiki.python.org/moin/TestPyPI diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index d1683b8b..724cb281 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -485,7 +485,7 @@ details. You can check the `psycogreen`_ project for further informations and resources about the topic. .. _coroutine: http://en.wikipedia.org/wiki/Coroutine -.. _greenlet: http://pypi.python.org/pypi/greenlet +.. _greenlet: https://pypi.org/project/greenlet/ .. _green threads: http://en.wikipedia.org/wiki/Green_threads .. _Eventlet: http://eventlet.net/ .. _gevent: http://www.gevent.org/ diff --git a/doc/src/install.rst b/doc/src/install.rst index 9afb0bc3..f5524a56 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -142,7 +142,7 @@ Make sure to use an up-to-date version of :program:`pip` (you can upgrade it using something like ``pip install -U pip``) .. __: PyPI-binary_ -.. _PyPI-binary: https://pypi.python.org/pypi/psycopg2-binary/ +.. _PyPI-binary: https://pypi.org/project/psycopg2-binary/ .. _wheel: http://pythonwheels.com/ .. note:: From a8d4f37b191399639ce80df0b5316702ca9f7e5f Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Wed, 7 Mar 2018 05:12:39 -0800 Subject: [PATCH 48/55] Complete tox testing matrix Now tests all supported Python environments as well. Allows testing the full matrix of Python version with a single command. Include the command make in the whitelist_externals configuration to avoid the runtime warning: WARNING:test command found but not installed in testenv cmd: /usr/bin/make env: .../psycopg2/.tox/flake8 Maybe you forgot to specify a dependency? See also the whitelist_externals envconfig setting. --- tox.ini | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 17612e25..a0eafa4d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,9 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - [tox] -envlist = py27 +envlist = py{27,34,35,36} [testenv] commands = make check +whitelist_externals = make [flake8] max-line-length = 85 From a3063900eeb1442b2c58ebb1b89a924548c8db6d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 19:18:42 +0100 Subject: [PATCH 49/55] Fixed code flow in encrypt_password() Fixed several shortcomings highlighted in #576 and not fixed as requested. Also fixed broken behaviour of ignoring the algorithm if the connection is missing. --- psycopg/psycopgmodule.c | 119 +++++++++++++++++++-------------------- tests/test_connection.py | 28 ++++----- 2 files changed, 73 insertions(+), 74 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 5ed1c1c6..121bf133 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -409,100 +409,99 @@ psyco_libpq_version(PyObject *self) /* encrypt_password - Prepare the encrypted password form */ #define psyco_encrypt_password_doc \ -"encrypt_password(password, user, [conn_or_curs], [algorithm]) -- Prepares the encrypted form of a PostgreSQL password.\n\n" +"encrypt_password(password, user, [scope], [algorithm]) -- Prepares the encrypted form of a PostgreSQL password.\n\n" static PyObject * -psyco_encrypt_password(PyObject *self, PyObject *args) +psyco_encrypt_password(PyObject *self, PyObject *args, PyObject *kwargs) { char *encrypted = NULL; + PyObject *password = NULL, *user = NULL; + PyObject *scope = Py_None, *algorithm = Py_None; + PyObject *res = NULL; - PyObject *obj = NULL, - *res = Py_None, - *password = NULL, - *user = NULL, - *algorithm = NULL; + static char *kwlist[] = {"password", "user", "scope", "algorithm", NULL}; connectionObject *conn = NULL; - if (!PyArg_ParseTuple(args, "OO|OO", - &password, &user, &obj, &algorithm)) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|OO", kwlist, + &password, &user, &scope, &algorithm)) { return NULL; } - if (obj != NULL && obj != Py_None) { - if (PyObject_TypeCheck(obj, &cursorType)) { - conn = ((cursorObject*)obj)->conn; + if (scope != Py_None) { + if (PyObject_TypeCheck(scope, &cursorType)) { + conn = ((cursorObject*)scope)->conn; } - else if (PyObject_TypeCheck(obj, &connectionType)) { - conn = (connectionObject*)obj; + else if (PyObject_TypeCheck(scope, &connectionType)) { + conn = (connectionObject*)scope; } else { PyErr_SetString(PyExc_TypeError, - "argument 3 must be a connection or a cursor"); - return NULL; + "the scope must be a connection or a cursor"); + goto exit; } } /* for ensure_bytes */ Py_INCREF(user); Py_INCREF(password); - if (algorithm) { - Py_INCREF(algorithm); - } + Py_INCREF(algorithm); if (!(user = psycopg_ensure_bytes(user))) { goto exit; } if (!(password = psycopg_ensure_bytes(password))) { goto exit; } - if (algorithm && !(algorithm = psycopg_ensure_bytes(algorithm))) { - goto exit; - } - - /* Use the libpq API 'PQencryptPassword', when no connection object is - available, or the algorithm is set to as 'md5', or the database server - version < 10 */ - if (conn == NULL || conn->server_version < 100000 || - (algorithm != NULL && algorithm != Py_None && - strcmp(Bytes_AS_STRING(algorithm), "md5") == 0)) { - encrypted = PQencryptPassword(Bytes_AS_STRING(password), - Bytes_AS_STRING(user)); - - if (encrypted != NULL) { - res = Text_FromUTF8(encrypted); - PQfreemem(encrypted); - } - goto exit; - } - -#if PG_VERSION_NUM >= 100000 - encrypted = PQencryptPasswordConn(conn->pgconn, Bytes_AS_STRING(password), - Bytes_AS_STRING(user), - algorithm ? Bytes_AS_STRING(algorithm) : NULL); - - if (!encrypted) { - const char *msg = PQerrorMessage(conn->pgconn); - if (msg && *msg) { - PyErr_Format(ProgrammingError, "%s", msg); - res = NULL; + if (algorithm != Py_None) { + if (!(algorithm = psycopg_ensure_bytes(algorithm))) { goto exit; } } - else { - res = Text_FromUTF8(encrypted); - PQfreemem(encrypted); + + /* If we have to encrypt md5 we can use the libpq < 10 API */ + if (algorithm != Py_None && + strcmp(Bytes_AS_STRING(algorithm), "md5") == 0) { + encrypted = PQencryptPassword( + Bytes_AS_STRING(password), Bytes_AS_STRING(user)); } + + /* If the algorithm is not md5 we have to use the API available from + * libpq 10. */ + else { +#if PG_VERSION_NUM >= 100000 + if (!conn) { + PyErr_SetString(ProgrammingError, + "password encryption (other than 'md5' algorithm)" + " requires a connection or cursor"); + goto exit; + } + + /* TODO: algo = will block: forbid on async/green conn? */ + encrypted = PQencryptPasswordConn(conn->pgconn, + Bytes_AS_STRING(password), Bytes_AS_STRING(user), + algorithm != Py_None ? Bytes_AS_STRING(algorithm) : NULL); #else - PyErr_SetString( - NotSupportedError, - "Password encryption (other than 'md5' algorithm) is not supported for the server version >= 10 in libpq < 10" - ); - res = NULL; + PyErr_SetString(NotSupportedError, + "password encryption (other than 'md5' algorithm)" + " requires libpq 10"); + goto exit; #endif + } + + if (encrypted) { + res = Text_FromUTF8(encrypted); + } + else { + const char *msg = PQerrorMessage(conn->pgconn); + PyErr_Format(ProgrammingError, + "password encryption failed: %s", msg ? msg : "no reason given"); + goto exit; + } exit: + if (encrypted) { + PQfreemem(encrypted); + } Py_XDECREF(user); Py_XDECREF(password); - if (algorithm) { - Py_XDECREF(algorithm); - } + Py_XDECREF(algorithm); return res; } diff --git a/tests/test_connection.py b/tests/test_connection.py index 6d696a41..f8d2d1d2 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1397,6 +1397,18 @@ class TransactionControlTests(ConnectingTestCase): cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'off') + def test_idempotence_check(self): + self.conn.autocommit = False + self.conn.readonly = True + self.conn.autocommit = True + self.conn.readonly = True + + cur = self.conn.cursor() + cur.execute("SHOW transaction_read_only") + self.assertEqual(cur.fetchone()[0], 'on') + + +class TestEncryptPassword(ConnectingTestCase): @skip_before_postgres(10) def test_encrypt_password_post_9_6(self): cur = self.conn.cursor() @@ -1441,20 +1453,8 @@ class TransactionControlTests(ConnectingTestCase): # Encryption algorithm will be ignored for postgres version < 10, it # will always use MD5. - self.assertEqual( - ext.encrypt_password('psycopg2', 'ashesh', self.conn, 'abc'), - 'md594839d658c28a357126f105b9cb14cfc' - ) - - def test_idempotence_check(self): - self.conn.autocommit = False - self.conn.readonly = True - self.conn.autocommit = True - self.conn.readonly = True - - cur = self.conn.cursor() - cur.execute("SHOW transaction_read_only") - self.assertEqual(cur.fetchone()[0], 'on') + self.assertRaises(psycopg2.ProgrammingError, + ext.encrypt_password, 'psycopg2', 'ashesh', self.conn, 'abc') class AutocommitTests(ConnectingTestCase): From 9e4f89a2a1e154d05f48b996a20fef1bdbbe9d51 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 20:13:04 +0100 Subject: [PATCH 50/55] encrypt_password docs moved to extension module and updated --- doc/src/connection.rst | 27 --------------------------- doc/src/extensions.rst | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index d656fa1b..2910f301 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -676,33 +676,6 @@ The ``connection`` class .. __: http://www.postgresql.org/docs/current/static/libpq-status.html#LIBPQ-PQTRANSACTIONSTATUS - .. method:: encrypt_password(password, user, [algorithm]) - - Returns the encrypted form of a PostgreSQL password based on the - current password encryption algorithm. - - Raises `~psycopg2.NotSupportedError` if the ``psycopg2`` module was - compiled with a ``libpq`` version lesser than 10 (which can be detected - by the `~psycopg2.__libpq_version__` constant), when encryption - algorithm other than 'md5' is specified for the server version greater - than, or equal to 10. - - Ignores the encrytion algorithm for servers version less than 10, and - always uses 'md5' as encryption algorithm. - - .. seealso:: libpq docs for `PQencryptPasswordConn()`__ for details. - - .. __: https://www.postgresql.org/docs/devel/static/libpq-misc.html#libpq-pqencryptpasswordconn - - .. seealso:: libpq docs for `PQencryptPassword()`__ for details. - - .. __: https://www.postgresql.org/docs/devel/static/libpq-misc.html#libpq-pqencryptpassword - - .. seealso:: libpq docs for `password_encryption`__ for details. - - .. __: https://www.postgresql.org/docs/devel/static/runtime-config-connection.html#guc-password-encryption - - .. index:: pair: Protocol; Version diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index 8545fcfa..34d53a7e 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -555,6 +555,38 @@ Other functions .. __: http://www.postgresql.org/docs/current/static/libpq-exec.html#LIBPQ-PQESCAPEIDENTIFIER +.. method:: encrypt_password(password, user, scope=None, algorithm=None) + + Return the encrypted form of a PostgreSQL password. + + :param password: the cleartext password to encrypt + :param user: the name of the user to use the password for + :param scope: the scope to encrypt the password into; if *algorithm* is + ``md5`` it can be `!None` + :type scope: `connection` or `cursor` + :param algorithm: the password encryption algorithm to use + + The *algorithm* ``md5`` is always supported. Other algorithms are only + supported if the client libpq version is at least 10 and may require a + compatible server version: check the `PostgreSQL encryption + documentation`__ to know the algorithms supported by your server. + + .. __: https://www.postgresql.org/docs/current/static/encryption-options.html + + Using `!None` as *algorithm* will result in querying the server to know the + current server password encryption setting, which is a blocking operation: + query the server separately and specify a value for *algorithm* if you + want to maintain a non-blocking behaviour. + + .. versionadded:: 2.8 + + .. seealso:: PostgreSQL docs for the `password_encryption`__ setting, libpq `PQencryptPasswordConn()`__, `PQencryptPassword()`__ functions. + + .. __: https://www.postgresql.org/docs/current/static/runtime-config-connection.html#GUC-PASSWORD-ENCRYPTION + .. __: https://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQENCRYPTPASSWORDCONN + .. __: https://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQENCRYPTPASSWORD + + .. index:: pair: Isolation level; Constants From abca14d6015dc88a8e66c4baa374ab7102798ee4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 20:50:04 +0100 Subject: [PATCH 51/55] Fixed keywords support for encrypt_password and tests completed --- psycopg/psycopgmodule.c | 2 +- tests/test_connection.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 121bf133..1b0567b3 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -957,7 +957,7 @@ static PyMethodDef psycopgMethods[] = { {"get_wait_callback", (PyCFunction)psyco_get_wait_callback, METH_NOARGS, psyco_get_wait_callback_doc}, {"encrypt_password", (PyCFunction)psyco_encrypt_password, - METH_VARARGS, psyco_encrypt_password_doc}, + METH_VARARGS|METH_KEYWORDS, psyco_encrypt_password_doc}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/tests/test_connection.py b/tests/test_connection.py index f8d2d1d2..7861ab71 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1421,6 +1421,13 @@ class TestEncryptPassword(ConnectingTestCase): 'md594839d658c28a357126f105b9cb14cfc' ) + # keywords + self.assertEqual( + ext.encrypt_password( + password='psycopg2', user='ashesh', + scope=self.conn, algorithm='md5'), + 'md594839d658c28a357126f105b9cb14cfc' + ) if libpq_version() < 100000: self.assertRaises( psycopg2.NotSupportedError, @@ -1444,6 +1451,9 @@ class TestEncryptPassword(ConnectingTestCase): )[:14], 'SCRAM-SHA-256$' ) + self.assertRaises(psycopg2.ProgrammingError, + ext.encrypt_password, 'psycopg2', 'ashesh', self.conn, 'abc') + @skip_after_postgres(10) def test_encrypt_password_pre_10(self): self.assertEqual( @@ -1451,11 +1461,27 @@ class TestEncryptPassword(ConnectingTestCase): 'md594839d658c28a357126f105b9cb14cfc' ) - # Encryption algorithm will be ignored for postgres version < 10, it - # will always use MD5. self.assertRaises(psycopg2.ProgrammingError, ext.encrypt_password, 'psycopg2', 'ashesh', self.conn, 'abc') + def test_encrypt_md5(self): + self.assertEqual( + ext.encrypt_password('psycopg2', 'ashesh', algorithm='md5'), + 'md594839d658c28a357126f105b9cb14cfc' + ) + + def test_encrypt_scram(self): + if libpq_version() >= 100000: + self.assert_( + ext.encrypt_password( + 'psycopg2', 'ashesh', self.conn, 'scram-sha-256') + .startswith('SCRAM-SHA-256$')) + else: + self.assertRaises(psycopg2.NotSupportedError, + ext.encrypt_password, + password='psycopg2', user='ashesh', + scope=self.conn, algorithm='scram-sha-256') + class AutocommitTests(ConnectingTestCase): def test_closed(self): From 9cf658ec6e2844c50fb7824a42f71cf965e607c8 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 21:18:36 +0100 Subject: [PATCH 52/55] Fixed refcount handling in encrypt_password Added tests to check bad types, which discovered the above problem: on type error we would have decref'd on exit something that was only borrowed (because we wouldn't have performed matching increfs). --- psycopg/psycopgmodule.c | 17 ++++++++--------- tests/test_connection.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 1b0567b3..23e648d2 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -418,16 +418,20 @@ psyco_encrypt_password(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *password = NULL, *user = NULL; PyObject *scope = Py_None, *algorithm = Py_None; PyObject *res = NULL; + connectionObject *conn = NULL; static char *kwlist[] = {"password", "user", "scope", "algorithm", NULL}; - connectionObject *conn = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|OO", kwlist, &password, &user, &scope, &algorithm)) { return NULL; } + /* for ensure_bytes */ + Py_INCREF(user); + Py_INCREF(password); + Py_INCREF(algorithm); + if (scope != Py_None) { if (PyObject_TypeCheck(scope, &cursorType)) { conn = ((cursorObject*)scope)->conn; @@ -437,16 +441,11 @@ psyco_encrypt_password(PyObject *self, PyObject *args, PyObject *kwargs) } else { PyErr_SetString(PyExc_TypeError, - "the scope must be a connection or a cursor"); + "the scope must be a connection or a cursor"); goto exit; } } - /* for ensure_bytes */ - Py_INCREF(user); - Py_INCREF(password); - Py_INCREF(algorithm); - if (!(user = psycopg_ensure_bytes(user))) { goto exit; } if (!(password = psycopg_ensure_bytes(password))) { goto exit; } if (algorithm != Py_None) { @@ -473,7 +472,7 @@ psyco_encrypt_password(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } - /* TODO: algo = will block: forbid on async/green conn? */ + /* TODO: algo = None will block: forbid on async/green conn? */ encrypted = PQencryptPasswordConn(conn->pgconn, Bytes_AS_STRING(password), Bytes_AS_STRING(user), algorithm != Py_None ? Bytes_AS_STRING(algorithm) : NULL); diff --git a/tests/test_connection.py b/tests/test_connection.py index 7861ab71..13635f1f 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1482,6 +1482,17 @@ class TestEncryptPassword(ConnectingTestCase): password='psycopg2', user='ashesh', scope=self.conn, algorithm='scram-sha-256') + def test_bad_types(self): + self.assertRaises(TypeError, ext.encrypt_password) + self.assertRaises(TypeError, ext.encrypt_password, + 'password', 42, self.conn, 'md5') + self.assertRaises(TypeError, ext.encrypt_password, + 42, 'user', self.conn, 'md5') + self.assertRaises(TypeError, ext.encrypt_password, + 42, 'user', 'wat', 'abc') + self.assertRaises(TypeError, ext.encrypt_password, + 'password', 'user', 'wat', 42) + class AutocommitTests(ConnectingTestCase): def test_closed(self): From 9eb3e0cb79aa58eaa566ff8d7ca080038f6c670a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 22:31:22 +0100 Subject: [PATCH 53/55] encrypt_password() reported in the news file --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index c44639da..77d547c5 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,10 @@ Current release What's new in psycopg 2.8 ------------------------- +New features: + +- Added `~psycopg2.extensions.encrypt_password()` function (:ticket:`#576`). + Other changes: - Dropped support for Python 2.6, 3.2, 3.3. From 82ae44ac3ae3bb8d474b5f4f50dd031d06f00147 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 23:35:26 +0100 Subject: [PATCH 54/55] DictCursor/RealDictCursor tests split --- tests/test_extras_dictcursor.py | 160 ++++++++++++++++---------------- 1 file changed, 82 insertions(+), 78 deletions(-) diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index d43980af..13477791 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -22,9 +22,7 @@ import unittest from .testutils import ConnectingTestCase, skip_before_postgres, skip_before_python -class ExtrasDictCursorTests(ConnectingTestCase): - """Test if DictCursor extension class works.""" - +class _DictCursorBase(ConnectingTestCase): def setUp(self): ConnectingTestCase.setUp(self) curs = self.conn.cursor() @@ -32,6 +30,30 @@ class ExtrasDictCursorTests(ConnectingTestCase): curs.execute("INSERT INTO ExtrasDictCursorTests VALUES ('bar')") self.conn.commit() + def _testIterRowNumber(self, curs): + # Only checking for dataset < itersize: + # see CursorTests.test_iter_named_cursor_rownumber + curs.itersize = 20 + curs.execute("""select * from generate_series(1,10)""") + for i, r in enumerate(curs): + self.assertEqual(i + 1, curs.rownumber) + + def _testNamedCursorNotGreedy(self, curs): + curs.itersize = 2 + curs.execute("""select clock_timestamp() as ts from generate_series(1,3)""") + recs = [] + for t in curs: + time.sleep(0.01) + recs.append(t) + + # check that the dataset was not fetched in a single gulp + self.assert_(recs[1]['ts'] - recs[0]['ts'] < timedelta(seconds=0.005)) + self.assert_(recs[2]['ts'] - recs[1]['ts'] > timedelta(seconds=0.0099)) + + +class ExtrasDictCursorTests(_DictCursorBase): + """Test if DictCursor extension class works.""" + def testDictConnCursorArgs(self): self.conn.close() self.conn = self.connect(connection_factory=psycopg2.extras.DictConnection) @@ -81,35 +103,6 @@ class ExtrasDictCursorTests(ConnectingTestCase): self.failUnless(row[0] == 'bar') return row - def testDictCursorWithPlainCursorRealFetchOne(self): - self._testWithPlainCursorReal(lambda curs: curs.fetchone()) - - def testDictCursorWithPlainCursorRealFetchMany(self): - self._testWithPlainCursorReal(lambda curs: curs.fetchmany(100)[0]) - - def testDictCursorWithPlainCursorRealFetchManyNoarg(self): - self._testWithPlainCursorReal(lambda curs: curs.fetchmany()[0]) - - def testDictCursorWithPlainCursorRealFetchAll(self): - self._testWithPlainCursorReal(lambda curs: curs.fetchall()[0]) - - def testDictCursorWithPlainCursorRealIter(self): - def getter(curs): - for row in curs: - return row - self._testWithPlainCursorReal(getter) - - @skip_before_postgres(8, 0) - def testDictCursorWithPlainCursorRealIterRowNumber(self): - curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - self._testIterRowNumber(curs) - - def _testWithPlainCursorReal(self, getter): - curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - curs.execute("SELECT * FROM ExtrasDictCursorTests") - row = getter(curs) - self.failUnless(row['foo'] == 'bar') - def testDictCursorWithNamedCursorFetchOne(self): self._testWithNamedCursor(lambda curs: curs.fetchone()) @@ -145,6 +138,63 @@ class ExtrasDictCursorTests(ConnectingTestCase): self.failUnless(row['foo'] == 'bar') self.failUnless(row[0] == 'bar') + def testPickleDictRow(self): + import pickle + curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs.execute("select 10 as a, 20 as b") + r = curs.fetchone() + d = pickle.dumps(r) + r1 = pickle.loads(d) + self.assertEqual(r, r1) + self.assertEqual(r[0], r1[0]) + self.assertEqual(r[1], r1[1]) + self.assertEqual(r['a'], r1['a']) + self.assertEqual(r['b'], r1['b']) + self.assertEqual(r._index, r1._index) + + +class ExtrasDictCursorRealTests(_DictCursorBase): + def testDictCursorWithPlainCursorRealFetchOne(self): + self._testWithPlainCursorReal(lambda curs: curs.fetchone()) + + def testDictCursorWithPlainCursorRealFetchMany(self): + self._testWithPlainCursorReal(lambda curs: curs.fetchmany(100)[0]) + + def testDictCursorWithPlainCursorRealFetchManyNoarg(self): + self._testWithPlainCursorReal(lambda curs: curs.fetchmany()[0]) + + def testDictCursorWithPlainCursorRealFetchAll(self): + self._testWithPlainCursorReal(lambda curs: curs.fetchall()[0]) + + def testDictCursorWithPlainCursorRealIter(self): + def getter(curs): + for row in curs: + return row + self._testWithPlainCursorReal(getter) + + @skip_before_postgres(8, 0) + def testDictCursorWithPlainCursorRealIterRowNumber(self): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + self._testIterRowNumber(curs) + + def _testWithPlainCursorReal(self, getter): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute("SELECT * FROM ExtrasDictCursorTests") + row = getter(curs) + self.failUnless(row['foo'] == 'bar') + + def testPickleRealDictRow(self): + import pickle + curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute("select 10 as a, 20 as b") + r = curs.fetchone() + d = pickle.dumps(r) + r1 = pickle.loads(d) + self.assertEqual(r, r1) + self.assertEqual(r['a'], r1['a']) + self.assertEqual(r['b'], r1['b']) + self.assertEqual(r._column_mapping, r1._column_mapping) + def testDictCursorRealWithNamedCursorFetchOne(self): self._testWithNamedCursorReal(lambda curs: curs.fetchone()) @@ -180,52 +230,6 @@ class ExtrasDictCursorTests(ConnectingTestCase): row = getter(curs) self.failUnless(row['foo'] == 'bar') - def _testNamedCursorNotGreedy(self, curs): - curs.itersize = 2 - curs.execute("""select clock_timestamp() as ts from generate_series(1,3)""") - recs = [] - for t in curs: - time.sleep(0.01) - recs.append(t) - - # check that the dataset was not fetched in a single gulp - self.assert_(recs[1]['ts'] - recs[0]['ts'] < timedelta(seconds=0.005)) - self.assert_(recs[2]['ts'] - recs[1]['ts'] > timedelta(seconds=0.0099)) - - def _testIterRowNumber(self, curs): - # Only checking for dataset < itersize: - # see CursorTests.test_iter_named_cursor_rownumber - curs.itersize = 20 - curs.execute("""select * from generate_series(1,10)""") - for i, r in enumerate(curs): - self.assertEqual(i + 1, curs.rownumber) - - def testPickleDictRow(self): - import pickle - curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - curs.execute("select 10 as a, 20 as b") - r = curs.fetchone() - d = pickle.dumps(r) - r1 = pickle.loads(d) - self.assertEqual(r, r1) - self.assertEqual(r[0], r1[0]) - self.assertEqual(r[1], r1[1]) - self.assertEqual(r['a'], r1['a']) - self.assertEqual(r['b'], r1['b']) - self.assertEqual(r._index, r1._index) - - def testPickleRealDictRow(self): - import pickle - curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - curs.execute("select 10 as a, 20 as b") - r = curs.fetchone() - d = pickle.dumps(r) - r1 = pickle.loads(d) - self.assertEqual(r, r1) - self.assertEqual(r['a'], r1['a']) - self.assertEqual(r['b'], r1['b']) - self.assertEqual(r._column_mapping, r1._column_mapping) - class NamedTupleCursorTest(ConnectingTestCase): def setUp(self): From a0f7027ad578c6c2707430e9a208cc3f7510cad4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 20 May 2018 23:36:03 +0100 Subject: [PATCH 55/55] Added tests to verify iter methods on dict cursors ISTM the refactoring in #648 broke something --- tests/test_extras_dictcursor.py | 65 ++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index 13477791..a9201f12 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -19,7 +19,8 @@ from datetime import timedelta import psycopg2 import psycopg2.extras import unittest -from .testutils import ConnectingTestCase, skip_before_postgres, skip_before_python +from .testutils import ConnectingTestCase, skip_before_postgres, \ + skip_before_python, skip_from_python class _DictCursorBase(ConnectingTestCase): @@ -152,6 +153,37 @@ class ExtrasDictCursorTests(_DictCursorBase): self.assertEqual(r['b'], r1['b']) self.assertEqual(r._index, r1._index) + @skip_from_python(3) + def test_iter_methods_2(self): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs.execute("select 10 as a, 20 as b") + r = curs.fetchone() + self.assert_(isinstance(r.keys(), list)) + self.assertEqual(len(r.keys()), 2) + self.assert_(isinstance(r.values(), tuple)) # sic? + self.assertEqual(len(r.values()), 2) + self.assert_(isinstance(r.items(), list)) + self.assertEqual(len(r.items()), 2) + + self.assert_(not isinstance(r.iterkeys(), list)) + self.assertEqual(len(list(r.iterkeys())), 2) + self.assert_(not isinstance(r.itervalues(), list)) + self.assertEqual(len(list(r.itervalues())), 2) + self.assert_(not isinstance(r.iteritems(), list)) + self.assertEqual(len(list(r.iteritems())), 2) + + @skip_before_python(3) + def test_iter_methods_3(self): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs.execute("select 10 as a, 20 as b") + r = curs.fetchone() + self.assert_(not isinstance(r.keys(), list)) + self.assertEqual(len(list(r.keys())), 2) + self.assert_(not isinstance(r.values(), list)) + self.assertEqual(len(list(r.values())), 2) + self.assert_(not isinstance(r.items(), list)) + self.assertEqual(len(list(r.items())), 2) + class ExtrasDictCursorRealTests(_DictCursorBase): def testDictCursorWithPlainCursorRealFetchOne(self): @@ -230,6 +262,37 @@ class ExtrasDictCursorRealTests(_DictCursorBase): row = getter(curs) self.failUnless(row['foo'] == 'bar') + @skip_from_python(3) + def test_iter_methods_2(self): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute("select 10 as a, 20 as b") + r = curs.fetchone() + self.assert_(isinstance(r.keys(), list)) + self.assertEqual(len(r.keys()), 2) + self.assert_(isinstance(r.values(), list)) + self.assertEqual(len(r.values()), 2) + self.assert_(isinstance(r.items(), list)) + self.assertEqual(len(r.items()), 2) + + self.assert_(not isinstance(r.iterkeys(), list)) + self.assertEqual(len(list(r.iterkeys())), 2) + self.assert_(not isinstance(r.itervalues(), list)) + self.assertEqual(len(list(r.itervalues())), 2) + self.assert_(not isinstance(r.iteritems(), list)) + self.assertEqual(len(list(r.iteritems())), 2) + + @skip_before_python(3) + def test_iter_methods_3(self): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute("select 10 as a, 20 as b") + r = curs.fetchone() + self.assert_(not isinstance(r.keys(), list)) + self.assertEqual(len(list(r.keys())), 2) + self.assert_(not isinstance(r.values(), list)) + self.assertEqual(len(list(r.values())), 2) + self.assert_(not isinstance(r.items(), list)) + self.assertEqual(len(list(r.items())), 2) + class NamedTupleCursorTest(ConnectingTestCase): def setUp(self):