From 4bb6f9cef2ac2631b2af881d945b770b64fa23bd Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 18:05:11 +0200 Subject: [PATCH 01/13] Add libpq version discovery --- doc/src/extensions.rst | 10 ++++++++++ doc/src/module.rst | 6 ++++++ lib/__init__.py | 2 +- lib/extensions.py | 2 +- psycopg/psycopgmodule.c | 16 ++++++++++++++++ setup.py | 3 +++ tests/testutils.py | 37 +++++++++++++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 2 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index dea10417..3f010665 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -197,6 +197,16 @@ functionalities defined by the |DBAPI|_. .. versionadded:: 2.2.0 +.. function:: libpq_version() + + Query actual ``libpq`` version loaded. + + This function throws `NotSupportedError` if it was compiled with + ``libpq < 9.1``. + + .. seealso:: libpq docs for `PQlibVersion()`__ + + .. __: http://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQLIBVERSION .. _sql-adaptation-objects: diff --git a/doc/src/module.rst b/doc/src/module.rst index 8de9f87e..bd6bcf45 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -109,6 +109,12 @@ The module interface respects the standard defined in the |DBAPI|_. by the interface. For `psycopg2` is ``pyformat``. See also :ref:`query-parameters`. +.. data:: __libpq_version__ + + Integer contant containing the version of ``libpq`` this `psycopg2` + module was compiled with. If this value is ``>= 90100`` then you + may query for the actually loaded version of libpq using + `~psycopg2.extensions.libpq_version()`. .. index:: diff --git a/lib/__init__.py b/lib/__init__.py index cf8c06ae..994b15a8 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -57,7 +57,7 @@ from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle -from psycopg2._psycopg import __version__ +from psycopg2._psycopg import __version__, __libpq_version__ from psycopg2 import tz diff --git a/lib/extensions.py b/lib/extensions.py index 216d8ad2..c40e3369 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -56,7 +56,7 @@ try: except ImportError: pass -from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid +from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid, libpq_version from psycopg2._psycopg import string_types, binary_types, new_type, new_array_type, register_type from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics, Column diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 61e2de57..63abb03d 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -300,6 +300,19 @@ exit: return rv; } +#define psyco_libpq_version_doc "Query actual libpq version loaded." + +static PyObject* +psyco_libpq_version(PyObject *self) +{ +#if PG_VERSION_HEX >= 0x090100 + return PyInt_FromLong(PQlibVersion()); +#else + PyErr_SetString(NotSupportedError, "version discovery is not supported in libpq < 9.1"); + return NULL; +#endif +} + /* psyco_encodings_fill Fill the module's postgresql<->python encoding table */ @@ -704,6 +717,8 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS|METH_KEYWORDS, typecast_from_python_doc}, {"new_array_type", (PyCFunction)typecast_array_from_python, METH_VARARGS|METH_KEYWORDS, typecast_array_from_python_doc}, + {"libpq_version", (PyCFunction)psyco_libpq_version, + METH_NOARGS, psyco_libpq_version_doc}, {"Date", (PyCFunction)psyco_Date, METH_VARARGS, psyco_Date_doc}, @@ -899,6 +914,7 @@ INIT_MODULE(_psycopg)(void) /* set some module's parameters */ PyModule_AddStringConstant(module, "__version__", PSYCOPG_VERSION); PyModule_AddStringConstant(module, "__doc__", "psycopg PostgreSQL driver"); + PyModule_AddIntConstant(module, "__libpq_version__", PG_VERSION_NUM); PyModule_AddObject(module, "apilevel", Text_FromUTF8(APILEVEL)); PyModule_AddObject(module, "threadsafety", PyInt_FromLong(THREADSAFETY)); PyModule_AddObject(module, "paramstyle", Text_FromUTF8(PARAMSTYLE)); diff --git a/setup.py b/setup.py index fc4f1711..e42a5c1a 100644 --- a/setup.py +++ b/setup.py @@ -416,6 +416,9 @@ class psycopg_build_ext(build_ext): % pgversion) sys.exit(1) + define_macros.append(("PG_VERSION_NUM", "%d%02d%02d" % + (pgmajor, pgminor, pgpatch))) + define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % (pgmajor, pgminor, pgpatch))) diff --git a/tests/testutils.py b/tests/testutils.py index 6a784320..987bd7b6 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -236,6 +236,43 @@ def skip_after_postgres(*ver): return skip_after_postgres__ return skip_after_postgres_ +def libpq_version(): + import psycopg2 + v = psycopg2.__libpq_version__ + if v >= 90100: + v = psycopg2.extensions.libpq_version() + return v + +def skip_before_libpq(*ver): + """Skip a test if libpq we're linked to is older than a certain version.""" + ver = ver + (0,) * (3 - len(ver)) + def skip_before_libpq_(f): + @wraps(f) + def skip_before_libpq__(self): + v = libpq_version() + if v < int("%d%02d%02d" % ver): + return self.skipTest("skipped because libpq %d" % v) + else: + return f(self) + + return skip_before_libpq__ + return skip_before_libpq_ + +def skip_after_libpq(*ver): + """Skip a test if libpq we're linked to is newer than a certain version.""" + ver = ver + (0,) * (3 - len(ver)) + def skip_after_libpq_(f): + @wraps(f) + def skip_after_libpq__(self): + v = libpq_version() + if v >= int("%d%02d%02d" % ver): + return self.skipTest("skipped because libpq %s" % v) + else: + return f(self) + + return skip_after_libpq__ + return skip_after_libpq_ + def skip_before_python(*ver): """Skip a test on Python before a certain version.""" def skip_before_python_(f): From ffd98a82c04642b73d76a4e60f8f58c355ae1126 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 2 Jun 2015 11:12:16 +0200 Subject: [PATCH 02/13] Add test for libpq_version --- tests/test_module.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_module.py b/tests/test_module.py index 608f703d..62b85ee2 100755 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -320,6 +320,15 @@ import _psycopg self.assertEqual(0, proc.returncode) +class TestVersionDiscovery(unittest.TestCase): + def test_libpq_version(self): + self.assertTrue(type(psycopg2.__libpq_version__) is int) + try: + self.assertTrue(type(psycopg2.extensions.libpq_version()) is int) + except NotSupportedError: + self.assertTrue(psycopg2.__libpq_version__ < 90100) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 73d17e3c5e80c66738435cd6733e999cb9c21ca7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 10:54:08 +0100 Subject: [PATCH 03/13] Dropped PG_VERSION_HEX constant At PostgreSQL 10.0 it would have become awkward. --- psycopg/adapter_binary.c | 2 +- psycopg/lobject_int.c | 4 ++-- psycopg/lobject_type.c | 6 +++--- psycopg/psycopgmodule.c | 4 ++-- psycopg/utils.c | 2 +- setup.py | 3 --- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 485dc5a4..597048d2 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -39,7 +39,7 @@ static unsigned char * binary_escape(unsigned char *from, size_t from_length, size_t *to_length, PGconn *conn) { -#if PG_VERSION_HEX >= 0x080104 +#if PG_VERSION_NUM >= 80104 if (conn) return PQescapeByteaConn(conn, from, from_length, to_length); else diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 6b55d42b..8788c100 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -474,7 +474,7 @@ lobject_export(lobjectObject *self, const char *filename) return retvalue; } -#if PG_VERSION_HEX >= 0x080300 +#if PG_VERSION_NUM >= 80300 RAISES_NEG int lobject_truncate(lobjectObject *self, size_t len) @@ -511,4 +511,4 @@ lobject_truncate(lobjectObject *self, size_t len) } -#endif /* PG_VERSION_HEX >= 0x080300 */ +#endif /* PG_VERSION_NUM >= 80300 */ diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index ec95b5cf..a43325d4 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -266,7 +266,7 @@ psyco_lobj_get_closed(lobjectObject *self, void *closure) return closed; } -#if PG_VERSION_HEX >= 0x080300 +#if PG_VERSION_NUM >= 80300 #define psyco_lobj_truncate_doc \ "truncate(len=0) -- Truncate large object to given size." @@ -327,10 +327,10 @@ static struct PyMethodDef lobjectObject_methods[] = { METH_NOARGS, psyco_lobj_unlink_doc}, {"export",(PyCFunction)psyco_lobj_export, METH_VARARGS, psyco_lobj_export_doc}, -#if PG_VERSION_HEX >= 0x080300 +#if PG_VERSION_NUM >= 80300 {"truncate",(PyCFunction)psyco_lobj_truncate, METH_VARARGS, psyco_lobj_truncate_doc}, -#endif /* PG_VERSION_HEX >= 0x080300 */ +#endif /* PG_VERSION_NUM >= 80300 */ {NULL} }; diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 63abb03d..34fc25e8 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -185,7 +185,7 @@ psyco_libcrypto_threads_init(void) if (PyImport_ImportModule("ssl") != NULL) { /* disable libcrypto setup in libpq, so it won't stomp on the callbacks that have already been set up */ -#if PG_VERSION_HEX >= 0x080400 +#if PG_VERSION_NUM >= 80400 PQinitOpenSSL(1, 0); #endif } @@ -305,7 +305,7 @@ exit: static PyObject* psyco_libpq_version(PyObject *self) { -#if PG_VERSION_HEX >= 0x090100 +#if PG_VERSION_NUM >= 90100 return PyInt_FromLong(PQlibVersion()); #else PyErr_SetString(NotSupportedError, "version discovery is not supported in libpq < 9.1"); diff --git a/psycopg/utils.c b/psycopg/utils.c index 6b035cfa..836f6129 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -62,7 +62,7 @@ psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, } { - #if PG_VERSION_HEX >= 0x080104 + #if PG_VERSION_NUM >= 80104 int err; if (conn && conn->pgconn) ql = PQescapeStringConn(conn->pgconn, to+eq+1, from, len, &err); diff --git a/setup.py b/setup.py index e42a5c1a..2de8c5ef 100644 --- a/setup.py +++ b/setup.py @@ -419,9 +419,6 @@ class psycopg_build_ext(build_ext): define_macros.append(("PG_VERSION_NUM", "%d%02d%02d" % (pgmajor, pgminor, pgpatch))) - define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % - (pgmajor, pgminor, pgpatch))) - # enable lo64 if libpq >= 9.3 and Python 64 bits if (pgmajor, pgminor) >= (9, 3) and is_py_64(): define_macros.append(("HAVE_LO64", "1")) From 5a21da43ee240d66d5fdefd3476a5329e6d730a9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 11:01:10 +0100 Subject: [PATCH 04/13] Mention libpq version inspection in news file --- NEWS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NEWS b/NEWS index 3e8864f0..de5ead30 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,17 @@ Current release --------------- +What's new in psycopg 2.7 +------------------------- + +New features: + +- Added `~psycopg2.__libpq_version__` and + `~psycopg2.extensions.libpq_version()` to inspect the version of the + ``libpq`` library the module was compiled/loaded with + (:tickets:`#35, #323`). + + What's new in psycopg 2.6.1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From c2955fb8fc95afd288af22444be8fa11e4132717 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 11:14:22 +0100 Subject: [PATCH 05/13] Version function/constant docs improved --- doc/src/extensions.rst | 10 ++++++---- doc/src/module.rst | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index 3f010665..84e12412 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -199,12 +199,14 @@ functionalities defined by the |DBAPI|_. .. function:: libpq_version() - Query actual ``libpq`` version loaded. + Return the version number of the ``libpq`` dynamic library loaded as an + integer, in the same format of `~connection.server_version`. - This function throws `NotSupportedError` if it was compiled with - ``libpq < 9.1``. + Raise `~psycopg2.NotSupportedError` if the ``psycopg2`` module was + compiled with a ``libpq`` version lesser than 9.1 (which can be detected + by the `~psycopg2.__libpq_version__` constant). - .. seealso:: libpq docs for `PQlibVersion()`__ + .. seealso:: libpq docs for `PQlibVersion()`__. .. __: http://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQLIBVERSION diff --git a/doc/src/module.rst b/doc/src/module.rst index bd6bcf45..bd121e9d 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -111,10 +111,11 @@ The module interface respects the standard defined in the |DBAPI|_. .. data:: __libpq_version__ - Integer contant containing the version of ``libpq`` this `psycopg2` - module was compiled with. If this value is ``>= 90100`` then you - may query for the actually loaded version of libpq using - `~psycopg2.extensions.libpq_version()`. + Integer constant reporting the version of the ``libpq`` library this + ``psycopg2`` module was compiled with (in the same format of + `~connection.server_version`). If this value is lesser than ``90100`` + then you may query the version of the actually loaded library using the + `~psycopg2.extensions.libpq_version()` function. .. index:: From b0058c0cc85a0e9ce88ccacef0bc34a26d8e5d89 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 12:38:59 +0100 Subject: [PATCH 06/13] Fixed adaptation of lists of None Note: lists of lists of None are not supported yet. --- NEWS | 1 + psycopg/adapter_list.c | 16 +++++++++++++++- tests/test_types_basic.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index de5ead30..4aebbff3 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,7 @@ New features: What's new in psycopg 2.6.1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Lists consisting of only `None` are escaped correctly (:ticket:`#285`). - Fixed deadlock in multithread programs using OpenSSL (:ticket:`#290`). - Correctly unlock the connection after error in flush (:ticket:`#294`). - Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`). diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index e68b1978..dec17b4c 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -39,6 +39,14 @@ list_quote(listObject *self) /* adapt the list by calling adapt() recursively and then wrapping everything into "ARRAY[]" */ PyObject *tmp = NULL, *str = NULL, *joined = NULL, *res = NULL; + + /* list consisting of only NULL don't work with the ARRAY[] construct + * so we use the {NULL,...} syntax. Note however that list of lists where + * some element is a list of only null still fails: for that we should use + * the '{...}' syntax uniformly but we cannot do it in the current + * infrastructure. TODO in psycopg3 */ + int all_nulls = 1; + Py_ssize_t i, len; len = PyList_GET_SIZE(self->wrapped); @@ -60,6 +68,7 @@ list_quote(listObject *self) quoted = microprotocol_getquoted(wrapped, (connectionObject*)self->connection); if (quoted == NULL) goto error; + all_nulls = 0; } /* here we don't loose a refcnt: SET_ITEM does not change the @@ -74,7 +83,12 @@ list_quote(listObject *self) joined = PyObject_CallMethod(str, "join", "(O)", tmp); if (joined == NULL) goto error; - res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined)); + /* PG doesn't like ARRAY[NULL..] */ + if (!all_nulls) { + res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined)); + } else { + res = Bytes_FromFormat("'{%s}'", Bytes_AsString(joined)); + } error: Py_XDECREF(tmp); diff --git a/tests/test_types_basic.py b/tests/test_types_basic.py index 6c4cc970..199dc1b6 100755 --- a/tests/test_types_basic.py +++ b/tests/test_types_basic.py @@ -192,6 +192,40 @@ class TypesBasicTests(ConnectingTestCase): self.assertRaises(psycopg2.DataError, psycopg2.extensions.STRINGARRAY, b(s), curs) + def testArrayOfNulls(self): + curs = self.conn.cursor() + curs.execute(""" + create table na ( + texta text[], + inta int[], + boola boolean[], + + textaa text[][], + intaa int[][], + boolaa boolean[][] + )""") + + curs.execute("insert into na (texta) values (%s)", ([None],)) + curs.execute("insert into na (texta) values (%s)", (['a', None],)) + curs.execute("insert into na (texta) values (%s)", ([None, None],)) + curs.execute("insert into na (inta) values (%s)", ([None],)) + curs.execute("insert into na (inta) values (%s)", ([42, None],)) + curs.execute("insert into na (inta) values (%s)", ([None, None],)) + curs.execute("insert into na (boola) values (%s)", ([None],)) + curs.execute("insert into na (boola) values (%s)", ([True, None],)) + curs.execute("insert into na (boola) values (%s)", ([None, None],)) + + # TODO: array of array of nulls are not supported yet + # curs.execute("insert into na (textaa) values (%s)", ([[None]],)) + curs.execute("insert into na (textaa) values (%s)", ([['a', None]],)) + # curs.execute("insert into na (textaa) values (%s)", ([[None, None]],)) + # curs.execute("insert into na (intaa) values (%s)", ([[None]],)) + curs.execute("insert into na (intaa) values (%s)", ([[42, None]],)) + # curs.execute("insert into na (intaa) values (%s)", ([[None, None]],)) + # curs.execute("insert into na (boolaa) values (%s)", ([[None]],)) + curs.execute("insert into na (boolaa) values (%s)", ([[True, None]],)) + # curs.execute("insert into na (boolaa) values (%s)", ([[None, None]],)) + @testutils.skip_from_python(3) def testTypeRoundtripBuffer(self): o1 = buffer("".join(map(chr, range(256)))) From 5712f3016970948d17df8b790a2146d1c166c032 Mon Sep 17 00:00:00 2001 From: Pete Hollobon Date: Tue, 19 May 2015 14:06:43 +0100 Subject: [PATCH 07/13] Change "non desiderable" to "undesirable" in docs "desiderable" is considered obsolete --- doc/src/connection.rst | 4 ++-- doc/src/usage.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 07e494a2..92178f34 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -419,8 +419,8 @@ The ``connection`` class By default, any query execution, including a simple :sql:`SELECT` will start a transaction: for long-running programs, if no further - action is taken, the session will remain "idle in transaction", a - condition non desiderable for several reasons (locks are held by + action is taken, the session will remain "idle in transaction", an + undesirable condition for several reasons (locks are held by the session, tables bloat...). For long lived scripts, either ensure to terminate a transaction as soon as possible or use an autocommit connection. diff --git a/doc/src/usage.rst b/doc/src/usage.rst index e83b1280..9dd31df2 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -679,7 +679,7 @@ older versions). By default even a simple :sql:`SELECT` will start a transaction: in long-running programs, if no further action is taken, the session will - remain "idle in transaction", a condition non desiderable for several + remain "idle in transaction", an undesirable condition for several reasons (locks are held by the session, tables bloat...). For long lived scripts, either make sure to terminate a transaction as soon as possible or use an autocommit connection. From 0a7261268bcfd225cbd66a3b93a730d69a32f047 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 13:11:46 +0100 Subject: [PATCH 08/13] Dropped unused notice_filter connection member --- psycopg/connection.h | 1 - psycopg/connection_type.c | 2 -- 2 files changed, 3 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index 07dfe2e7..c52abc9f 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -106,7 +106,6 @@ struct connectionObject { /* notice processing */ PyObject *notice_list; - PyObject *notice_filter; struct connectionObject_notice *notice_pending; /* notifies */ diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 43abe8a3..9931399b 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -1105,7 +1105,6 @@ connection_clear(connectionObject *self) Py_CLEAR(self->tpc_xid); Py_CLEAR(self->async_cursor); Py_CLEAR(self->notice_list); - Py_CLEAR(self->notice_filter); Py_CLEAR(self->notifies); Py_CLEAR(self->string_types); Py_CLEAR(self->binary_types); @@ -1181,7 +1180,6 @@ connection_traverse(connectionObject *self, visitproc visit, void *arg) Py_VISIT((PyObject *)(self->tpc_xid)); Py_VISIT(self->async_cursor); Py_VISIT(self->notice_list); - Py_VISIT(self->notice_filter); Py_VISIT(self->notifies); Py_VISIT(self->string_types); Py_VISIT(self->binary_types); From b326a277743b80dde7e5263a232db8df99149164 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 14:24:48 +0100 Subject: [PATCH 09/13] Dropped unneeded constness on the notice message That's a strdup result, we 0wn it. --- psycopg/connection.h | 2 +- psycopg/connection_int.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index c52abc9f..d15c9c64 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -71,7 +71,7 @@ extern HIDDEN PyTypeObject connectionType; struct connectionObject_notice { struct connectionObject_notice *next; - const char *message; + char *message; }; /* the typedef is forward-declared in psycopg.h */ diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index aea2841c..8fce908d 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -154,7 +154,7 @@ conn_notice_clean(connectionObject *self) while (notice != NULL) { tmp = notice; notice = notice->next; - free((void*)tmp->message); + free(tmp->message); free(tmp); } From 2ad82b973b86fa71126657aacee89a30d2211894 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 14:25:46 +0100 Subject: [PATCH 10/13] Pending notice list converted into a forward list This allows inserting the elements in order without using list.insert(). --- psycopg/connection.h | 1 + psycopg/connection_int.c | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index d15c9c64..ec107429 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -107,6 +107,7 @@ struct connectionObject { /* notice processing */ PyObject *notice_list; struct connectionObject_notice *notice_pending; + struct connectionObject_notice *last_notice; /* notifies */ PyObject *notifies; diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 8fce908d..40f7e6ca 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -87,13 +87,20 @@ conn_notice_callback(void *args, const char *message) /* Discard the notice in case of failed allocation. */ return; } + notice->next = NULL; notice->message = strdup(message); if (NULL == notice->message) { free(notice); return; } - notice->next = self->notice_pending; - self->notice_pending = notice; + + if (NULL == self->last_notice) { + self->notice_pending = self->last_notice = notice; + } + else { + self->last_notice->next = notice; + self->last_notice = notice; + } } /* Expose the notices received as Python objects. @@ -111,17 +118,14 @@ conn_notice_process(connectionObject *self) } notice = self->notice_pending; - nnotices = PyList_GET_SIZE(self->notice_list); while (notice != NULL) { PyObject *msg; msg = conn_text_from_chars(self, notice->message); Dprintf("conn_notice_process: %s", notice->message); - /* Respect the order in which notices were produced, - because in notice_list they are reversed (see ticket #9) */ if (msg) { - PyList_Insert(self->notice_list, nnotices, msg); + PyList_Append(self->notice_list, msg); Py_DECREF(msg); } else { @@ -158,7 +162,7 @@ conn_notice_clean(connectionObject *self) free(tmp); } - self->notice_pending = NULL; + self->last_notice = self->notice_pending = NULL; } From 1f330e9cac9c5d40c33f4f58d0dbfc0109c62edc Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 17:02:04 +0100 Subject: [PATCH 11/13] Allow connection.notices and notifies to be replaced. Close #326 --- NEWS | 3 ++ doc/src/advanced.rst | 4 +++ doc/src/connection.rst | 18 ++++++++-- psycopg/connection_int.c | 69 +++++++++++++++++++++++++++++---------- psycopg/connection_type.c | 4 +-- tests/test_connection.py | 36 ++++++++++++++++++++ tests/test_notify.py | 22 +++++++++++++ 7 files changed, 134 insertions(+), 22 deletions(-) diff --git a/NEWS b/NEWS index 4aebbff3..fe6cea41 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,9 @@ New features: `~psycopg2.extensions.libpq_version()` to inspect the version of the ``libpq`` library the module was compiled/loaded with (:tickets:`#35, #323`). +- The attributes `~connection.notices` and `~connection.notifies` can be + customized replacing them with any object exposing an `!append()` method + (:ticket:`#326`). What's new in psycopg 2.6.1 diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index eecbcfda..f0483cea 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -312,6 +312,10 @@ received from a previous version server will have the Added `~psycopg2.extensions.Notify` object and handling notification payload. +.. versionchanged:: 2.7 + The `~connection.notifies` attribute is writable: it is possible to + replace it with any object exposing an `!append()` method. An useful + example would be to use a `~collections.deque` object. .. index:: diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 92178f34..cceef1e5 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -483,13 +483,21 @@ The ``connection`` class ['NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "foo_pkey" for table "foo"\n', 'NOTICE: CREATE TABLE will create implicit sequence "foo_id_seq" for serial column "foo.id"\n'] + .. versionchanged:: 2.7 + The `!notices` attribute is writable: the user may replace it + with any Python object exposing an `!append()` method. If + appending raises an exception the notice is silently + dropped. + To avoid a leak in case excessive notices are generated, only the last - 50 messages are kept. + 50 messages are kept. This check is only in place if the `!notices` + attribute is a list: if any other object is used it will be up to the + user to guard from leakage. You can configure what messages to receive using `PostgreSQL logging configuration parameters`__ such as ``log_statement``, ``client_min_messages``, ``log_min_duration_statement`` etc. - + .. __: http://www.postgresql.org/docs/current/static/runtime-config-logging.html @@ -506,6 +514,12 @@ The ``connection`` class the payload was not accessible. To keep backward compatibility, `!Notify` objects can still be accessed as 2 items tuples. + .. versionchanged:: 2.7 + The `!notifies` attribute is writable: the user may replace it + with any Python object exposing an `!append()` method. If + appending raises an exception the notification is silently + dropped. + .. attribute:: cursor_factory diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 40f7e6ca..43d0fdae 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -111,41 +111,60 @@ void conn_notice_process(connectionObject *self) { struct connectionObject_notice *notice; - Py_ssize_t nnotices; + PyObject *msg = NULL; + PyObject *tmp = NULL; + static PyObject *append; if (NULL == self->notice_pending) { return; } - notice = self->notice_pending; + if (!append) { + if (!(append = Text_FromUTF8("append"))) { + goto error; + } + } + notice = self->notice_pending; while (notice != NULL) { - PyObject *msg; - msg = conn_text_from_chars(self, notice->message); Dprintf("conn_notice_process: %s", notice->message); - if (msg) { - PyList_Append(self->notice_list, msg); - Py_DECREF(msg); - } - else { - /* We don't really have a way to report errors, so gulp it. - * The function should only fail for out of memory, so we are - * likely going to die anyway. */ - PyErr_Clear(); + if (!(msg = conn_text_from_chars(self, notice->message))) { goto error; } + + if (!(tmp = PyObject_CallMethodObjArgs( + self->notice_list, append, msg, NULL))) { + + goto error; } + Py_DECREF(tmp); tmp = NULL; + Py_DECREF(msg); msg = NULL; + notice = notice->next; } /* Remove the oldest item if the queue is getting too long. */ - nnotices = PyList_GET_SIZE(self->notice_list); - if (nnotices > CONN_NOTICES_LIMIT) { - PySequence_DelSlice(self->notice_list, - 0, nnotices - CONN_NOTICES_LIMIT); + if (PyList_Check(self->notice_list)) { + Py_ssize_t nnotices; + nnotices = PyList_GET_SIZE(self->notice_list); + if (nnotices > CONN_NOTICES_LIMIT) { + if (-1 == PySequence_DelSlice(self->notice_list, + 0, nnotices - CONN_NOTICES_LIMIT)) { + PyErr_Clear(); + } + } } conn_notice_clean(self); + return; + +error: + Py_XDECREF(tmp); + Py_XDECREF(msg); + conn_notice_clean(self); + + /* TODO: the caller doesn't expects errors from us */ + PyErr_Clear(); } void @@ -177,6 +196,15 @@ conn_notifies_process(connectionObject *self) PGnotify *pgn = NULL; PyObject *notify = NULL; PyObject *pid = NULL, *channel = NULL, *payload = NULL; + PyObject *tmp = NULL; + + static PyObject *append; + + if (!append) { + if (!(append = Text_FromUTF8("append"))) { + goto error; + } + } while ((pgn = PQnotifies(self->pgconn)) != NULL) { @@ -196,7 +224,11 @@ conn_notifies_process(connectionObject *self) Py_DECREF(channel); channel = NULL; Py_DECREF(payload); payload = NULL; - PyList_Append(self->notifies, (PyObject *)notify); + if (!(tmp = PyObject_CallMethodObjArgs( + self->notifies, append, notify, NULL))) { + goto error; + } + Py_DECREF(tmp); tmp = NULL; Py_DECREF(notify); notify = NULL; PQfreemem(pgn); pgn = NULL; @@ -205,6 +237,7 @@ conn_notifies_process(connectionObject *self) error: if (pgn) { PQfreemem(pgn); } + Py_XDECREF(tmp); Py_XDECREF(notify); Py_XDECREF(pid); Py_XDECREF(channel); diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 9931399b..2c1dddf2 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -1001,8 +1001,8 @@ static struct PyMemberDef connectionObject_members[] = { "True if the connection is closed."}, {"encoding", T_STRING, offsetof(connectionObject, encoding), READONLY, "The current client encoding."}, - {"notices", T_OBJECT, offsetof(connectionObject, notice_list), READONLY}, - {"notifies", T_OBJECT, offsetof(connectionObject, notifies), READONLY}, + {"notices", T_OBJECT, offsetof(connectionObject, notice_list), 0}, + {"notifies", T_OBJECT, offsetof(connectionObject, notifies), 0}, {"dsn", T_STRING, offsetof(connectionObject, dsn), READONLY, "The current connection string."}, {"async", T_LONG, offsetof(connectionObject, async), READONLY, diff --git a/tests/test_connection.py b/tests/test_connection.py index 340693e2..fa78eb37 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -129,6 +129,42 @@ class ConnectionTests(ConnectingTestCase): self.assertEqual(50, len(conn.notices)) self.assert_('table99' in conn.notices[-1], conn.notices[-1]) + def test_notices_deque(self): + from collections import deque + + conn = self.conn + self.conn.notices = deque() + cur = conn.cursor() + if self.conn.server_version >= 90300: + cur.execute("set client_min_messages=debug1") + + cur.execute("create temp table table1 (id serial); create temp table table2 (id serial);") + cur.execute("create temp table table3 (id serial); create temp table table4 (id serial);") + self.assertEqual(len(conn.notices), 4) + self.assert_('table1' in conn.notices.popleft()) + self.assert_('table2' in conn.notices.popleft()) + self.assert_('table3' in conn.notices.popleft()) + self.assert_('table4' in conn.notices.popleft()) + self.assertEqual(len(conn.notices), 0) + + # not limited, but no error + for i in range(0, 100, 10): + sql = " ".join(["create temp table table2_%d (id serial);" % j for j in range(i, i+10)]) + cur.execute(sql) + + self.assertEqual(100, len(conn.notices)) + + def test_notices_noappend(self): + conn = self.conn + self.conn.notices = None # will make an error swallowes ok + cur = conn.cursor() + if self.conn.server_version >= 90300: + cur.execute("set client_min_messages=debug1") + + cur.execute("create temp table table1 (id serial);") + + self.assertEqual(self.conn.notices, None) + def test_server_version(self): self.assert_(self.conn.server_version) diff --git a/tests/test_notify.py b/tests/test_notify.py index f8383899..fc6224d7 100755 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -155,6 +155,27 @@ conn.close() self.assertEqual('foo', notify.channel) self.assertEqual('Hello, world!', notify.payload) + def test_notify_deque(self): + from collections import deque + self.autocommit(self.conn) + self.conn.notifies = deque() + self.listen('foo') + self.notify('foo').communicate() + time.sleep(0.5) + self.conn.poll() + notify = self.conn.notifies.popleft() + self.assert_(isinstance(notify, psycopg2.extensions.Notify)) + self.assertEqual(len(self.conn.notifies), 0) + + def test_notify_noappend(self): + self.autocommit(self.conn) + self.conn.notifies = None + self.listen('foo') + self.notify('foo').communicate() + time.sleep(0.5) + self.conn.poll() + self.assertEqual(self.conn.notifies, None) + def test_notify_init(self): n = psycopg2.extensions.Notify(10, 'foo') self.assertEqual(10, n.pid) @@ -192,6 +213,7 @@ conn.close() self.assertNotEqual(hash(Notify(10, 'foo', 'bar')), hash(Notify(10, 'foo'))) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 06b4b1de94a1ae9365b94069f7680f87e5e9b87b Mon Sep 17 00:00:00 2001 From: btubbs Date: Thu, 21 May 2015 01:24:00 -0600 Subject: [PATCH 12/13] Notify example should pop the oldest message in conn.notifies, not the newest. --- doc/src/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index f0483cea..82754ee0 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -291,7 +291,7 @@ something to read:: else: conn.poll() while conn.notifies: - notify = conn.notifies.pop() + notify = conn.notifies.pop(0) print "Got NOTIFY:", notify.pid, notify.channel, notify.payload Running the script and executing a command such as :sql:`NOTIFY test, 'hello'` From 925fdf57311b7168202e83d3d8e6040b986e5691 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 17:11:09 +0100 Subject: [PATCH 13/13] Fixed doc about libpq version availability --- doc/src/module.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/module.rst b/doc/src/module.rst index bd121e9d..7f8a29b6 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -113,9 +113,9 @@ The module interface respects the standard defined in the |DBAPI|_. Integer constant reporting the version of the ``libpq`` library this ``psycopg2`` module was compiled with (in the same format of - `~connection.server_version`). If this value is lesser than ``90100`` - then you may query the version of the actually loaded library using the - `~psycopg2.extensions.libpq_version()` function. + `~connection.server_version`). If this value is greater or equal than + ``90100`` then you may query the version of the actually loaded library + using the `~psycopg2.extensions.libpq_version()` function. .. index::