Merge branch 'master' into parse-dsn

NEWS

@@ -10,11 +10,23 @@ New features:
   `~psycopg2.extensions.libpq_version()` to inspect the version of the
   ``libpq`` library the module was compiled/loaded with
   (:tickets:`#35, #323`).
+- The attributes `~connection.notices` and `~connection.notifies` can be
+  customized replacing them with any object exposing an `!append()` method
+  (:ticket:`#326`).
+
+
+What's new in psycopg 2.6.2
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+- Report the server response status on errors (such as :ticket:`#281`).
+- Raise NotSupportedError on unhandled server response status
+  (:ticket:`#352`).
 
 
 What's new in psycopg 2.6.1
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Lists consisting of only `None` are escaped correctly (:ticket:`#285`).
 - Fixed deadlock in multithread programs using OpenSSL (:ticket:`#290`).
 - Correctly unlock the connection after error in flush (:ticket:`#294`).
 - Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`).

doc/src/advanced.rst

@@ -291,7 +291,7 @@ something to read::
         else:
             conn.poll()
             while conn.notifies:
-                notify = conn.notifies.pop()
+                notify = conn.notifies.pop(0)
                 print "Got NOTIFY:", notify.pid, notify.channel, notify.payload
 
 Running the script and executing a command such as :sql:`NOTIFY test, 'hello'`
@@ -312,6 +312,10 @@ received from a previous version server will have the
     Added `~psycopg2.extensions.Notify` object and handling notification
     payload.
 
+.. versionchanged:: 2.7
+    The `~connection.notifies` attribute is writable: it is possible to
+    replace it with any object exposing an `!append()` method. An useful
+    example would be to use a `~collections.deque` object.
 
 
 .. index::

doc/src/connection.rst

@@ -419,8 +419,8 @@ The ``connection`` class
 
             By default, any query execution, including a simple :sql:`SELECT`
             will start a transaction: for long-running programs, if no further
-            action is taken, the session will remain "idle in transaction", a
+            action is taken, the session will remain "idle in transaction", an
-            condition non desiderable for several reasons (locks are held by
+            undesirable condition for several reasons (locks are held by
             the session, tables bloat...). For long lived scripts, either
             ensure to terminate a transaction as soon as possible or use an
             autocommit connection.
@@ -483,8 +483,16 @@ The ``connection`` class
             ['NOTICE:  CREATE TABLE / PRIMARY KEY will create implicit index "foo_pkey" for table "foo"\n',
              'NOTICE:  CREATE TABLE will create implicit sequence "foo_id_seq" for serial column "foo.id"\n']
 
+        .. versionchanged:: 2.7
+            The `!notices` attribute is writable: the user may replace it
+            with any Python object exposing an `!append()` method. If
+            appending raises an exception the notice is silently
+            dropped.
+
         To avoid a leak in case excessive notices are generated, only the last
-        50 messages are kept.
+        50 messages are kept. This check is only in place if the `!notices`
+        attribute is a list: if any other object is used it will be up to the
+        user to guard from leakage.
 
         You can configure what messages to receive using `PostgreSQL logging
         configuration parameters`__ such as ``log_statement``,
@@ -506,6 +514,12 @@ The ``connection`` class
             the payload was not accessible. To keep backward compatibility,
             `!Notify` objects can still be accessed as 2 items tuples.
 
+        .. versionchanged:: 2.7
+            The `!notifies` attribute is writable: the user may replace it
+            with any Python object exposing an `!append()` method. If
+            appending raises an exception the notification is silently
+            dropped.
+
 
     .. attribute:: cursor_factory
 

doc/src/module.rst

@@ -113,9 +113,9 @@ The module interface respects the standard defined in the |DBAPI|_.
 
    Integer constant reporting the version of the ``libpq`` library this
    ``psycopg2`` module was compiled with (in the same format of
-   `~connection.server_version`).  If this value is lesser than ``90100``
+   `~connection.server_version`).  If this value is greater or equal than
-   then you may query the version of the actually loaded library using the
+   ``90100`` then you may query the version of the actually loaded library
-   `~psycopg2.extensions.libpq_version()` function.
+   using the `~psycopg2.extensions.libpq_version()` function.
 
 
 .. index::

doc/src/usage.rst

@@ -679,7 +679,7 @@ older versions).
 
     By default even a simple :sql:`SELECT` will start a transaction: in
     long-running programs, if no further action is taken, the session will
-    remain "idle in transaction", a condition non desiderable for several
+    remain "idle in transaction", an undesirable condition for several
     reasons (locks are held by the session, tables bloat...). For long lived
     scripts, either make sure to terminate a transaction as soon as possible or
     use an autocommit connection.

psycopg/adapter_list.c

@@ -39,6 +39,14 @@ list_quote(listObject *self)
     /*  adapt the list by calling adapt() recursively and then wrapping
         everything into "ARRAY[]" */
     PyObject *tmp = NULL, *str = NULL, *joined = NULL, *res = NULL;
+
+    /*  list consisting of only NULL don't work with the ARRAY[] construct
+     *  so we use the {NULL,...} syntax. Note however that list of lists where
+     *  some element is a list of only null still fails: for that we should use
+     *  the '{...}' syntax uniformly but we cannot do it in the current
+     *  infrastructure. TODO in psycopg3 */
+    int all_nulls = 1;
+
     Py_ssize_t i, len;
 
     len = PyList_GET_SIZE(self->wrapped);
@@ -60,6 +68,7 @@ list_quote(listObject *self)
             quoted = microprotocol_getquoted(wrapped,
                                        (connectionObject*)self->connection);
             if (quoted == NULL) goto error;
+            all_nulls = 0;
         }
 
         /* here we don't loose a refcnt: SET_ITEM does not change the
@@ -74,7 +83,12 @@ list_quote(listObject *self)
     joined = PyObject_CallMethod(str, "join", "(O)", tmp);
     if (joined == NULL) goto error;
 
+    /* PG doesn't like ARRAY[NULL..] */
+    if (!all_nulls) {
         res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined));
+    } else {
+        res = Bytes_FromFormat("'{%s}'", Bytes_AsString(joined));
+    }
 
  error:
     Py_XDECREF(tmp);

psycopg/config.h

@@ -129,28 +129,33 @@ static int pthread_mutex_init(pthread_mutex_t *mutex, void* fake)
 /* remove the inline keyword, since it doesn't work unless C++ file */
 #define inline
 
-/* Hmmm, MSVC doesn't have a isnan/isinf function, but has _isnan function */
+/* Hmmm, MSVC <2015 doesn't have a isnan/isinf function, but has _isnan function */
 #if defined (_MSC_VER)
+#if !defined(isnan)
 #define isnan(x) (_isnan(x))
 /* The following line was hacked together from simliar code by Bjorn Reese
  * in libxml2 code */
 #define isinf(x) ((_fpclass(x) == _FPCLASS_PINF) ? 1 \
 	: ((_fpclass(x) == _FPCLASS_NINF) ? -1 : 0))
-
+#endif
 #define strcasecmp(x, y) lstrcmpi(x, y)
 #endif
 #endif
 
+/* what's this, we have no round function either? */
 #if (defined(__FreeBSD__) && __FreeBSD_version < 503000) \
     || (defined(_WIN32) && !defined(__GNUC__)) \
     || (defined(sun) || defined(__sun__)) \
         && (defined(__SunOS_5_8) || defined(__SunOS_5_9))
-/* what's this, we have no round function either? */
+
+/* round has been added in the standard library with MSVC 2015 */
+#if _MSC_VER < 1900
 static double round(double num)
 {
   return (num >= 0) ? floor(num + 0.5) : ceil(num - 0.5);
 }
 #endif
+#endif
 
 /* resolve missing isinf() function for Solaris */
 #if defined (__SVR4) && defined (__sun)

psycopg/connection.h

@@ -71,7 +71,7 @@ extern HIDDEN PyTypeObject connectionType;
 
 struct connectionObject_notice {
     struct connectionObject_notice *next;
-    const char *message;
+    char *message;
 };
 
 /* the typedef is forward-declared in psycopg.h */
@@ -106,8 +106,8 @@ struct connectionObject {
 
     /* notice processing */
     PyObject *notice_list;
-    PyObject *notice_filter;
     struct connectionObject_notice *notice_pending;
+    struct connectionObject_notice *last_notice;
 
     /* notifies */
     PyObject *notifies;

psycopg/connection_int.c

@@ -87,13 +87,20 @@ conn_notice_callback(void *args, const char *message)
         /* Discard the notice in case of failed allocation. */
         return;
     }
+    notice->next = NULL;
     notice->message = strdup(message);
     if (NULL == notice->message) {
         free(notice);
         return;
     }
-    notice->next = self->notice_pending;
+
-    self->notice_pending = notice;
+    if (NULL == self->last_notice) {
+        self->notice_pending = self->last_notice = notice;
+    }
+    else {
+        self->last_notice->next = notice;
+        self->last_notice = notice;
+    }
 }
 
 /* Expose the notices received as Python objects.
@@ -104,44 +111,60 @@ void
 conn_notice_process(connectionObject *self)
 {
     struct connectionObject_notice *notice;
-    Py_ssize_t nnotices;
+    PyObject *msg = NULL;
+    PyObject *tmp = NULL;
+    static PyObject *append;
 
     if (NULL == self->notice_pending) {
         return;
     }
 
-    notice =  self->notice_pending;
+    if (!append) {
-    nnotices = PyList_GET_SIZE(self->notice_list);
+        if (!(append = Text_FromUTF8("append"))) {
+            goto error;
+        }
+    }
 
+    notice = self->notice_pending;
     while (notice != NULL) {
-        PyObject *msg;
-        msg = conn_text_from_chars(self, notice->message);
         Dprintf("conn_notice_process: %s", notice->message);
 
-        /* Respect the order in which notices were produced,
+        if (!(msg = conn_text_from_chars(self, notice->message))) { goto error; }
-           because in notice_list they are reversed (see ticket #9) */
+
-        if (msg) {
+        if (!(tmp = PyObject_CallMethodObjArgs(
-            PyList_Insert(self->notice_list, nnotices, msg);
+                self->notice_list, append, msg, NULL))) {
-            Py_DECREF(msg);
+
-        }
+            goto error;
-        else {
-            /* We don't really have a way to report errors, so gulp it.
-             * The function should only fail for out of memory, so we are
-             * likely going to die anyway. */
-            PyErr_Clear();
         }
 
+        Py_DECREF(tmp); tmp = NULL;
+        Py_DECREF(msg); msg = NULL;
+
         notice = notice->next;
     }
 
     /* Remove the oldest item if the queue is getting too long. */
+    if (PyList_Check(self->notice_list)) {
+        Py_ssize_t nnotices;
         nnotices = PyList_GET_SIZE(self->notice_list);
         if (nnotices > CONN_NOTICES_LIMIT) {
-        PySequence_DelSlice(self->notice_list,
+            if (-1 == PySequence_DelSlice(self->notice_list,
-            0, nnotices - CONN_NOTICES_LIMIT);
+                    0, nnotices - CONN_NOTICES_LIMIT)) {
+                PyErr_Clear();
+            }
+        }
     }
 
     conn_notice_clean(self);
+    return;
+
+error:
+    Py_XDECREF(tmp);
+    Py_XDECREF(msg);
+    conn_notice_clean(self);
+
+    /* TODO: the caller doesn't expects errors from us */
+    PyErr_Clear();
 }
 
 void
@@ -154,11 +177,11 @@ conn_notice_clean(connectionObject *self)
     while (notice != NULL) {
         tmp = notice;
         notice = notice->next;
-        free((void*)tmp->message);
+        free(tmp->message);
         free(tmp);
     }
 
-    self->notice_pending = NULL;
+    self->last_notice = self->notice_pending = NULL;
 }
 
 
@@ -173,6 +196,15 @@ conn_notifies_process(connectionObject *self)
     PGnotify *pgn = NULL;
     PyObject *notify = NULL;
     PyObject *pid = NULL, *channel = NULL, *payload = NULL;
+    PyObject *tmp = NULL;
+
+    static PyObject *append;
+
+    if (!append) {
+        if (!(append = Text_FromUTF8("append"))) {
+            goto error;
+        }
+    }
 
     while ((pgn = PQnotifies(self->pgconn)) != NULL) {
 
@@ -192,7 +224,11 @@ conn_notifies_process(connectionObject *self)
         Py_DECREF(channel); channel = NULL;
         Py_DECREF(payload); payload = NULL;
 
-        PyList_Append(self->notifies, (PyObject *)notify);
+        if (!(tmp = PyObject_CallMethodObjArgs(
+                self->notifies, append, notify, NULL))) {
+            goto error;
+        }
+        Py_DECREF(tmp); tmp = NULL;
 
         Py_DECREF(notify); notify = NULL;
         PQfreemem(pgn); pgn = NULL;
@@ -201,6 +237,7 @@ conn_notifies_process(connectionObject *self)
 
 error:
     if (pgn) { PQfreemem(pgn); }
+    Py_XDECREF(tmp);
     Py_XDECREF(notify);
     Py_XDECREF(pid);
     Py_XDECREF(channel);

psycopg/connection_type.c

@@ -1001,8 +1001,8 @@ static struct PyMemberDef connectionObject_members[] = {
         "True if the connection is closed."},
     {"encoding", T_STRING, offsetof(connectionObject, encoding), READONLY,
         "The current client encoding."},
-    {"notices", T_OBJECT, offsetof(connectionObject, notice_list), READONLY},
+    {"notices", T_OBJECT, offsetof(connectionObject, notice_list), 0},
-    {"notifies", T_OBJECT, offsetof(connectionObject, notifies), READONLY},
+    {"notifies", T_OBJECT, offsetof(connectionObject, notifies), 0},
     {"dsn", T_STRING, offsetof(connectionObject, dsn), READONLY,
         "The current connection string."},
     {"async", T_LONG, offsetof(connectionObject, async), READONLY,
@@ -1105,7 +1105,6 @@ connection_clear(connectionObject *self)
     Py_CLEAR(self->tpc_xid);
     Py_CLEAR(self->async_cursor);
     Py_CLEAR(self->notice_list);
-    Py_CLEAR(self->notice_filter);
     Py_CLEAR(self->notifies);
     Py_CLEAR(self->string_types);
     Py_CLEAR(self->binary_types);
@@ -1181,7 +1180,6 @@ connection_traverse(connectionObject *self, visitproc visit, void *arg)
     Py_VISIT((PyObject *)(self->tpc_xid));
     Py_VISIT(self->async_cursor);
     Py_VISIT(self->notice_list);
-    Py_VISIT(self->notice_filter);
     Py_VISIT(self->notifies);
     Py_VISIT(self->string_types);
     Py_VISIT(self->binary_types);

psycopg/pqpath.c

@@ -190,8 +190,10 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult **pgres)
        raise and a meaningful message is better than an empty one.
        Note: it can happen without it being our error: see ticket #82 */
     if (err == NULL || err[0] == '\0') {
-        PyErr_SetString(DatabaseError,
+        PyErr_Format(DatabaseError,
-            "error with no message from the libpq");
+            "error with status %s and no message from the libpq",
+            PQresStatus(pgres == NULL ?
+                PQstatus(conn->pgconn) : PQresultStatus(*pgres)));
         return;
     }
 
@@ -1595,11 +1597,26 @@ pq_fetch(cursorObject *curs, int no_result)
         ex = -1;
         break;
 
-    default:
+    case PGRES_BAD_RESPONSE:
-        Dprintf("pq_fetch: uh-oh, something FAILED: pgconn = %p", curs->conn);
+    case PGRES_NONFATAL_ERROR:
+    case PGRES_FATAL_ERROR:
+        Dprintf("pq_fetch: uh-oh, something FAILED: status = %d pgconn = %p",
+            status, curs->conn);
         pq_raise(curs->conn, curs, NULL);
         ex = -1;
         break;
+
+    default:
+        /* PGRES_COPY_BOTH, PGRES_SINGLE_TUPLE, future statuses */
+        Dprintf("pq_fetch: got unsupported result: status = %d pgconn = %p",
+            status, curs->conn);
+        PyErr_Format(NotSupportedError,
+            "got server response with unsupported status %s",
+            PQresStatus(curs->pgres == NULL ?
+                PQstatus(curs->conn->pgconn) : PQresultStatus(curs->pgres)));
+        CLEARPGRES(curs->pgres);
+        ex = -1;
+        break;
     }
 
     /* error checking, close the connection if necessary (some critical errors

tests/test_connection.py

@@ -26,6 +26,7 @@ import os
 import time
 import threading
 from operator import attrgetter
+from StringIO import StringIO
 
 import psycopg2
 import psycopg2.errorcodes
@@ -129,6 +130,42 @@ class ConnectionTests(ConnectingTestCase):
         self.assertEqual(50, len(conn.notices))
         self.assert_('table99' in conn.notices[-1], conn.notices[-1])
 
+    def test_notices_deque(self):
+        from collections import deque
+
+        conn = self.conn
+        self.conn.notices = deque()
+        cur = conn.cursor()
+        if self.conn.server_version >= 90300:
+            cur.execute("set client_min_messages=debug1")
+
+        cur.execute("create temp table table1 (id serial); create temp table table2 (id serial);")
+        cur.execute("create temp table table3 (id serial); create temp table table4 (id serial);")
+        self.assertEqual(len(conn.notices), 4)
+        self.assert_('table1' in conn.notices.popleft())
+        self.assert_('table2' in conn.notices.popleft())
+        self.assert_('table3' in conn.notices.popleft())
+        self.assert_('table4' in conn.notices.popleft())
+        self.assertEqual(len(conn.notices), 0)
+
+        # not limited, but no error
+        for i in range(0, 100, 10):
+            sql = " ".join(["create temp table table2_%d (id serial);" % j for j in range(i, i+10)])
+            cur.execute(sql)
+
+        self.assertEqual(100, len(conn.notices))
+
+    def test_notices_noappend(self):
+        conn = self.conn
+        self.conn.notices = None    # will make an error swallowes ok
+        cur = conn.cursor()
+        if self.conn.server_version >= 90300:
+            cur.execute("set client_min_messages=debug1")
+
+        cur.execute("create temp table table1 (id serial);")
+
+        self.assertEqual(self.conn.notices, None)
+
     def test_server_version(self):
         self.assert_(self.conn.server_version)
 
@@ -1118,6 +1155,17 @@ class AutocommitTests(ConnectingTestCase):
         self.assertEqual(cur.fetchone()[0], 'on')
 
 
+class ReplicationTest(ConnectingTestCase):
+    @skip_before_postgres(9, 0)
+    def test_replication_not_supported(self):
+        conn = self.repl_connect()
+        if conn is None: return
+        cur = conn.cursor()
+        f = StringIO()
+        self.assertRaises(psycopg2.NotSupportedError,
+            cur.copy_expert, "START_REPLICATION 0/0", f)
+
+
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)
 

tests/test_notify.py

@@ -155,6 +155,27 @@ conn.close()
         self.assertEqual('foo', notify.channel)
         self.assertEqual('Hello, world!', notify.payload)
 
+    def test_notify_deque(self):
+        from collections import deque
+        self.autocommit(self.conn)
+        self.conn.notifies = deque()
+        self.listen('foo')
+        self.notify('foo').communicate()
+        time.sleep(0.5)
+        self.conn.poll()
+        notify = self.conn.notifies.popleft()
+        self.assert_(isinstance(notify, psycopg2.extensions.Notify))
+        self.assertEqual(len(self.conn.notifies), 0)
+
+    def test_notify_noappend(self):
+        self.autocommit(self.conn)
+        self.conn.notifies = None
+        self.listen('foo')
+        self.notify('foo').communicate()
+        time.sleep(0.5)
+        self.conn.poll()
+        self.assertEqual(self.conn.notifies, None)
+
     def test_notify_init(self):
         n = psycopg2.extensions.Notify(10, 'foo')
         self.assertEqual(10, n.pid)
@@ -192,6 +213,7 @@ conn.close()
         self.assertNotEqual(hash(Notify(10, 'foo', 'bar')),
             hash(Notify(10, 'foo')))
 
+
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)
 

tests/test_types_basic.py

@@ -192,6 +192,40 @@ class TypesBasicTests(ConnectingTestCase):
             self.assertRaises(psycopg2.DataError,
                 psycopg2.extensions.STRINGARRAY, b(s), curs)
 
+    def testArrayOfNulls(self):
+        curs = self.conn.cursor()
+        curs.execute("""
+            create table na (
+              texta text[],
+              inta int[],
+              boola boolean[],
+
+              textaa text[][],
+              intaa int[][],
+              boolaa boolean[][]
+            )""")
+
+        curs.execute("insert into na (texta) values (%s)", ([None],))
+        curs.execute("insert into na (texta) values (%s)", (['a', None],))
+        curs.execute("insert into na (texta) values (%s)", ([None, None],))
+        curs.execute("insert into na (inta) values (%s)",  ([None],))
+        curs.execute("insert into na (inta) values (%s)",  ([42, None],))
+        curs.execute("insert into na (inta) values (%s)",  ([None, None],))
+        curs.execute("insert into na (boola) values (%s)", ([None],))
+        curs.execute("insert into na (boola) values (%s)", ([True, None],))
+        curs.execute("insert into na (boola) values (%s)", ([None, None],))
+
+        # TODO: array of array of nulls are not supported yet
+        # curs.execute("insert into na (textaa) values (%s)", ([[None]],))
+        curs.execute("insert into na (textaa) values (%s)", ([['a', None]],))
+        # curs.execute("insert into na (textaa) values (%s)", ([[None, None]],))
+        # curs.execute("insert into na (intaa) values (%s)",  ([[None]],))
+        curs.execute("insert into na (intaa) values (%s)",  ([[42, None]],))
+        # curs.execute("insert into na (intaa) values (%s)",  ([[None, None]],))
+        # curs.execute("insert into na (boolaa) values (%s)", ([[None]],))
+        curs.execute("insert into na (boolaa) values (%s)", ([[True, None]],))
+        # curs.execute("insert into na (boolaa) values (%s)", ([[None, None]],))
+
     @testutils.skip_from_python(3)
     def testTypeRoundtripBuffer(self):
         o1 = buffer("".join(map(chr, range(256))))

tests/testconfig.py

@@ -7,6 +7,8 @@ dbhost = os.environ.get('PSYCOPG2_TESTDB_HOST', None)
 dbport = os.environ.get('PSYCOPG2_TESTDB_PORT', None)
 dbuser = os.environ.get('PSYCOPG2_TESTDB_USER', None)
 dbpass = os.environ.get('PSYCOPG2_TESTDB_PASSWORD', None)
+repl_dsn = os.environ.get('PSYCOPG2_TEST_REPL_DSN',
+    "dbname=psycopg2_test replication=1")
 
 # Check if we want to test psycopg's green path.
 green = os.environ.get('PSYCOPG2_TEST_GREEN', None)

tests/testutils.py

@@ -28,7 +28,7 @@ import os
 import platform
 import sys
 from functools import wraps
-from testconfig import dsn
+from testconfig import dsn, repl_dsn
 
 try:
     import unittest2
@@ -103,11 +103,35 @@ class ConnectingTestCase(unittest.TestCase):
                 "%s (did you remember calling ConnectingTestCase.setUp()?)"
                 % e)
 
+        if 'dsn' in kwargs:
+            conninfo = kwargs.pop('dsn')
+        else:
+            conninfo = dsn
         import psycopg2
-        conn = psycopg2.connect(dsn, **kwargs)
+        conn = psycopg2.connect(conninfo, **kwargs)
         self._conns.append(conn)
         return conn
 
+    def repl_connect(self, **kwargs):
+        """Return a connection set up for replication
+
+        The connection is on "PSYCOPG2_TEST_REPL_DSN" unless overridden by
+        a *dsn* kwarg.
+
+        Should raise a skip test if not available, but guard for None on
+        old Python versions.
+        """
+        if 'dsn' not in kwargs:
+            kwargs['dsn'] = repl_dsn
+        import psycopg2
+        try:
+            conn = self.connect(**kwargs)
+        except psycopg2.OperationalError, e:
+            return self.skipTest("replication db not configured: %s" % e)
+
+        conn.autocommit = True
+        return conn
+
     def _get_conn(self):
         if not hasattr(self, '_the_conn'):
             self._the_conn = self.connect()
@@ -388,4 +412,3 @@ class py3_raises_typeerror(object):
         if sys.version_info[0] >= 3:
             assert type is TypeError
             return True
-