From 4a8b5f98f3be4108d1b0ac532d7a1aef10103ff1 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 4 Mar 2011 20:18:22 +0000
Subject: [PATCH 01/19] Bump to work on 2.4.1

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index c626d8f0..90244a9a 100644
--- a/setup.py
+++ b/setup.py
@@ -79,7 +79,7 @@ except ImportError:
 # Take a look at http://www.python.org/dev/peps/pep-0386/
 # for a consistent versioning pattern.
 
-PSYCOPG_VERSION = '2.4'
+PSYCOPG_VERSION = '2.4.1a0'
 
 version_flags   = ['dt', 'dec']
 

From f1d69f6dec10003577bcbca71bd1fd448d4ff272 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 4 Mar 2011 20:20:56 +0000
Subject: [PATCH 02/19] Fixed detection of empty error from pq_raise

Avoid a system error in case err is set to an empty string.
---
 psycopg/pqpath.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c
index 8136d0af..1a5278b5 100644
--- a/psycopg/pqpath.c
+++ b/psycopg/pqpath.c
@@ -172,16 +172,19 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult *pgres)
     if (pgres) {
         err = PQresultErrorMessage(pgres);
         if (err != NULL) {
+            Dprintf("pq_raise: PQresultErrorMessage: err=%s", err);
             code = PQresultErrorField(pgres, PG_DIAG_SQLSTATE);
         }
     }
-    if (err == NULL)
+    if (err == NULL) {
         err = PQerrorMessage(conn->pgconn);
+        Dprintf("pq_raise: PQerrorMessage: err=%s", err);
+    }
 
     /* if the is no error message we probably called pq_raise without reason:
        we need to set an exception anyway because the caller will probably
        raise and a meaningful message is better than an empty one */
-    if (err == NULL) {
+    if (err == NULL || err[0] == '\0') {
         PyErr_SetString(Error, "psycopg went psycotic without error set");
         return;
     }
@@ -194,6 +197,7 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult *pgres)
 
     /* try to remove the initial "ERROR: " part from the postgresql error */
     err2 = strip_severity(err);
+    Dprintf("pq_raise: err2=%s", err2);
 
     psyco_set_error(exc, curs, err2, err, code);
 }

From fcbe0466a6eee8d76aa4aec23e5cc6f5de1a59d8 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 4 Mar 2011 20:30:43 +0000
Subject: [PATCH 03/19] Correctly detect an empty query sent to the backend

Closes ticket #46.
---
 NEWS                 | 6 ++++++
 psycopg/pqpath.c     | 7 +++++++
 tests/test_cursor.py | 6 ++++++
 3 files changed, 19 insertions(+)

diff --git a/NEWS b/NEWS
index a7e634e3..1eea8af6 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+What's new in psycopg 2.4.1
+---------------------------
+
+  - Correctly detect an empty query sent to the backend (ticket #46).
+
+
 What's new in psycopg 2.4
 -------------------------
 
diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c
index 1a5278b5..8d144dce 100644
--- a/psycopg/pqpath.c
+++ b/psycopg/pqpath.c
@@ -1359,6 +1359,13 @@ pq_fetch(cursorObject *curs)
         /* don't clear curs->pgres, because it contains the results! */
         break;
 
+    case PGRES_EMPTY_QUERY:
+        PyErr_SetString(ProgrammingError,
+            "can't execute an empty query");
+        IFCLEARPGRES(curs->pgres);
+        ex = -1;
+        break;
+
     default:
         Dprintf("pq_fetch: uh-oh, something FAILED: pgconn = %p", curs->conn);
         pq_raise(curs->conn, curs, NULL);
diff --git a/tests/test_cursor.py b/tests/test_cursor.py
index 1860ddc6..836f710d 100755
--- a/tests/test_cursor.py
+++ b/tests/test_cursor.py
@@ -37,6 +37,12 @@ class CursorTests(unittest.TestCase):
     def tearDown(self):
         self.conn.close()
 
+    def test_empty_query(self):
+        cur = self.conn.cursor()
+        self.assertRaises(psycopg2.ProgrammingError, cur.execute, "")
+        self.assertRaises(psycopg2.ProgrammingError, cur.execute, " ")
+        self.assertRaises(psycopg2.ProgrammingError, cur.execute, ";")
+
     def test_executemany_propagate_exceptions(self):
         conn = self.conn
         cur = conn.cursor()

From 66c543b16c5d26f1a48fa71664b6fa32712b082d Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Sat, 26 Mar 2011 10:59:27 +0000
Subject: [PATCH 04/19] Parse bytea output format ourselves instead of using
 the libpq

PG 9.0 uses the hex format by default, and clients < 9.0 can't parse that
format, requiring client update and great care in what is linked at runtime,
and generally giving headache to users and transitively us.
---
 psycopg/typecast_binary.c | 229 ++++++++++++++++++++++++++------------
 tests/testutils.py        |  18 ---
 tests/types_basic.py      |  10 +-
 3 files changed, 160 insertions(+), 97 deletions(-)

diff --git a/psycopg/typecast_binary.c b/psycopg/typecast_binary.c
index fa371e2e..62b10829 100644
--- a/psycopg/typecast_binary.c
+++ b/psycopg/typecast_binary.c
@@ -40,7 +40,7 @@ chunk_dealloc(chunkObject *self)
         FORMAT_CODE_PY_SSIZE_T,
         self->base, self->len
       );
-    PQfreemem(self->base);
+    PyMem_Free(self->base);
     Py_TYPE(self)->tp_free((PyObject *)self);
 }
 
@@ -127,95 +127,184 @@ PyTypeObject chunkType = {
     chunk_doc                   /* tp_doc */
 };
 
+
+static char *psycopg_parse_hex(
+        const char *bufin, Py_ssize_t sizein, Py_ssize_t *sizeout);
+static char *psycopg_parse_escape(
+        const char *bufin, Py_ssize_t sizein, Py_ssize_t *sizeout);
+
 static PyObject *
 typecast_BINARY_cast(const char *s, Py_ssize_t l, PyObject *curs)
 {
     chunkObject *chunk = NULL;
     PyObject *res = NULL;
-    char *str = NULL, *buffer = NULL;
-    size_t len;
+    char *buffer = NULL;
+    Py_ssize_t len;
 
     if (s == NULL) {Py_INCREF(Py_None); return Py_None;}
 
-    /* PQunescapeBytea absolutely wants a 0-terminated string and we don't
-       want to copy the whole buffer, right? Wrong, but there isn't any other
-       way <g> */
-    if (s[l] != '\0') {
-        if ((buffer = PyMem_Malloc(l+1)) == NULL) {
-            PyErr_NoMemory();
-            goto fail;
+    if (s[0] == '\\' && s[1] == 'x') {
+        /* This is a buffer escaped in hex format: libpq before 9.0 can't
+         * parse it and we can't detect reliably the libpq version at runtime.
+         * So the only robust option is to parse it ourselves - luckily it's
+         * an easy format.
+         */
+        if (NULL == (buffer = psycopg_parse_hex(s, l, &len))) {
+            goto exit;
         }
-        /* Py_ssize_t->size_t cast is safe, as long as the Py_ssize_t is
-         * >= 0: */
-        assert (l >= 0);
-        strncpy(buffer, s, (size_t) l);
-
-        buffer[l] = '\0';
-        s = buffer;
     }
-    str = (char*)PQunescapeBytea((unsigned char*)s, &len);
-    Dprintf("typecast_BINARY_cast: unescaped " FORMAT_CODE_SIZE_T " bytes",
-      len);
-
-    /* The type of the second parameter to PQunescapeBytea is size_t *, so it's
-     * possible (especially with Python < 2.5) to get a return value too large
-     * to fit into a Python container. */
-    if (len > (size_t) PY_SSIZE_T_MAX) {
-      PyErr_SetString(PyExc_IndexError, "PG buffer too large to fit in Python"
-                                        " buffer.");
-      goto fail;
-    }
-
-    /* Check the escaping was successful */
-    if (s[0] == '\\' && s[1] == 'x'     /* input encoded in hex format */
-        && str[0] == 'x'                /* output resulted in an 'x' */
-        && s[2] != '7' && s[3] != '8')  /* input wasn't really an x (0x78) */
-    {
-        PyErr_SetString(InterfaceError,
-            "can't receive bytea data from server >= 9.0 with the current "
-            "libpq client library: please update the libpq to at least 9.0 "
-            "or set bytea_output to 'escape' in the server config "
-            "or with a query");
-        goto fail;
+    else {
+        /* This is a buffer in the classic bytea format. So we can handle it
+         * to the PQunescapeBytea to have it parsed, rignt? ...Wrong. We
+         * could, but then we'd have to record whether buffer was allocated by
+         * Python or by the libpq to dispose it properly. Furthermore the
+         * PQunescapeBytea interface is not the most brilliant as it wants a
+         * null-terminated string even if we have known its length thus
+         * requiring a useless memcpy and strlen.
+         * So we'll just have our better integrated parser, let's finish this
+         * story.
+         */
+        if (NULL == (buffer = psycopg_parse_escape(s, l, &len))) {
+            goto exit;
+        }
     }
 
     chunk = (chunkObject *) PyObject_New(chunkObject, &chunkType);
-    if (chunk == NULL) goto fail;
+    if (chunk == NULL) goto exit;
 
-    /* **Transfer** ownership of str's memory to the chunkObject: */
-    chunk->base = str;
-    str = NULL;
+    /* **Transfer** ownership of buffer's memory to the chunkObject: */
+    chunk->base = buffer;
+    buffer = NULL;
+    chunk->len = (Py_ssize_t)len;
 
-    /* size_t->Py_ssize_t cast was validated above: */
-    chunk->len = (Py_ssize_t) len;
 #if PY_MAJOR_VERSION < 3
     if ((res = PyBuffer_FromObject((PyObject *)chunk, 0, chunk->len)) == NULL)
-        goto fail;
+        goto exit;
 #else
     if ((res = PyMemoryView_FromObject((PyObject*)chunk)) == NULL)
-        goto fail;
+        goto exit;
 #endif
-    /* PyBuffer_FromObject() created a new reference.  We'll release our
-     * reference held in 'chunk' in the 'cleanup' clause. */
 
-    goto cleanup;
-    fail:
-      assert (PyErr_Occurred());
-      if (res != NULL) {
-          Py_DECREF(res);
-          res = NULL;
-      }
-      /* Fall through to cleanup: */
-    cleanup:
-      if (chunk != NULL) {
-          Py_DECREF((PyObject *) chunk);
-      }
-      if (str != NULL) {
-          /* str's mem was allocated by PQunescapeBytea; must use PQfreemem: */
-          PQfreemem(str);
-      }
-      /* We allocated buffer with PyMem_Malloc; must use PyMem_Free: */
-      PyMem_Free(buffer);
+exit:
+    Py_XDECREF((PyObject *)chunk);
+    PyMem_Free(buffer);
 
-      return res;
+    return res;
 }
+
+
+static const char hex_lut[128] = {
+    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+     0,  1,  2,  3,  4,  5,  6,  7,  8,  9, -1, -1, -1, -1, -1, -1,
+    -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+    -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+};
+
+/* Parse a bytea output buffer encoded in 'hex' format.
+ *
+ * the format is described in
+ * http://www.postgresql.org/docs/9.0/static/datatype-binary.html
+ *
+ * Parse the buffer in 'bufin', whose length is 'sizein'.
+ * Return a new buffer allocated by PyMem_Malloc and set 'sizeout' to its size.
+ * In case of error set an exception and return NULL.
+ */
+static char *
+psycopg_parse_hex(const char *bufin, Py_ssize_t sizein, Py_ssize_t *sizeout)
+{
+    char *ret = NULL;
+    const char *bufend = bufin + sizein;
+    const char *pi = bufin + 2;     /* past the \x */
+    char *bufout;
+    char *po;
+
+    po = bufout = PyMem_Malloc((sizein - 2) >> 1);   /* output size upper bound */
+    if (NULL == bufout) {
+        PyErr_NoMemory();
+        goto exit;
+    }
+
+    /* Implementation note: we call this function upon database response, not
+     * user input (because we are parsing the output format of a buffer) so we
+     * don't expect errors. On bad input we reserve the right to return a bad
+     * output, not an error.
+     */
+    while (pi < bufend) {
+        char c;
+        while (-1 == (c = hex_lut[*pi++ & '\x7f'])) {
+            if (pi >= bufend) { goto endloop; }
+        }
+        *po = c << 4;
+
+        while (-1 == (c = hex_lut[*pi++ & '\x7f'])) {
+            if (pi >= bufend) { goto endloop; }
+        }
+        *po++ |= c;
+    }
+endloop:
+
+    ret = bufout;
+    *sizeout = po - bufout;
+
+exit:
+    return ret;
+}
+
+/* Parse a bytea output buffer encoded in 'escape' format.
+ *
+ * the format is described in
+ * http://www.postgresql.org/docs/9.0/static/datatype-binary.html
+ *
+ * Parse the buffer in 'bufin', whose length is 'sizein'.
+ * Return a new buffer allocated by PyMem_Malloc and set 'sizeout' to its size.
+ * In case of error set an exception and return NULL.
+ */
+static char *
+psycopg_parse_escape(const char *bufin, Py_ssize_t sizein, Py_ssize_t *sizeout)
+{
+    char *ret = NULL;
+    const char *bufend = bufin + sizein;
+    const char *pi = bufin;
+    char *bufout;
+    char *po;
+
+    po = bufout = PyMem_Malloc(sizein);   /* output size upper bound */
+    if (NULL == bufout) {
+        PyErr_NoMemory();
+        goto exit;
+    }
+
+    while (pi < bufend) {
+        if (*pi != '\\') {
+            /* Unescaped char */
+            *po++ = *pi++;
+            continue;
+        }
+        if ((pi[1] >= '0' && pi[1] <= '3') &&
+            (pi[2] >= '0' && pi[2] <= '7') &&
+            (pi[3] >= '0' && pi[3] <= '7'))
+        {
+            /* Escaped octal value */
+            *po++ = ((pi[1] - '0') << 6) |
+                    ((pi[2] - '0') << 3) |
+                    ((pi[3] - '0'));
+            pi += 4;
+        }
+        else {
+            /* Escaped char */
+            *po++ = pi[1];
+            pi += 2;
+        }
+    }
+
+    ret = bufout;
+    *sizeout = po - bufout;
+
+exit:
+    return ret;
+}
+
diff --git a/tests/testutils.py b/tests/testutils.py
index 2459894f..26551d4e 100644
--- a/tests/testutils.py
+++ b/tests/testutils.py
@@ -140,24 +140,6 @@ def skip_if_no_namedtuple(f):
     return skip_if_no_namedtuple_
 
 
-def skip_if_broken_hex_binary(f):
-    """Decorator to detect libpq < 9.0 unable to parse bytea in hex format"""
-    def cope_with_hex_binary_(self):
-        from psycopg2 import InterfaceError
-        try:
-            return f(self)
-        except InterfaceError, e:
-            if '9.0' in str(e) and self.conn.server_version >= 90000:
-                return self.skipTest(
-                    # FIXME: we are only assuming the libpq is older here,
-                    # but we don't have a reliable way to detect the libpq
-                    # version, not pre-9 at least.
-                    "bytea broken with server >= 9.0, libpq < 9")
-            else:
-                raise
-
-    return cope_with_hex_binary_
-
 def skip_if_no_iobase(f):
     """Skip a test if io.TextIOBase is not available."""
     def skip_if_no_iobase_(self):
diff --git a/tests/types_basic.py b/tests/types_basic.py
index 40106310..83d526ff 100755
--- a/tests/types_basic.py
+++ b/tests/types_basic.py
@@ -28,7 +28,7 @@ except:
     pass
 import sys
 import testutils
-from testutils import unittest, skip_if_broken_hex_binary
+from testutils import unittest
 from testconfig import dsn
 
 import psycopg2
@@ -116,7 +116,6 @@ class TypesBasicTests(unittest.TestCase):
         s = self.execute("SELECT %s AS foo", (float("-inf"),))
         self.failUnless(str(s) == "-inf", "wrong float quoting: " + str(s))      
 
-    @skip_if_broken_hex_binary
     def testBinary(self):
         if sys.version_info[0] < 3:
             s = ''.join([chr(x) for x in range(256)])
@@ -143,7 +142,6 @@ class TypesBasicTests(unittest.TestCase):
             b = psycopg2.Binary(bytes([]))
             self.assertEqual(str(b), "''::bytea")
 
-    @skip_if_broken_hex_binary
     def testBinaryRoundTrip(self):
         # test to make sure buffers returned by psycopg2 are
         # understood by execute:
@@ -191,7 +189,6 @@ class TypesBasicTests(unittest.TestCase):
         s = self.execute("SELECT '{}'::text AS foo")
         self.failUnlessEqual(s, "{}")
 
-    @skip_if_broken_hex_binary
     @testutils.skip_from_python(3)
     def testTypeRoundtripBuffer(self):
         o1 = buffer("".join(map(chr, range(256))))
@@ -204,7 +201,6 @@ class TypesBasicTests(unittest.TestCase):
         self.assertEqual(type(o1), type(o2))
         self.assertEqual(str(o1), str(o2))
 
-    @skip_if_broken_hex_binary
     @testutils.skip_from_python(3)
     def testTypeRoundtripBufferArray(self):
         o1 = buffer("".join(map(chr, range(256))))
@@ -213,7 +209,6 @@ class TypesBasicTests(unittest.TestCase):
         self.assertEqual(type(o1[0]), type(o2[0]))
         self.assertEqual(str(o1[0]), str(o2[0]))
 
-    @skip_if_broken_hex_binary
     @testutils.skip_before_python(3)
     def testTypeRoundtripBytes(self):
         o1 = bytes(range(256))
@@ -225,7 +220,6 @@ class TypesBasicTests(unittest.TestCase):
         o2 = self.execute("select %s;", (o1,))
         self.assertEqual(memoryview, type(o2))
 
-    @skip_if_broken_hex_binary
     @testutils.skip_before_python(3)
     def testTypeRoundtripBytesArray(self):
         o1 = bytes(range(256))
@@ -233,7 +227,6 @@ class TypesBasicTests(unittest.TestCase):
         o2 = self.execute("select %s;", (o1,))
         self.assertEqual(memoryview, type(o2[0]))
 
-    @skip_if_broken_hex_binary
     @testutils.skip_before_python(2, 6)
     def testAdaptBytearray(self):
         o1 = bytearray(range(256))
@@ -258,7 +251,6 @@ class TypesBasicTests(unittest.TestCase):
         else:
             self.assertEqual(memoryview, type(o2))
 
-    @skip_if_broken_hex_binary
     @testutils.skip_before_python(2, 7)
     def testAdaptMemoryview(self):
         o1 = memoryview(bytearray(range(256)))

From e0cd6f0f00562e065f19da5c977843008dcddd24 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Sat, 26 Mar 2011 12:57:14 +0000
Subject: [PATCH 05/19] Added tests for our own bytea parser

Because the parse function is not supposed to be exposed in Python,
use ctypes to directly inspect the C function.
---
 psycopg/typecast_binary.c |  3 +-
 tests/types_basic.py      | 86 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/psycopg/typecast_binary.c b/psycopg/typecast_binary.c
index 62b10829..b145b1b7 100644
--- a/psycopg/typecast_binary.c
+++ b/psycopg/typecast_binary.c
@@ -133,7 +133,8 @@ static char *psycopg_parse_hex(
 static char *psycopg_parse_escape(
         const char *bufin, Py_ssize_t sizein, Py_ssize_t *sizeout);
 
-static PyObject *
+/* The function is not static and not hidden as we use ctypes to test it. */
+PyObject *
 typecast_BINARY_cast(const char *s, Py_ssize_t l, PyObject *curs)
 {
     chunkObject *chunk = NULL;
diff --git a/tests/types_basic.py b/tests/types_basic.py
index 83d526ff..0eb6ac48 100755
--- a/tests/types_basic.py
+++ b/tests/types_basic.py
@@ -327,6 +327,92 @@ class AdaptSubclassTest(unittest.TestCase):
            del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote]
 
 
+class ByteaParserTest(unittest.TestCase):
+    """Unit test for our bytea format parser."""
+    def setUp(self):
+        try:
+            self._cast = self._import_cast()
+        except Exception, e:
+            return self.skipTest("can't test bytea parser: %s - %s"
+                % (e.__class__.__name__, e))
+
+    def _import_cast(self):
+        """Use ctypes to access the C function.
+
+        Raise any sort of error: we just support this where ctypes works as
+        expected.
+        """
+        import ctypes
+        lib = ctypes.cdll.LoadLibrary(psycopg2._psycopg.__file__)
+        cast = lib.typecast_BINARY_cast
+        cast.argtypes = [ctypes.c_char_p, ctypes.c_size_t, ctypes.py_object]
+        cast.restype = ctypes.py_object
+        return cast
+
+    def cast(self, buffer):
+        """Cast a buffer from the output format"""
+        l = buffer and len(buffer) or 0
+        rv = self._cast(buffer, l, None)
+
+        if rv is None:
+            return None
+
+        if sys.version_info[0] < 3:
+            return str(rv)
+        else:
+            return rv.tobytes()
+
+    def test_null(self):
+        rv = self.cast(None)
+        self.assertEqual(rv, None)
+
+    def test_blank(self):
+        rv = self.cast(b(''))
+        self.assertEqual(rv, b(''))
+
+    def test_blank_hex(self):
+        # Reported as problematic in ticket #48
+        rv = self.cast(b('\\x'))
+        self.assertEqual(rv, b(''))
+
+    def test_full_hex(self, upper=False):
+        buf = ''.join(("%02x" % i) for i in range(256))
+        if upper: buf = buf.upper()
+        buf = '\\x' + buf
+        rv = self.cast(b(buf))
+        if sys.version_info[0] < 3:
+            self.assertEqual(rv, ''.join(map(chr, range(256))))
+        else:
+            self.assertEqual(rv, bytes(range(256)))
+
+    def test_full_hex_upper(self):
+        return self.test_full_hex(upper=True)
+
+    def test_full_escaped_octal(self):
+        buf = ''.join(("\\%03o" % i) for i in range(256))
+        rv = self.cast(b(buf))
+        if sys.version_info[0] < 3:
+            self.assertEqual(rv, ''.join(map(chr, range(256))))
+        else:
+            self.assertEqual(rv, bytes(range(256)))
+
+    def test_escaped_mixed(self):
+        import string
+        buf = ''.join(("\\%03o" % i) for i in range(32))
+        buf += string.ascii_letters
+        buf += ''.join('\\' + c for c in string.ascii_letters)
+        buf += '\\\\'
+        rv = self.cast(b(buf))
+        if sys.version_info[0] < 3:
+            tgt = ''.join(map(chr, range(32))) \
+                + string.ascii_letters * 2 + '\\'
+        else:
+            tgt = bytes(range(32)) + \
+                (string.ascii_letters * 2 + '\\').encode('ascii')
+
+        self.assertEqual(rv, tgt)
+
+
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)
 

From da58bee70af5ee84362a000ab2ab726e20f12df8 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Sat, 26 Mar 2011 11:19:01 +0000
Subject: [PATCH 06/19] Added documentation for the bytea parser

---
 NEWS              |  2 ++
 doc/src/faq.rst   |  4 +++-
 doc/src/usage.rst | 23 ++++++++++++-----------
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/NEWS b/NEWS
index 1eea8af6..8aef82e2 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,8 @@
 What's new in psycopg 2.4.1
 ---------------------------
 
+  - Use own parser for bytea output, not requiring anymore the libpq 9.0
+    to parse the hex format.
   - Correctly detect an empty query sent to the backend (ticket #46).
 
 
diff --git a/doc/src/faq.rst b/doc/src/faq.rst
index 642c3e78..4ebf15a5 100644
--- a/doc/src/faq.rst
+++ b/doc/src/faq.rst
@@ -97,7 +97,9 @@ Psycopg converts :sql:`decimal`\/\ :sql:`numeric` database types into Python `!D
 Transferring binary data from PostgreSQL 9.0 doesn't work.
     PostgreSQL 9.0 uses by default `the "hex" format`__ to transfer
     :sql:`bytea` data: the format can't be parsed by the libpq 8.4 and
-    earlier. Three options to solve the problem are:
+    earlier. The problem is solved in Psycopg 2.4.1, that uses its own parser
+    for the :sql:`bytea` format. For previous Psycopg releases, three options
+    to solve the problem are:
 
     - set the bytea_output__ parameter to ``escape`` in the server;
     - execute the database command ``SET bytea_output TO escape;`` in the
diff --git a/doc/src/usage.rst b/doc/src/usage.rst
index 47b78bec..4d039dee 100644
--- a/doc/src/usage.rst
+++ b/doc/src/usage.rst
@@ -271,6 +271,10 @@ the SQL string that would be sent to the database.
   .. versionchanged:: 2.4
      only strings were supported before.
 
+  .. versionchanged:: 2.4.1
+     can parse the 'hex' format from 9.0 servers without relying on the
+     version of the client library.
+
   .. note::
 
     In Python 2, if you have binary data in a `!str` object, you can pass them
@@ -282,18 +286,15 @@ the SQL string that would be sent to the database.
 
   .. warning::
 
-     PostgreSQL 9 uses by default `a new "hex" format`__ to emit :sql:`bytea`
-     fields. Unfortunately this format can't be parsed by libpq versions
-     before 9.0. This means that using a library client with version lesser
-     than 9.0 to talk with a server 9.0 or later you may have problems
-     receiving :sql:`bytea` data. To work around this problem you can set the
-     `bytea_output`__ parameter to ``escape``, either in the server
-     configuration or in the client session using a query such as ``SET
-     bytea_output TO escape;`` before trying to receive binary data.
+     Since version 9.0 PostgreSQL uses by default `a new "hex" format`__ to
+     emit :sql:`bytea` fields. Starting from Psycopg 2.4.1 the format is
+     correctly supported.  If you use a previous version you will need some
+     extra care when receiving bytea from PostgreSQL: you must have at least
+     the libpq 9.0 installed on the client or alternatively you can set the
+     `bytea_output`__ configutation parameter to ``escape``, either in the
+     server configuration file or in the client session (using a query such as
+     ``SET bytea_output TO escape;``) before receiving binary data.
      
-     Starting from Psycopg 2.4 this condition is detected and signaled with a
-     `~psycopg2.InterfaceError`.
-
      .. __: http://www.postgresql.org/docs/9.0/static/datatype-binary.html
      .. __: http://www.postgresql.org/docs/9.0/static/runtime-config-client.html#GUC-BYTEA-OUTPUT
 

From 7716cc6a0c311fd7032f3345fd86dac6f474195f Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Sat, 26 Mar 2011 13:48:37 +0000
Subject: [PATCH 07/19] Allow to specify --static-libpq on setup.py command
 line

Patch provided by Matthew Ryan (ticket #48).
---
 NEWS     | 2 ++
 setup.py | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/NEWS b/NEWS
index 8aef82e2..b952c274 100644
--- a/NEWS
+++ b/NEWS
@@ -4,6 +4,8 @@ What's new in psycopg 2.4.1
   - Use own parser for bytea output, not requiring anymore the libpq 9.0
     to parse the hex format.
   - Correctly detect an empty query sent to the backend (ticket #46).
+  - Allow to specify --static-libpq on setup.py command line instead of
+    just in 'setup.cfg'. Patch provided by Matthew Ryan (ticket #48).
 
 
 What's new in psycopg 2.4
diff --git a/setup.py b/setup.py
index 90244a9a..17eada11 100644
--- a/setup.py
+++ b/setup.py
@@ -133,6 +133,7 @@ class psycopg_build_ext(build_ext):
         self.mx_include_dir = None
         self.use_pydatetime = 1
         self.have_ssl = have_ssl
+        self.static_libpq = static_libpq
         self.pg_config = None
 
     def get_compiler(self):
@@ -263,7 +264,7 @@ or with the pg_config option in 'setup.cfg'.
             sys.exit(1)
 
         self.include_dirs.append(".")
-        if static_libpq:
+        if self.static_libpq:
             if not self.link_objects: self.link_objects = []
             self.link_objects.append(
                     os.path.join(self.get_pg_config("libdir"), "libpq.a"))

From 2dab7d52f203ce76639bf2fb19e041d5f63f8008 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Sat, 26 Mar 2011 14:27:58 +0000
Subject: [PATCH 08/19] Fixed bytea encoding tests skipping when ctypes is not
 available

---
 tests/types_basic.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/tests/types_basic.py b/tests/types_basic.py
index 0eb6ac48..1ca668de 100755
--- a/tests/types_basic.py
+++ b/tests/types_basic.py
@@ -28,7 +28,7 @@ except:
     pass
 import sys
 import testutils
-from testutils import unittest
+from testutils import unittest, decorate_all_tests
 from testconfig import dsn
 
 import psycopg2
@@ -333,8 +333,8 @@ class ByteaParserTest(unittest.TestCase):
         try:
             self._cast = self._import_cast()
         except Exception, e:
-            return self.skipTest("can't test bytea parser: %s - %s"
-                % (e.__class__.__name__, e))
+            self._cast = None
+            self._exc = e
 
     def _import_cast(self):
         """Use ctypes to access the C function.
@@ -412,6 +412,18 @@ class ByteaParserTest(unittest.TestCase):
 
         self.assertEqual(rv, tgt)
 
+def skip_if_cant_cast(f):
+    def skip_if_cant_cast_(self, *args, **kwargs):
+        if self._cast is None:
+            return self.skipTest("can't test bytea parser: %s - %s"
+                % (self._exc.__class__.__name__, self._exc))
+
+        return f(self, *args, **kwargs)
+
+    return skip_if_cant_cast_
+
+decorate_all_tests(ByteaParserTest, skip_if_cant_cast)
+
 
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)

From bf4870686888288ca3d259c2098a227a85f31fce Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Wed, 30 Mar 2011 15:52:49 +0100
Subject: [PATCH 09/19] Don't check the test db exists at psycopg2.tests import
 time

---
 tests/__init__.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/__init__.py b/tests/__init__.py
index 2eaf6ce8..a0575270 100755
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -27,17 +27,6 @@ import sys
 from testconfig import dsn
 from testutils import unittest
 
-# If connection to test db fails, bail out early.
-import psycopg2
-try:
-    cnn = psycopg2.connect(dsn)
-except Exception, e:
-    print "Failed connection to test db:", e.__class__.__name__, e
-    print "Please set env vars 'PSYCOPG2_TESTDB*' to valid values."
-    sys.exit(1)
-else:
-    cnn.close()
-
 import bug_gc
 import bugX000
 import extras_dictcursor
@@ -57,6 +46,17 @@ import test_green
 import test_cancel
 
 def test_suite():
+    # If connection to test db fails, bail out early.
+    import psycopg2
+    try:
+        cnn = psycopg2.connect(dsn)
+    except Exception, e:
+        print "Failed connection to test db:", e.__class__.__name__, e
+        print "Please set env vars 'PSYCOPG2_TESTDB*' to valid values."
+        sys.exit(1)
+    else:
+        cnn.close()
+
     suite = unittest.TestSuite()
     suite.addTest(bug_gc.test_suite())
     suite.addTest(bugX000.test_suite())

From 88803695ac3158f76d29ff1cdd3621740e6af53f Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 8 Apr 2011 11:27:45 +0100
Subject: [PATCH 10/19] Normalize the encoding name at connection

The encoding can be set by PGCLIENTENCODING, which may be an alternative
spelling. Bug reported by Peter Eisentraut.

At this point the idea of considering one of the random spellings such as
EUC_CN as somewhat "blessed" is debunked. So just store the cleaned-up
version of the encoding in the mapping table. Note that the cleaned-up
version was needed by the unicode adapter: this requirement has been
surpassed as the connection now contains a copy of the Python codec name
set whenever the client encoding is set.
---
 psycopg/connection_int.c  | 51 ++++++++++++++++++++++++++++++++-------
 psycopg/connection_type.c | 22 ++---------------
 tests/test_connection.py  | 14 +++++++++++
 3 files changed, 58 insertions(+), 29 deletions(-)

diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c
index fa714f66..6006b15c 100644
--- a/psycopg/connection_int.c
+++ b/psycopg/connection_int.c
@@ -236,6 +236,39 @@ conn_get_standard_conforming_strings(PGconn *pgconn)
     return equote;
 }
 
+
+/* Remove irrelevant chars from encoding name and turn it uppercase.
+ *
+ * Return a buffer allocated on Python heap,
+ * NULL and set an exception on error.
+ */
+static char *
+clean_encoding_name(const char *enc)
+{
+    const char *i = enc;
+    char *rv, *j;
+
+    /* convert to upper case and remove '-' and '_' from string */
+    if (!(j = rv = PyMem_Malloc(strlen(enc) + 1))) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+
+    while (*i) {
+        if (!isalnum(*i)) {
+            ++i;
+        }
+        else {
+            *j++ = toupper(*i++);
+        }
+    }
+    *j = '\0';
+
+    Dprintf("clean_encoding_name: %s -> %s", enc, rv);
+
+    return rv;
+}
+
 /* Convert a PostgreSQL encoding to a Python codec.
  *
  * Return a new copy of the codec name allocated on the Python heap,
@@ -246,11 +279,16 @@ conn_encoding_to_codec(const char *enc)
 {
     char *tmp;
     Py_ssize_t size;
+    char *norm_enc = NULL;
     PyObject *pyenc = NULL;
     char *rv = NULL;
 
+    if (!(norm_enc = clean_encoding_name(enc))) {
+        goto exit;
+    }
+
     /* Find the Py codec name from the PG encoding */
-    if (!(pyenc = PyDict_GetItemString(psycoEncodings, enc))) {
+    if (!(pyenc = PyDict_GetItemString(psycoEncodings, norm_enc))) {
         PyErr_Format(OperationalError,
             "no Python codec for client encoding '%s'", enc);
         goto exit;
@@ -270,6 +308,7 @@ conn_encoding_to_codec(const char *enc)
     rv = psycopg_strdup(tmp, size);
 
 exit:
+    PyMem_Free(norm_enc);
     Py_XDECREF(pyenc);
     return rv;
 }
@@ -285,7 +324,7 @@ exit:
 static int
 conn_read_encoding(connectionObject *self, PGconn *pgconn)
 {
-    char *enc = NULL, *codec = NULL, *j;
+    char *enc = NULL, *codec = NULL;
     const char *tmp;
     int rv = -1;
 
@@ -297,16 +336,10 @@ conn_read_encoding(connectionObject *self, PGconn *pgconn)
         goto exit;
     }
 
-    if (!(enc = PyMem_Malloc(strlen(tmp)+1))) {
-        PyErr_NoMemory();
+    if (!(enc = psycopg_strdup(tmp, 0))) {
         goto exit;
     }
 
-    /* turn encoding in uppercase */
-    j = enc;
-    while (*tmp) { *j++ = toupper(*tmp++); }
-    *j = '\0';
-
     /* Look for this encoding in Python codecs. */
     if (!(codec = conn_encoding_to_codec(enc))) {
         goto exit;
diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c
index b0c9ddcc..7ca395dc 100644
--- a/psycopg/connection_type.c
+++ b/psycopg/connection_type.c
@@ -423,36 +423,18 @@ static PyObject *
 psyco_conn_set_client_encoding(connectionObject *self, PyObject *args)
 {
     const char *enc;
-    char *buffer, *dest;
     PyObject *rv = NULL;
-    Py_ssize_t len;
 
     EXC_IF_CONN_CLOSED(self);
     EXC_IF_CONN_ASYNC(self, set_client_encoding);
     EXC_IF_TPC_PREPARED(self, set_client_encoding);
 
-    if (!PyArg_ParseTuple(args, "s#", &enc, &len)) return NULL;
+    if (!PyArg_ParseTuple(args, "s", &enc)) return NULL;
 
-    /* convert to upper case and remove '-' and '_' from string */
-    if (!(dest = buffer = PyMem_Malloc(len+1))) {
-        return PyErr_NoMemory();
-    }
-
-    while (*enc) {
-        if (*enc == '_' || *enc == '-') {
-            ++enc;
-        }
-        else {
-            *dest++ = toupper(*enc++);
-        }
-    }
-    *dest = '\0';
-
-    if (conn_set_client_encoding(self, buffer) == 0) {
+    if (conn_set_client_encoding(self, enc) == 0) {
         Py_INCREF(Py_None);
         rv = Py_None;
     }
-    PyMem_Free(buffer);
     return rv;
 }
 
diff --git a/tests/test_connection.py b/tests/test_connection.py
index e237524b..d9da471f 100755
--- a/tests/test_connection.py
+++ b/tests/test_connection.py
@@ -22,6 +22,7 @@
 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 # License for more details.
 
+import os
 import time
 import threading
 from testutils import unittest, decorate_all_tests, skip_before_postgres
@@ -141,6 +142,19 @@ class ConnectionTests(unittest.TestCase):
         cur.execute("select 'foo'::text;")
         self.assertEqual(cur.fetchone()[0], u'foo')
 
+    def test_connect_nonnormal_envvar(self):
+        # We must perform encoding normalization at connection time
+        self.conn.close()
+        oldenc = os.environ.get('PGCLIENTENCODING')
+        os.environ['PGCLIENTENCODING'] = 'utf-8'    # malformed spelling
+        try:
+            self.conn = psycopg2.connect(dsn)
+        finally:
+            if oldenc is not None:
+                os.environ['PGCLIENTENCODING'] = oldenc
+            else:
+                del os.environ['PGCLIENTENCODING']
+
     def test_weakref(self):
         from weakref import ref
         conn = psycopg2.connect(dsn)

From 19653a88ec4f85b040ca3054b14887c8105f846d Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 8 Apr 2011 13:27:11 +0100
Subject: [PATCH 11/19] Store a normalized version of the PG encoding in the
 connection

This way looking up into extensions.encodings will not break.
---
 psycopg/connection_int.c | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c
index 6006b15c..22c5bc59 100644
--- a/psycopg/connection_int.c
+++ b/psycopg/connection_int.c
@@ -273,22 +273,19 @@ clean_encoding_name(const char *enc)
  *
  * Return a new copy of the codec name allocated on the Python heap,
  * NULL with exception in case of error.
+ *
+ * 'enc' should be already normalized (uppercase, no - or _).
  */
 static char *
 conn_encoding_to_codec(const char *enc)
 {
     char *tmp;
     Py_ssize_t size;
-    char *norm_enc = NULL;
     PyObject *pyenc = NULL;
     char *rv = NULL;
 
-    if (!(norm_enc = clean_encoding_name(enc))) {
-        goto exit;
-    }
-
     /* Find the Py codec name from the PG encoding */
-    if (!(pyenc = PyDict_GetItemString(psycoEncodings, norm_enc))) {
+    if (!(pyenc = PyDict_GetItemString(psycoEncodings, enc))) {
         PyErr_Format(OperationalError,
             "no Python codec for client encoding '%s'", enc);
         goto exit;
@@ -308,7 +305,6 @@ conn_encoding_to_codec(const char *enc)
     rv = psycopg_strdup(tmp, size);
 
 exit:
-    PyMem_Free(norm_enc);
     Py_XDECREF(pyenc);
     return rv;
 }
@@ -336,7 +332,7 @@ conn_read_encoding(connectionObject *self, PGconn *pgconn)
         goto exit;
     }
 
-    if (!(enc = psycopg_strdup(tmp, 0))) {
+    if (!(enc = clean_encoding_name(tmp))) {
         goto exit;
     }
 
@@ -998,21 +994,23 @@ conn_set_client_encoding(connectionObject *self, const char *enc)
     PGresult *pgres = NULL;
     char *error = NULL;
     char query[48];
-    int res = 0;
-    char *codec;
+    int res = 1;
+    char *codec = NULL;
+    char *clean_enc = NULL;
 
     /* If the current encoding is equal to the requested one we don't
        issue any query to the backend */
     if (strcmp(self->encoding, enc) == 0) return 0;
 
     /* We must know what python codec this encoding is. */
-    if (!(codec = conn_encoding_to_codec(enc))) { return -1; }
+    if (!(clean_enc = clean_encoding_name(enc))) { goto exit; }
+    if (!(codec = conn_encoding_to_codec(clean_enc))) { goto exit; }
 
     Py_BEGIN_ALLOW_THREADS;
     pthread_mutex_lock(&self->lock);
 
     /* set encoding, no encoding string is longer than 24 bytes */
-    PyOS_snprintf(query, 47, "SET client_encoding = '%s'", enc);
+    PyOS_snprintf(query, 47, "SET client_encoding = '%s'", clean_enc);
 
     /* abort the current transaction, to set the encoding ouside of
        transactions */
@@ -1027,21 +1025,18 @@ conn_set_client_encoding(connectionObject *self, const char *enc)
     /* no error, we can proceeed and store the new encoding */
     {
         char *tmp = self->encoding;
-        self->encoding = NULL;
+        self->encoding = clean_enc;
         PyMem_Free(tmp);
-    }
-    if (!(self->encoding = psycopg_strdup(enc, 0))) {
-        res = 1;  /* don't call pq_complete_error below */
-        goto endlock;
+        clean_enc = NULL;
     }
 
     /* Store the python codec too. */
     {
         char *tmp = self->codec;
-        self->codec = NULL;
+        self->codec = codec;
         PyMem_Free(tmp);
+        codec = NULL;
     }
-    self->codec = codec;
 
     Dprintf("conn_set_client_encoding: set encoding to %s (codec: %s)",
             self->encoding, self->codec);
@@ -1054,6 +1049,10 @@ endlock:
     if (res < 0)
         pq_complete_error(self, &pgres, &error);
 
+exit:
+    PyMem_Free(clean_enc);
+    PyMem_Free(codec);
+
     return res;
 }
 

From e3605b33c13ddc4f35d30f8ea9b536c2a9b494c0 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 8 Apr 2011 14:36:49 +0100
Subject: [PATCH 12/19] Updated NEWS with the connection encoding fix

---
 NEWS | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/NEWS b/NEWS
index b952c274..7e2de335 100644
--- a/NEWS
+++ b/NEWS
@@ -3,6 +3,8 @@ What's new in psycopg 2.4.1
 
   - Use own parser for bytea output, not requiring anymore the libpq 9.0
     to parse the hex format.
+  - Don't fail connection if the client encoding is a non-normalized
+    variant. Issue reported by Peter Eisentraut.
   - Correctly detect an empty query sent to the backend (ticket #46).
   - Allow to specify --static-libpq on setup.py command line instead of
     just in 'setup.cfg'. Patch provided by Matthew Ryan (ticket #48).

From 746afdf69fdb0da76bc1bc3b5dcfd674576f19d0 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 15 Apr 2011 01:11:22 +0100
Subject: [PATCH 13/19] Added missing vertical spaces in NEWS

---
 NEWS | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/NEWS b/NEWS
index 7e2de335..6f8e48a4 100644
--- a/NEWS
+++ b/NEWS
@@ -175,6 +175,8 @@ doc/html/advanced.html.
   - Fixed TimestampFromTicks() and TimeFromTicks() for seconds >= 59.5.
   - Fixed spurious exception raised when calling C typecasters from Python
     ones.
+
+
 What's new in psycopg 2.0.14
 ----------------------------
 

From c08799b0b03b65d3a03f1bee86fc51bc682a4549 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Sun, 24 Apr 2011 02:57:04 +0100
Subject: [PATCH 14/19] Fixed SystemError clobbering libpq errors raised
 without SQLSTATE

Bug vivisectioned by Eric Snow
<http://archives.postgresql.org/psycopg/2011-04/msg00019.php>.
---
 NEWS             | 2 ++
 psycopg/pqpath.c | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/NEWS b/NEWS
index 6f8e48a4..60c7d4b3 100644
--- a/NEWS
+++ b/NEWS
@@ -6,6 +6,8 @@ What's new in psycopg 2.4.1
   - Don't fail connection if the client encoding is a non-normalized
     variant. Issue reported by Peter Eisentraut.
   - Correctly detect an empty query sent to the backend (ticket #46).
+  - Fixed a SystemError clobbering libpq errors raised without SQLSTATE.
+    Bug vivisectioned by Eric Snow.
   - Allow to specify --static-libpq on setup.py command line instead of
     just in 'setup.cfg'. Patch provided by Matthew Ryan (ticket #48).
 
diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c
index 8d144dce..6a6d05a3 100644
--- a/psycopg/pqpath.c
+++ b/psycopg/pqpath.c
@@ -194,6 +194,11 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult *pgres)
     if (code != NULL) {
         exc = exception_from_sqlstate(code);
     }
+    else {
+        /* Fallback if there is no exception code (reported happening e.g.
+         * when the connection is closed). */
+        exc = DatabaseError;
+    }
 
     /* try to remove the initial "ERROR: " part from the postgresql error */
     err2 = strip_severity(err);

From 80891e64b3f0280cfa28eba18b40d8b25bb0d284 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Tue, 26 Apr 2011 19:16:10 +0100
Subject: [PATCH 15/19] Dropped unused import

---
 lib/extras.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/extras.py b/lib/extras.py
index 21c5849c..06d36572 100644
--- a/lib/extras.py
+++ b/lib/extras.py
@@ -28,7 +28,6 @@ and classes untill a better place in the distribution is found.
 import os
 import sys
 import time
-import codecs
 import warnings
 import re as regex
 

From ffa7a62b9313b52af0c70f61b888703de1491777 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Tue, 26 Apr 2011 19:18:39 +0100
Subject: [PATCH 16/19] Fixed interaction between NamedTuple and named cursor

Build the nametuple after fetching the first resutl, or else
cursor.description will be empty.
---
 lib/extras.py              |  4 ++--
 tests/extras_dictcursor.py | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/lib/extras.py b/lib/extras.py
index 06d36572..dcbd65ed 100644
--- a/lib/extras.py
+++ b/lib/extras.py
@@ -290,17 +290,17 @@ class NamedTupleCursor(_cursor):
             return nt(*t)
 
     def fetchmany(self, size=None):
+        ts = _cursor.fetchmany(self, size)
         nt = self.Record
         if nt is None:
             nt = self.Record = self._make_nt()
-        ts = _cursor.fetchmany(self, size)
         return [nt(*t) for t in ts]
 
     def fetchall(self):
+        ts = _cursor.fetchall(self)
         nt = self.Record
         if nt is None:
             nt = self.Record = self._make_nt()
-        ts = _cursor.fetchall(self)
         return [nt(*t) for t in ts]
 
     def __iter__(self):
diff --git a/tests/extras_dictcursor.py b/tests/extras_dictcursor.py
index 70f51d23..a92968b1 100755
--- a/tests/extras_dictcursor.py
+++ b/tests/extras_dictcursor.py
@@ -261,6 +261,38 @@ class NamedTupleCursorTest(unittest.TestCase):
         finally:
             NamedTupleCursor._make_nt = f_orig
 
+    @skip_if_no_namedtuple
+    @skip_before_postgres(8, 0)
+    def test_named(self):
+        curs = self.conn.cursor('tmp')
+        curs.execute("""select i from generate_series(0,9) i""")
+        recs = []
+        recs.extend(curs.fetchmany(5))
+        recs.append(curs.fetchone())
+        recs.extend(curs.fetchall())
+        self.assertEqual(range(10), [t.i for t in recs])
+
+    @skip_if_no_namedtuple
+    def test_named_fetchone(self):
+        curs = self.conn.cursor('tmp')
+        curs.execute("""select 42 as i""")
+        t = curs.fetchone()
+        self.assertEqual(t.i, 42)
+
+    @skip_if_no_namedtuple
+    def test_named_fetchmany(self):
+        curs = self.conn.cursor('tmp')
+        curs.execute("""select 42 as i""")
+        recs = curs.fetchmany(10)
+        self.assertEqual(recs[0].i, 42)
+
+    @skip_if_no_namedtuple
+    def test_named_fetchall(self):
+        curs = self.conn.cursor('tmp')
+        curs.execute("""select 42 as i""")
+        recs = curs.fetchall()
+        self.assertEqual(recs[0].i, 42)
+
 
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)

From c61ec094a38ce2b0ecc3c6673065766f7ad9ea32 Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Tue, 26 Apr 2011 19:26:19 +0100
Subject: [PATCH 17/19] Don't fetch all the records iterating a NamedTuple
 cursor on a named cursor

---
 NEWS                       |  1 +
 lib/extras.py              |  9 ++++++++-
 tests/extras_dictcursor.py | 19 ++++++++++++++++++-
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 60c7d4b3..0037c3bb 100644
--- a/NEWS
+++ b/NEWS
@@ -8,6 +8,7 @@ What's new in psycopg 2.4.1
   - Correctly detect an empty query sent to the backend (ticket #46).
   - Fixed a SystemError clobbering libpq errors raised without SQLSTATE.
     Bug vivisectioned by Eric Snow.
+  - Fixed interaction between NamedTuple and server-side cursors.
   - Allow to specify --static-libpq on setup.py command line instead of
     just in 'setup.cfg'. Patch provided by Matthew Ryan (ticket #48).
 
diff --git a/lib/extras.py b/lib/extras.py
index dcbd65ed..1a4b730f 100644
--- a/lib/extras.py
+++ b/lib/extras.py
@@ -304,7 +304,14 @@ class NamedTupleCursor(_cursor):
         return [nt(*t) for t in ts]
 
     def __iter__(self):
-        return iter(self.fetchall())
+        # Invoking _cursor.__iter__(self) goes to infinite recursion,
+        # so we do pagination by hand
+        while 1:
+            recs = self.fetchmany(self.itersize)
+            if not recs:
+                return
+            for rec in recs:
+                yield rec
 
     try:
         from collections import namedtuple
diff --git a/tests/extras_dictcursor.py b/tests/extras_dictcursor.py
index a92968b1..494eca88 100755
--- a/tests/extras_dictcursor.py
+++ b/tests/extras_dictcursor.py
@@ -14,9 +14,11 @@
 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 # License for more details.
 
+import time
+from datetime import timedelta
 import psycopg2
 import psycopg2.extras
-from testutils import unittest, skip_if_no_namedtuple
+from testutils import unittest, skip_before_postgres, skip_if_no_namedtuple
 from testconfig import dsn
 
 
@@ -293,6 +295,21 @@ class NamedTupleCursorTest(unittest.TestCase):
         recs = curs.fetchall()
         self.assertEqual(recs[0].i, 42)
 
+    @skip_if_no_namedtuple
+    @skip_before_postgres(8, 0)
+    def test_not_greedy(self):
+        curs = self.conn.cursor('tmp')
+        curs.itersize = 2
+        curs.execute("""select clock_timestamp() as ts from generate_series(1,3)""")
+        recs = []
+        for t in curs:
+            time.sleep(0.01)
+            recs.append(t)
+
+        # check that the dataset was not fetched in a single gulp
+        self.assert_(recs[1].ts - recs[0].ts < timedelta(seconds=0.005))
+        self.assert_(recs[2].ts - recs[1].ts > timedelta(seconds=0.0099))
+
 
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)

From de6aff31b84ef03c22f08c24de9fcd318c3c83fb Mon Sep 17 00:00:00 2001
From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Wed, 27 Apr 2011 12:18:50 +0100
Subject: [PATCH 18/19] Skip a test on the proper PG function

---
 tests/extras_dictcursor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/extras_dictcursor.py b/tests/extras_dictcursor.py
index 494eca88..898c16c3 100755
--- a/tests/extras_dictcursor.py
+++ b/tests/extras_dictcursor.py
@@ -296,7 +296,7 @@ class NamedTupleCursorTest(unittest.TestCase):
         self.assertEqual(recs[0].i, 42)
 
     @skip_if_no_namedtuple
-    @skip_before_postgres(8, 0)
+    @skip_before_postgres(8, 2)
     def test_not_greedy(self):
         curs = self.conn.cursor('tmp')
         curs.itersize = 2

From 9080b30741e9a6f6b80a6700197445bff48df917 Mon Sep 17 00:00:00 2001
From: Federico Di Gregorio <fog@initd.org>
Date: Wed, 11 May 2011 09:58:34 +0200
Subject: [PATCH 19/19] Preparing release 2.4.1

---
 ZPsycopgDA/DA.py | 2 +-
 setup.py         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ZPsycopgDA/DA.py b/ZPsycopgDA/DA.py
index 8635ec5d..7a681e42 100644
--- a/ZPsycopgDA/DA.py
+++ b/ZPsycopgDA/DA.py
@@ -16,7 +16,7 @@
 # their work without bothering about the module dependencies.
 
 
-ALLOWED_PSYCOPG_VERSIONS = ('2.4-beta1', '2.4-beta2', '2.4')
+ALLOWED_PSYCOPG_VERSIONS = ('2.4-beta1', '2.4-beta2', '2.4', '2.4.1')
 
 import sys
 import time
diff --git a/setup.py b/setup.py
index 17eada11..9ae8117f 100644
--- a/setup.py
+++ b/setup.py
@@ -79,7 +79,7 @@ except ImportError:
 # Take a look at http://www.python.org/dev/peps/pep-0386/
 # for a consistent versioning pattern.
 
-PSYCOPG_VERSION = '2.4.1a0'
+PSYCOPG_VERSION = '2.4.1'
 
 version_flags   = ['dt', 'dec']