Merge branch 'master' into encrypt-pass

.travis.yml

@@ -6,12 +6,14 @@ language: python
 
 python:
   - 2.7
+  - 3.7-dev
   - 3.6
   - 3.5
   - 3.4
 
 install:
-  - python setup.py install
+  - pip install -U pip setuptools wheel
+  - pip install .
   - rm -rf psycopg2.egg-info
   - sudo scripts/travis_prepare.sh
 

LICENSE

@@ -1,5 +1,5 @@
 psycopg2 and the LGPL
-=====================
+---------------------
 
 psycopg2 is free software: you can redistribute it and/or modify it
 under the terms of the GNU Lesser General Public License as published
@@ -29,15 +29,15 @@ If not, see <http://www.gnu.org/licenses/>.
 
 
 Alternative licenses
-====================
+--------------------
 
 If you prefer you can use the Zope Database Adapter ZPsycopgDA (i.e.,
-every file inside the ZPsycopgDA directory) user the ZPL license as
+every file inside the ZPsycopgDA directory) using the ZPL license as
 published on the Zope web site, http://www.zope.org/Resources/ZPL.
 
 Also, the following BSD-like license applies (at your option) to the
-files following the pattern psycopg/adapter*.{h,c} and
-psycopg/microprotocol*.{h,c}:
+files following the pattern ``psycopg/adapter*.{h,c}`` and
+``psycopg/microprotocol*.{h,c}``:
 
  Permission is granted to anyone to use this software for any purpose,
  including commercial applications, and to alter it and redistribute it

NEWS

@@ -17,10 +17,14 @@ Other changes:
 What's new in psycopg 2.7.5
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Allow non-ascii chars in namedtuple fields (regression introduced fixing
+  :ticket:`#211`).
+- Fixed adaptation of arrays of arrays of nulls (:ticket:`#325`).
 - Fixed building on Solaris 11 and derivatives such as SmartOS and illumos
   (:ticket:`#677`).
 - Maybe fixed building on MSYS2 (as reported in :ticket:`#658`).
 - Allow string subclasses in connection and other places (:ticket:`#679`).
+- Don't raise an exception closing an unused named cursor (:ticket:`#716`).
 
 
 What's new in psycopg 2.7.4

README.rst

@@ -54,8 +54,8 @@ external libraries, by installing the `psycopg2-binary`_ package from PyPI::
 The binary package is a practical choice for development and testing but in
 production it is advised to use the package built from sources.
 
-.. _PyPI: https://pypi.python.org/pypi/psycopg2
-.. _psycopg2-binary: https://pypi.python.org/pypi/psycopg2-binary
+.. _PyPI: https://pypi.org/project/psycopg2/
+.. _psycopg2-binary: https://pypi.org/project/psycopg2-binary/
 .. _install: http://initd.org/psycopg/docs/install.html#install-from-source
 .. _faq: http://initd.org/psycopg/docs/faq.html#faq-compile
 

doc/Makefile

@@ -29,8 +29,6 @@ doctest:
 upload:
 	# this command requires ssh configured to the proper target
 	tar czf - -C html . | ssh psycoweb tar xzvf - -C docs/current
-	# this command requires a .pypirc with the right privileges
-	# python src/tools/pypi_docs_upload.py psycopg2 $$(pwd)/html
 
 clean:
 	$(MAKE) $(SPHOPTS) -C src $@

doc/release.rst

@@ -100,5 +100,5 @@ Test packages may be uploaded on the `PyPI testing site`__ using::
 
 assuming `proper configuration`__ of ``~/.pypirc``.
 
-.. __: https://testpypi.python.org/pypi/psycopg2
+.. __: https://test.pypi.org/project/psycopg2/
 .. __: https://wiki.python.org/moin/TestPyPI

doc/src/advanced.rst

@@ -485,7 +485,7 @@ details. You can check the `psycogreen`_ project for further informations and
 resources about the topic.
 
 .. _coroutine: http://en.wikipedia.org/wiki/Coroutine
-.. _greenlet: http://pypi.python.org/pypi/greenlet
+.. _greenlet: https://pypi.org/project/greenlet/
 .. _green threads: http://en.wikipedia.org/wiki/Green_threads
 .. _Eventlet: http://eventlet.net/
 .. _gevent: http://www.gevent.org/

doc/src/conf.py

@@ -61,8 +61,8 @@ except ImportError:
     release = version
 
 intersphinx_mapping = {
-    'py': ('http://docs.python.org/2', None),
-    'py3': ('http://docs.python.org/3', None),
+    'py': ('https://docs.python.org/2', None),
+    'py3': ('https://docs.python.org/3', None),
     }
 
 # Pattern to generate links to the bug tracker

doc/src/index.rst

@@ -48,6 +48,7 @@ Psycopg 2 is both Unicode and Python 3 friendly.
    errorcodes
    faq
    news
+   license
 
 
 .. ifconfig:: builder != 'text'

doc/src/install.rst

@@ -45,7 +45,9 @@ Build prerequisites
 ^^^^^^^^^^^^^^^^^^^
 
 The build prerequisites are to be met in order to install Psycopg from source
-code, either from a source distribution package or from PyPI.
+code, from a source distribution package, GitHub_ or from PyPI.
+
+.. _GitHub: https://github.com/psycopg/psycopg2
 
 Psycopg is a C wrapper around the libpq_ PostgreSQL client library. To install
 it from sources you will need:
@@ -140,7 +142,7 @@ Make sure to use an up-to-date version of :program:`pip` (you can upgrade it
 using something like ``pip install -U pip``)
 
 .. __: PyPI-binary_
-.. _PyPI-binary: https://pypi.python.org/pypi/psycopg2-binary/
+.. _PyPI-binary: https://pypi.org/project/psycopg2-binary/
 .. _wheel: http://pythonwheels.com/
 
 .. note::
@@ -302,10 +304,14 @@ Try the following. *In order:*
 - Google for `!psycopg2` *your error message*. Especially useful the week
   after the release of a new OS X version.
 
-- Write to the `Mailing List`__.
+- Write to the `Mailing List`_.
+
+- If you think that you have discovered a bug, test failure or missing feature
+  please raise a ticket in the `bug tracker`_.
 
 - Complain on your blog or on Twitter that `!psycopg2` is the worst package
   ever and about the quality time you have wasted figuring out the correct
   :envvar:`ARCHFLAGS`. Especially useful from the Starbucks near you.
 
-.. __: https://lists.postgresql.org/mj/mj_wwwusr?func=lists-long-full&extra=psycopg
+.. _mailing list: https://lists.postgresql.org/mj/mj_wwwusr?func=lists-long-full&extra=psycopg
+.. _bug tracker: https://github.com/psycopg/psycopg2/issues

doc/src/license.rst

@@ -0,0 +1,7 @@
+.. index::
+    single: License
+
+License
+=======
+
+.. include:: ../../LICENSE

doc/src/news.rst

@@ -1,3 +1,7 @@
+.. index::
+    single: Release notes
+    single: News
+
 Release notes
 =============
 

lib/extras.py

@@ -363,12 +363,15 @@ class NamedTupleCursor(_cursor):
             return
 
     def _make_nt(self):
+        # ascii except alnum and underscore
+        nochars = ' !"#$%&\'()*+,-./:;<=>?@[\\]^`{|}~'
+        re_clean = _re.compile('[' + _re.escape(nochars) + ']')
+
         def f(s):
-            # NOTE: Python 3 actually allows unicode chars in fields
-            s = _re.sub('[^a-zA-Z0-9_]', '_', s)
+            s = re_clean.sub('_', s)
             # Python identifier cannot start with numbers, namedtuple fields
             # cannot start with underscore. So...
-            if _re.match('^[0-9_]', s):
+            if s[0] == '_' or '0' <= s[0] <= '9':
                 s = 'f' + s
 
             return s

psycopg/adapter_list.c

@@ -38,13 +38,14 @@ list_quote(listObject *self)
 {
     /*  adapt the list by calling adapt() recursively and then wrapping
         everything into "ARRAY[]" */
-    PyObject *tmp = NULL, *str = NULL, *joined = NULL, *res = NULL;
+    PyObject *res = NULL;
+    PyObject **qs = NULL;
+    Py_ssize_t bufsize = 0;
+    char *buf = NULL, *ptr;
 
     /*  list consisting of only NULL don't work with the ARRAY[] construct
-     *  so we use the {NULL,...} syntax. Note however that list of lists where
-     *  some element is a list of only null still fails: for that we should use
-     *  the '{...}' syntax uniformly but we cannot do it in the current
-     *  infrastructure. TODO in psycopg3 */
+     *  so we use the {NULL,...} syntax. The same syntax is also necessary
+     *  to convert array of arrays containing only nulls. */
     int all_nulls = 1;
 
     Py_ssize_t i, len;
@@ -53,47 +54,95 @@ list_quote(listObject *self)
 
     /* empty arrays are converted to NULLs (still searching for a way to
        insert an empty array in postgresql */
-    if (len == 0) return Bytes_FromString("'{}'");
+    if (len == 0) {
+        res = Bytes_FromString("'{}'");
+        goto exit;
+    }
 
-    tmp = PyTuple_New(len);
+    if (!(qs = PyMem_New(PyObject *, len))) {
+        PyErr_NoMemory();
+        goto exit;
+    }
+    memset(qs, 0, len * sizeof(PyObject *));
 
-    for (i=0; i<len; i++) {
-        PyObject *quoted;
+    for (i = 0; i < len; i++) {
         PyObject *wrapped = PyList_GET_ITEM(self->wrapped, i);
         if (wrapped == Py_None) {
             Py_INCREF(psyco_null);
-            quoted = psyco_null;
+            qs[i] = psyco_null;
         }
         else {
-            quoted = microprotocol_getquoted(wrapped,
-                                       (connectionObject*)self->connection);
-            if (quoted == NULL) goto error;
-            all_nulls = 0;
+            if (!(qs[i] = microprotocol_getquoted(
+                    wrapped, (connectionObject*)self->connection))) {
+                goto exit;
+            }
+
+            /* Lists of arrays containing only nulls are also not supported
+             * by the ARRAY construct so we should do some special casing */
+            if (!PyList_Check(wrapped) || Bytes_AS_STRING(qs[i])[0] == 'A') {
+                all_nulls = 0;
+            }
         }
-
-        /* here we don't loose a refcnt: SET_ITEM does not change the
-           reference count and we are just transferring ownership of the tmp
-           object to the tuple */
-        PyTuple_SET_ITEM(tmp, i, quoted);
+        bufsize += Bytes_GET_SIZE(qs[i]) + 1;      /* this, and a comma */
     }
 
-    /* now that we have a tuple of adapted objects we just need to join them
-       and put "ARRAY[] around the result */
-    str = Bytes_FromString(", ");
-    joined = PyObject_CallMethod(str, "join", "(O)", tmp);
-    if (joined == NULL) goto error;
+    /* Create an array literal, usually ARRAY[...] but if the contents are
+     * all NULL or array of NULL we must use the '{...}' syntax
+     */
+    if (!(ptr = buf = PyMem_Malloc(bufsize + 8))) {
+        PyErr_NoMemory();
+        goto exit;
+    }
 
-    /* PG doesn't like ARRAY[NULL..] */
     if (!all_nulls) {
-        res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined));
-    } else {
-        res = Bytes_FromFormat("'{%s}'", Bytes_AsString(joined));
+        strcpy(ptr, "ARRAY[");
+        ptr += 6;
+        for (i = 0; i < len; i++) {
+            Py_ssize_t sl;
+            sl = Bytes_GET_SIZE(qs[i]);
+            memcpy(ptr, Bytes_AS_STRING(qs[i]), sl);
+            ptr += sl;
+            *ptr++ = ',';
+        }
+        *(ptr - 1) = ']';
+    }
+    else {
+        *ptr++ = '\'';
+        *ptr++ = '{';
+        for (i = 0; i < len; i++) {
+            /* in case all the adapted things are nulls (or array of nulls),
+             * the quoted string is either NULL or an array of the form
+             * '{NULL,...}', in which case we have to strip the extra quotes */
+            char *s;
+            Py_ssize_t sl;
+            s = Bytes_AS_STRING(qs[i]);
+            sl = Bytes_GET_SIZE(qs[i]);
+            if (s[0] != '\'') {
+                memcpy(ptr, s, sl);
+                ptr += sl;
+            }
+            else {
+                memcpy(ptr, s + 1, sl - 2);
+                ptr += sl - 2;
+            }
+            *ptr++ = ',';
+        }
+        *(ptr - 1) = '}';
+        *ptr++ = '\'';
     }
 
- error:
-    Py_XDECREF(tmp);
-    Py_XDECREF(str);
-    Py_XDECREF(joined);
+    res = Bytes_FromStringAndSize(buf, ptr - buf);
+
+exit:
+    if (qs) {
+        for (i = 0; i < len; i++) {
+            PyObject *q = qs[i];
+            Py_XDECREF(q);
+        }
+        PyMem_Free(qs);
+    }
+    PyMem_Free(buf);
+
     return res;
 }
 

psycopg/cursor_type.c

@@ -59,6 +59,11 @@ psyco_curs_close(cursorObject *self)
         char buffer[128];
         PGTransactionStatusType status;
 
+        if (!self->query) {
+            Dprintf("skipping named cursor close because unused");
+            goto close;
+        }
+
         if (self->conn) {
             status = PQtransactionStatus(self->conn->pgconn);
         }
@@ -66,17 +71,18 @@ psyco_curs_close(cursorObject *self)
             status = PQTRANS_UNKNOWN;
         }
 
-        if (!(status == PQTRANS_UNKNOWN || status == PQTRANS_INERROR)) {
-            EXC_IF_NO_MARK(self);
-            PyOS_snprintf(buffer, 127, "CLOSE %s", self->qname);
-            if (pq_execute(self, buffer, 0, 0, 1) == -1) return NULL;
-        }
-        else {
+        if (status == PQTRANS_UNKNOWN || status == PQTRANS_INERROR) {
             Dprintf("skipping named curs close because tx status %d",
                 (int)status);
+            goto close;
         }
+
+        EXC_IF_NO_MARK(self);
+        PyOS_snprintf(buffer, 127, "CLOSE %s", self->qname);
+        if (pq_execute(self, buffer, 0, 0, 1) == -1) return NULL;
     }
 
+close:
     self->closed = 1;
     Dprintf("psyco_curs_close: cursor at %p closed", self);
 

psycopg/typecast_datetime.c

@@ -406,6 +406,11 @@ typecast_PYINTERVAL_cast(const char *str, Py_ssize_t len, PyObject *curs)
             }
             break;
 
+        case 'P':
+            PyErr_SetString(NotSupportedError,
+                "iso_8601 intervalstyle currently not supported");
+            return NULL;
+
         default:
             break;
         }

scripts/travis_test.sh

@@ -56,15 +56,15 @@ fi
 # Unsupported postgres versions that we still support
 # Images built by https://github.com/psycopg/psycopg2-wheels/tree/build-dinosaurs
 if [[ -n "$TEST_PAST" ]]; then
-    run_test 7.4
-    run_test 8.0
-    run_test 8.1
-    run_test 8.2
-    run_test 8.3
-    run_test 8.4
-    run_test 9.0
-    run_test 9.1
     run_test 9.2
+    run_test 9.1
+    run_test 9.0
+    run_test 8.4
+    run_test 8.3
+    run_test 8.2
+    run_test 8.1
+    run_test 8.0
+    run_test 7.4
 fi
 
 # Postgres built from master

tests/test_cursor.py

@@ -435,6 +435,11 @@ class CursorTests(ConnectingTestCase):
         self.assertEqual([(2,), (3,), (4,)], cur2.fetchmany(3))
         self.assertEqual([(5,), (6,), (7,)], cur2.fetchall())
 
+    @skip_before_postgres(8, 0)
+    def test_named_noop_close(self):
+        cur = self.conn.cursor('test')
+        cur.close()
+
     @skip_before_postgres(8, 0)
     def test_scroll(self):
         cur = self.conn.cursor()

tests/test_dates.py

@@ -438,6 +438,14 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin):
             r = cur.fetchone()[0]
             self.assertEqual(r, v, "%s -> %s != %s" % (s, r, v))
 
+    @skip_before_postgres(8, 4)
+    def test_interval_iso_8601_not_supported(self):
+        # We may end up supporting, but no pressure for it
+        cur = self.conn.cursor()
+        cur.execute("set local intervalstyle to iso_8601")
+        cur.execute("select '1 day 2 hours'::interval")
+        self.assertRaises(psycopg2.NotSupportedError, cur.fetchone)
+
 
 # Only run the datetime tests if psycopg was compiled with support.
 if not hasattr(psycopg2.extensions, 'PYDATETIME'):

tests/test_extras_dictcursor.py

@@ -19,7 +19,7 @@ from datetime import timedelta
 import psycopg2
 import psycopg2.extras
 import unittest
-from .testutils import ConnectingTestCase, skip_before_postgres
+from .testutils import ConnectingTestCase, skip_before_postgres, skip_before_python
 
 
 class ExtrasDictCursorTests(ConnectingTestCase):
@@ -357,6 +357,14 @@ class NamedTupleCursorTest(ConnectingTestCase):
         self.assertEqual(rv.f_column_, 2)
         self.assertEqual(rv.f3, 3)
 
+    @skip_before_python(3)
+    @skip_before_postgres(8)
+    def test_nonascii_name(self):
+        curs = self.conn.cursor()
+        curs.execute('select 1 as \xe5h\xe9')
+        rv = curs.fetchone()
+        self.assertEqual(getattr(rv, '\xe5h\xe9'), 1)
+
     def test_minimal_generation(self):
         # Instrument the class to verify it gets called the minimum number of times.
         from psycopg2.extras import NamedTupleCursor

tests/test_types_basic.py

@@ -224,16 +224,31 @@ class TypesBasicTests(ConnectingTestCase):
         curs.execute("insert into na (boola) values (%s)", ([True, None],))
         curs.execute("insert into na (boola) values (%s)", ([None, None],))
 
-        # TODO: array of array of nulls are not supported yet
-        # curs.execute("insert into na (textaa) values (%s)", ([[None]],))
+        curs.execute("insert into na (textaa) values (%s)", ([[None]],))
         curs.execute("insert into na (textaa) values (%s)", ([['a', None]],))
-        # curs.execute("insert into na (textaa) values (%s)", ([[None, None]],))
-        # curs.execute("insert into na (intaa) values (%s)",  ([[None]],))
+        curs.execute("insert into na (textaa) values (%s)", ([[None, None]],))
+
+        curs.execute("insert into na (intaa) values (%s)",  ([[None]],))
         curs.execute("insert into na (intaa) values (%s)", ([[42, None]],))
-        # curs.execute("insert into na (intaa) values (%s)",  ([[None, None]],))
-        # curs.execute("insert into na (boolaa) values (%s)", ([[None]],))
+        curs.execute("insert into na (intaa) values (%s)",  ([[None, None]],))
+
+        curs.execute("insert into na (boolaa) values (%s)", ([[None]],))
         curs.execute("insert into na (boolaa) values (%s)", ([[True, None]],))
-        # curs.execute("insert into na (boolaa) values (%s)", ([[None, None]],))
+        curs.execute("insert into na (boolaa) values (%s)", ([[None, None]],))
+
+    @testutils.skip_before_postgres(8, 2)
+    def testNestedArrays(self):
+        curs = self.conn.cursor()
+        for a in [
+            [[1]],
+            [[None]],
+            [[None, None, None]],
+            [[None, None], [1, None]],
+            [[None, None], [None, None]],
+            [[[None, None], [None, None]]],
+        ]:
+            curs.execute("select %s::int[]", (a,))
+            self.assertEqual(curs.fetchone()[0], a)
 
     @testutils.skip_from_python(3)
     def testTypeRoundtripBuffer(self):

tests/test_types_extras.py

@@ -179,8 +179,8 @@ class HstoreTestCase(ConnectingTestCase):
         m = re.match(br'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)', q)
         self.assert_(m, repr(q))
 
-        kk = m.group(1).split(b", ")
-        vv = m.group(2).split(b", ")
+        kk = m.group(1).split(b",")
+        vv = m.group(2).split(b",")
         ii = list(zip(kk, vv))
         ii.sort()
 

tests/test_with.py

@@ -26,7 +26,7 @@ import psycopg2
 import psycopg2.extensions as ext
 
 import unittest
-from .testutils import ConnectingTestCase
+from .testutils import ConnectingTestCase, skip_before_postgres
 
 
 class WithTestCase(ConnectingTestCase):
@@ -215,6 +215,11 @@ class WithCursorTestCase(WithTestCase):
         else:
             self.fail("where is my exception?")
 
+    @skip_before_postgres(8, 0)
+    def test_named_with_noop(self):
+        with self.conn.cursor('named') as cur:
+            pass
+
 
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)

tox.ini

@@ -1,13 +1,9 @@
-# Tox (http://tox.testrun.org/) is a tool for running tests
-# in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
-# and then run "tox" from this directory.
-
 [tox]
-envlist = py27
+envlist = py{27,34,35,36}
 
 [testenv]
 commands = make check
+whitelist_externals = make
 
 [flake8]
 max-line-length = 85