From c687f1d816823cd538ae6158f25b44da9ffa31c4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 11 May 2011 12:58:38 +0100 Subject: [PATCH 01/47] Bump to next development release --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9ae8117f..de7ef106 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ except ImportError: # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. -PSYCOPG_VERSION = '2.4.1' +PSYCOPG_VERSION = '2.4.2.dev0' version_flags = ['dt', 'dec'] From af424821b70299704449337904d3ab7fb1b40751 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 4 May 2011 01:07:07 +0100 Subject: [PATCH 02/47] Don't build mx.DateTime support if the module can't be imported Previously we only checked for the existence of the include files, but this doesn't imply the presence of the module. Particularly true in restricted environments such as virtualenv. Closes ticket #53. --- NEWS | 7 +++++++ setup.py | 18 ++++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 0037c3bb..ff9d0eb1 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +What's new in psycopg 2.4.2 +--------------------------- + + - Don't build mx.DateTime support if the module can't be imported + (ticket #53). + + What's new in psycopg 2.4.1 --------------------------- diff --git a/setup.py b/setup.py index de7ef106..46e1aeea 100644 --- a/setup.py +++ b/setup.py @@ -450,12 +450,18 @@ if parser.has_option('build_ext', 'mx_include_dir'): else: mxincludedir = os.path.join(get_python_inc(plat_specific=1), "mx") if os.path.exists(mxincludedir): - include_dirs.append(mxincludedir) - define_macros.append(('HAVE_MXDATETIME','1')) - sources.append('adapter_mxdatetime.c') - depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) - have_mxdatetime = True - version_flags.append('mx') + # Check if mx.datetime is importable at all: see ticket #53 + try: + import mx.DateTime + except ImportError: + pass + else: + include_dirs.append(mxincludedir) + define_macros.append(('HAVE_MXDATETIME','1')) + sources.append('adapter_mxdatetime.c') + depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) + have_mxdatetime = True + version_flags.append('mx') # now decide which package will be the default for date/time typecasts if have_pydatetime and (use_pydatetime or not have_mxdatetime): From 834c7d1288fb65e25190fa404de6a1c32b735ba4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 4 May 2011 01:15:46 +0100 Subject: [PATCH 03/47] Fixed a few docstrings mixed up --- psycopg/psycopgmodule.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 01069414..af101c3e 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -757,11 +757,11 @@ static PyMethodDef psycopgMethods[] = { {"QuotedString", (PyCFunction)psyco_QuotedString, METH_VARARGS, psyco_QuotedString_doc}, {"Boolean", (PyCFunction)psyco_Boolean, - METH_VARARGS, psyco_Float_doc}, - {"Float", (PyCFunction)psyco_Float, - METH_VARARGS, psyco_Decimal_doc}, - {"Decimal", (PyCFunction)psyco_Decimal, METH_VARARGS, psyco_Boolean_doc}, + {"Float", (PyCFunction)psyco_Float, + METH_VARARGS, psyco_Float_doc}, + {"Decimal", (PyCFunction)psyco_Decimal, + METH_VARARGS, psyco_Decimal_doc}, {"Binary", (PyCFunction)psyco_Binary, METH_VARARGS, psyco_Binary_doc}, {"Date", (PyCFunction)psyco_Date, From 19ec8809fde2a678eeb9762293cdfdb080bd6a01 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 11 May 2011 12:51:44 +0100 Subject: [PATCH 04/47] Use all the isolation levels accepted by PostgreSQL In PG 9.1 repeatable read and serializable are distinct levels. --- NEWS | 2 ++ lib/extensions.py | 12 +++++------- psycopg/connection.h | 8 +++++--- psycopg/connection_type.c | 4 ++-- psycopg/pqpath.c | 2 ++ tests/test_connection.py | 32 +++++++++++++++++++------------- 6 files changed, 35 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index ff9d0eb1..1c439323 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ What's new in psycopg 2.4.2 --------------------------- + - Allow using the isolation level "repeatable read" which is distinct + from "serializable" in PostgreSQL 9.1. - Don't build mx.DateTime support if the module can't be imported (ticket #53). diff --git a/lib/extensions.py b/lib/extensions.py index 82e17fa4..cb48b793 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -68,13 +68,11 @@ except ImportError: pass """Isolation level values.""" -ISOLATION_LEVEL_AUTOCOMMIT = 0 -ISOLATION_LEVEL_READ_COMMITTED = 1 -ISOLATION_LEVEL_SERIALIZABLE = 2 - -# PostgreSQL maps the the other standard values to already defined levels -ISOLATION_LEVEL_REPEATABLE_READ = ISOLATION_LEVEL_SERIALIZABLE -ISOLATION_LEVEL_READ_UNCOMMITTED = ISOLATION_LEVEL_READ_COMMITTED +ISOLATION_LEVEL_AUTOCOMMIT = 0 +ISOLATION_LEVEL_READ_UNCOMMITTED = 1 +ISOLATION_LEVEL_READ_COMMITTED = 2 +ISOLATION_LEVEL_REPEATABLE_READ = 3 +ISOLATION_LEVEL_SERIALIZABLE = 4 """psycopg connection status values.""" STATUS_SETUP = 0 diff --git a/psycopg/connection.h b/psycopg/connection.h index 552b93d7..262e6ac2 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -63,9 +63,11 @@ extern "C" { /* possible values for isolation_level */ typedef enum { - ISOLATION_LEVEL_AUTOCOMMIT = 0, - ISOLATION_LEVEL_READ_COMMITTED = 1, - ISOLATION_LEVEL_SERIALIZABLE = 2, + ISOLATION_LEVEL_AUTOCOMMIT = 0, + ISOLATION_LEVEL_READ_UNCOMMITTED = 1, + ISOLATION_LEVEL_READ_COMMITTED = 2, + ISOLATION_LEVEL_REPEATABLE_READ = 3, + ISOLATION_LEVEL_SERIALIZABLE = 4, } conn_isolation_level_t; extern HIDDEN PyTypeObject connectionType; diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 7ca395dc..2d02b86b 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -400,9 +400,9 @@ psyco_conn_set_isolation_level(connectionObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "i", &level)) return NULL; - if (level < 0 || level > 2) { + if (level < 0 || level > 4) { PyErr_SetString(PyExc_ValueError, - "isolation level must be between 0 and 2"); + "isolation level must be between 0 and 4"); return NULL; } diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 6a6d05a3..a6144229 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -408,7 +408,9 @@ pq_begin_locked(connectionObject *conn, PGresult **pgres, char **error, { const char *query[] = { NULL, + "BEGIN; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED", "BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED", + "BEGIN; SET TRANSACTION ISOLATION LEVEL REPEATABLE READ", "BEGIN; SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"}; int result; diff --git a/tests/test_connection.py b/tests/test_connection.py index d9da471f..89b104f4 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -201,24 +201,30 @@ class IsolationLevelsTestCase(unittest.TestCase): def test_set_isolation_level(self): conn = self.connect() + curs = conn.cursor() - conn.set_isolation_level( - psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - self.assertEqual(conn.isolation_level, - psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + for name, level in ( + (None, psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT), + ('read uncommitted', psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED), + ('read committed', psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED), + ('repeatable read', psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ), + ('serializable', psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE), + ): + conn.set_isolation_level(level) + self.assertEqual(conn.isolation_level, level) - conn.set_isolation_level( - psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) - self.assertEqual(conn.isolation_level, - psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) + curs.execute('show transaction_isolation;') + got_name = curs.fetchone()[0] - conn.set_isolation_level( - psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) - self.assertEqual(conn.isolation_level, - psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) + if name is None: + curs.execute('show default_transaction_isolation;') + name = curs.fetchone()[0] + + self.assertEqual(name, got_name) + conn.commit() self.assertRaises(ValueError, conn.set_isolation_level, -1) - self.assertRaises(ValueError, conn.set_isolation_level, 3) + self.assertRaises(ValueError, conn.set_isolation_level, 5) def test_set_isolation_level_abort(self): conn = self.connect() From 281427f450d6e9755d4c3cbc9fb159d45ca10ee6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 30 May 2011 22:00:20 +0100 Subject: [PATCH 05/47] Fixed escape for negative numbers prefixed by minus operator Closes ticket #57. --- NEWS | 7 + lib/extensions.py | 2 +- psycopg/adapter_pdecimal.c | 35 ++++- psycopg/adapter_pfloat.c | 25 +++- psycopg/adapter_pint.c | 266 +++++++++++++++++++++++++++++++++++++ psycopg/adapter_pint.h | 53 ++++++++ psycopg/psycopgmodule.c | 10 +- psycopg/python.h | 2 + setup.py | 4 +- tests/types_basic.py | 10 ++ 10 files changed, 401 insertions(+), 13 deletions(-) create mode 100644 psycopg/adapter_pint.c create mode 100644 psycopg/adapter_pint.h diff --git a/NEWS b/NEWS index 0037c3bb..08c01f20 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,10 @@ +What's new in psycopg 2.4.2 +--------------------------- + + - Fixed escape for negative numbers prefixed by minus operator + (ticket #57). + + What's new in psycopg 2.4.1 --------------------------- diff --git a/lib/extensions.py b/lib/extensions.py index 82e17fa4..838d8691 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -39,7 +39,7 @@ from psycopg2._psycopg import DECIMALARRAY, FLOATARRAY, INTEGERARRAY, INTERVALAR from psycopg2._psycopg import LONGINTEGERARRAY, ROWIDARRAY, STRINGARRAY, TIMEARRAY from psycopg2._psycopg import UNICODEARRAY -from psycopg2._psycopg import Binary, Boolean, Float, QuotedString, AsIs +from psycopg2._psycopg import Binary, Boolean, Int, Float, QuotedString, AsIs try: from psycopg2._psycopg import MXDATE, MXDATETIME, MXINTERVAL, MXTIME from psycopg2._psycopg import MXDATEARRAY, MXDATETIMEARRAY, MXINTERVALARRAY, MXTIMEARRAY diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index 9b573467..e14e7694 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -41,8 +41,10 @@ pdecimal_getquoted(pdecimalObject *self, PyObject *args) PyObject *check, *res = NULL; check = PyObject_CallMethod(self->wrapped, "is_finite", NULL); if (check == Py_True) { - res = PyObject_Str(self->wrapped); - goto end; + if (!(res = PyObject_Str(self->wrapped))) { + goto end; + } + goto output; } else if (check) { res = Bytes_FromString("'NaN'::numeric"); @@ -70,16 +72,39 @@ pdecimal_getquoted(pdecimalObject *self, PyObject *args) goto end; } - res = PyObject_Str(self->wrapped); + /* wrapped is finite */ + if (!(res = PyObject_Str(self->wrapped))) { + goto end; + } + + /* res may be unicode and may suffer for issue #57 */ +output: + #if PY_MAJOR_VERSION > 2 /* unicode to bytes in Py3 */ - if (res) { + { PyObject *tmp = PyUnicode_AsUTF8String(res); Py_DECREF(res); - res = tmp; + if (!(res = tmp)) { + goto end; + } } #endif + if ('-' == Bytes_AS_STRING(res)[0]) { + /* Prepend a space in front of negative numbers (ticket #57) */ + PyObject *tmp; + if (!(tmp = Bytes_FromString(" "))) { + Py_DECREF(res); + res = NULL; + goto end; + } + Bytes_ConcatAndDel(&tmp, res); + if (!(res = tmp)) { + goto end; + } + } + end: Py_XDECREF(check); return res; diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 715ed8f2..1b8074f9 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -49,18 +49,37 @@ pfloat_getquoted(pfloatObject *self, PyObject *args) rv = Bytes_FromString("'-Infinity'::float"); } else { - rv = PyObject_Repr(self->wrapped); + if (!(rv = PyObject_Repr(self->wrapped))) { + goto exit; + } #if PY_MAJOR_VERSION > 2 /* unicode to bytes in Py3 */ - if (rv) { + { PyObject *tmp = PyUnicode_AsUTF8String(rv); Py_DECREF(rv); - rv = tmp; + if (!(rv = tmp)) { + goto exit; + } } #endif + + if ('-' == Bytes_AS_STRING(rv)[0]) { + /* Prepend a space in front of negative numbers (ticket #57) */ + PyObject *tmp; + if (!(tmp = Bytes_FromString(" "))) { + Py_DECREF(rv); + rv = NULL; + goto exit; + } + Bytes_ConcatAndDel(&tmp, rv); + if (!(rv = tmp)) { + goto exit; + } + } } +exit: return rv; } diff --git a/psycopg/adapter_pint.c b/psycopg/adapter_pint.c new file mode 100644 index 00000000..ad89a06b --- /dev/null +++ b/psycopg/adapter_pint.c @@ -0,0 +1,266 @@ +/* adapter_int.c - psycopg pint type wrapper implementation + * + * Copyright (C) 2011 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/adapter_pint.h" +#include "psycopg/microprotocols_proto.h" + + +/** the Int object **/ + +static PyObject * +pint_getquoted(pintObject *self, PyObject *args) +{ + PyObject *res; + if (!(res = PyObject_Str(self->wrapped))) { + goto exit; + } + +#if PY_MAJOR_VERSION > 2 + /* unicode to bytes in Py3 */ + { + PyObject *tmp = PyUnicode_AsUTF8String(res); + Py_DECREF(res); + if (!(res = tmp)) { + goto exit; + } + } +#endif + + if ('-' == Bytes_AS_STRING(res)[0]) { + /* Prepend a space in front of negative numbers (ticket #57) */ + PyObject *tmp; + if (!(tmp = Bytes_FromString(" "))) { + Py_DECREF(res); + res = NULL; + goto exit; + } + Bytes_ConcatAndDel(&tmp, res); + if (!(res = tmp)) { + goto exit; + } + } + +exit: + return res; +} + +static PyObject * +pint_str(pintObject *self) +{ + return psycopg_ensure_text(pint_getquoted(self, NULL)); +} + +static PyObject * +pint_conform(pintObject *self, PyObject *args) +{ + PyObject *res, *proto; + + if (!PyArg_ParseTuple(args, "O", &proto)) return NULL; + + if (proto == (PyObject*)&isqlquoteType) + res = (PyObject*)self; + else + res = Py_None; + + Py_INCREF(res); + return res; +} + +/** the int object */ + +/* object member list */ + +static struct PyMemberDef pintObject_members[] = { + {"adapted", T_OBJECT, offsetof(pintObject, wrapped), READONLY}, + {NULL} +}; + +/* object method table */ + +static PyMethodDef pintObject_methods[] = { + {"getquoted", (PyCFunction)pint_getquoted, METH_NOARGS, + "getquoted() -> wrapped object value as SQL-quoted string"}, + {"__conform__", (PyCFunction)pint_conform, METH_VARARGS, NULL}, + {NULL} /* Sentinel */ +}; + +/* initialization and finalization methods */ + +static int +pint_setup(pintObject *self, PyObject *obj) +{ + Dprintf("pint_setup: init pint object at %p, refcnt = " + FORMAT_CODE_PY_SSIZE_T, + self, Py_REFCNT(self) + ); + + Py_INCREF(obj); + self->wrapped = obj; + + Dprintf("pint_setup: good pint object at %p, refcnt = " + FORMAT_CODE_PY_SSIZE_T, + self, Py_REFCNT(self) + ); + return 0; +} + +static int +pint_traverse(PyObject *obj, visitproc visit, void *arg) +{ + pintObject *self = (pintObject *)obj; + + Py_VISIT(self->wrapped); + return 0; +} + +static void +pint_dealloc(PyObject* obj) +{ + pintObject *self = (pintObject *)obj; + + Py_CLEAR(self->wrapped); + + Dprintf("pint_dealloc: deleted pint object at %p, refcnt = " + FORMAT_CODE_PY_SSIZE_T, + obj, Py_REFCNT(obj) + ); + + Py_TYPE(obj)->tp_free(obj); +} + +static int +pint_init(PyObject *obj, PyObject *args, PyObject *kwds) +{ + PyObject *o; + + if (!PyArg_ParseTuple(args, "O", &o)) + return -1; + + return pint_setup((pintObject *)obj, o); +} + +static PyObject * +pint_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + return type->tp_alloc(type, 0); +} + +static void +pint_del(PyObject* self) +{ + PyObject_GC_Del(self); +} + +static PyObject * +pint_repr(pintObject *self) +{ + return PyString_FromFormat("", + self); +} + + +/* object type */ + +#define pintType_doc \ +"Int(str) -> new Int adapter object" + +PyTypeObject pintType = { + PyVarObject_HEAD_INIT(NULL, 0) + "psycopg2._psycopg.Int", + sizeof(pintObject), + 0, + pint_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + + 0, /*tp_compare*/ + + (reprfunc)pint_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + + 0, /*tp_call*/ + (reprfunc)pint_str, /*tp_str*/ + + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + pintType_doc, /*tp_doc*/ + + pint_traverse, /*tp_traverse*/ + 0, /*tp_clear*/ + + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + + /* Attribute descriptor and subclassing stuff */ + + pintObject_methods, /*tp_methods*/ + pintObject_members, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + + pint_init, /*tp_init*/ + 0, /*tp_alloc will be set to PyType_GenericAlloc in module init*/ + pint_new, /*tp_new*/ + (freefunc)pint_del, /*tp_free Low-level free-memory routine */ + 0, /*tp_is_gc For PyObject_IS_GC */ + 0, /*tp_bases*/ + 0, /*tp_mro method resolution order */ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0 /*tp_weaklist*/ +}; + + +/** module-level functions **/ + +PyObject * +psyco_Int(PyObject *module, PyObject *args) +{ + PyObject *obj; + + if (!PyArg_ParseTuple(args, "O", &obj)) + return NULL; + + return PyObject_CallFunctionObjArgs((PyObject *)&pintType, obj, NULL); +} diff --git a/psycopg/adapter_pint.h b/psycopg/adapter_pint.h new file mode 100644 index 00000000..fd553e8b --- /dev/null +++ b/psycopg/adapter_pint.h @@ -0,0 +1,53 @@ +/* adapter_pint.h - definition for the psycopg int type wrapper + * + * Copyright (C) 2011 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#ifndef PSYCOPG_PINT_H +#define PSYCOPG_PINT_H 1 + +#ifdef __cplusplus +extern "C" { +#endif + +extern HIDDEN PyTypeObject pintType; + +typedef struct { + PyObject_HEAD + + /* this is the real object we wrap */ + PyObject *wrapped; + +} pintObject; + +/* functions exported to psycopgmodule.c */ + +HIDDEN PyObject *psyco_Int(PyObject *module, PyObject *args); +#define psyco_Int_doc \ + "Int(obj) -> new int value" + +#ifdef __cplusplus +} +#endif + +#endif /* !defined(PSYCOPG_PINT_H) */ diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 01069414..99d7c6c0 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -39,6 +39,7 @@ #include "psycopg/adapter_qstring.h" #include "psycopg/adapter_binary.h" #include "psycopg/adapter_pboolean.h" +#include "psycopg/adapter_pint.h" #include "psycopg/adapter_pfloat.h" #include "psycopg/adapter_pdecimal.h" #include "psycopg/adapter_asis.h" @@ -316,9 +317,9 @@ psyco_adapters_init(PyObject *mod) microprotocols_add(&PyFloat_Type, NULL, (PyObject*)&pfloatType); #if PY_MAJOR_VERSION < 3 - microprotocols_add(&PyInt_Type, NULL, (PyObject*)&asisType); + microprotocols_add(&PyInt_Type, NULL, (PyObject*)&pintType); #endif - microprotocols_add(&PyLong_Type, NULL, (PyObject*)&asisType); + microprotocols_add(&PyLong_Type, NULL, (PyObject*)&pintType); microprotocols_add(&PyBool_Type, NULL, (PyObject*)&pbooleanType); /* strings */ @@ -758,6 +759,8 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS, psyco_QuotedString_doc}, {"Boolean", (PyCFunction)psyco_Boolean, METH_VARARGS, psyco_Float_doc}, + {"Int", (PyCFunction)psyco_Int, + METH_VARARGS, psyco_Int_doc}, {"Float", (PyCFunction)psyco_Float, METH_VARARGS, psyco_Decimal_doc}, {"Decimal", (PyCFunction)psyco_Decimal, @@ -848,6 +851,7 @@ INIT_MODULE(_psycopg)(void) Py_TYPE(&binaryType) = &PyType_Type; Py_TYPE(&isqlquoteType) = &PyType_Type; Py_TYPE(&pbooleanType) = &PyType_Type; + Py_TYPE(&pintType) = &PyType_Type; Py_TYPE(&pfloatType) = &PyType_Type; Py_TYPE(&pdecimalType) = &PyType_Type; Py_TYPE(&asisType) = &PyType_Type; @@ -863,6 +867,7 @@ INIT_MODULE(_psycopg)(void) if (PyType_Ready(&binaryType) == -1) goto exit; if (PyType_Ready(&isqlquoteType) == -1) goto exit; if (PyType_Ready(&pbooleanType) == -1) goto exit; + if (PyType_Ready(&pintType) == -1) goto exit; if (PyType_Ready(&pfloatType) == -1) goto exit; if (PyType_Ready(&pdecimalType) == -1) goto exit; if (PyType_Ready(&asisType) == -1) goto exit; @@ -978,6 +983,7 @@ INIT_MODULE(_psycopg)(void) binaryType.tp_alloc = PyType_GenericAlloc; isqlquoteType.tp_alloc = PyType_GenericAlloc; pbooleanType.tp_alloc = PyType_GenericAlloc; + pintType.tp_alloc = PyType_GenericAlloc; pfloatType.tp_alloc = PyType_GenericAlloc; pdecimalType.tp_alloc = PyType_GenericAlloc; connectionType.tp_alloc = PyType_GenericAlloc; diff --git a/psycopg/python.h b/psycopg/python.h index fed0303e..ed69f4c3 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -129,6 +129,7 @@ typedef unsigned long Py_uhash_t; #define Bytes_FromString PyString_FromString #define Bytes_FromStringAndSize PyString_FromStringAndSize #define Bytes_FromFormat PyString_FromFormat +#define Bytes_ConcatAndDel PyString_ConcatAndDel #define _Bytes_Resize _PyString_Resize #else @@ -144,6 +145,7 @@ typedef unsigned long Py_uhash_t; #define Bytes_FromString PyBytes_FromString #define Bytes_FromStringAndSize PyBytes_FromStringAndSize #define Bytes_FromFormat PyBytes_FromFormat +#define Bytes_ConcatAndDel PyBytes_ConcatAndDel #define _Bytes_Resize _PyBytes_Resize #endif diff --git a/setup.py b/setup.py index 9ae8117f..bb754116 100644 --- a/setup.py +++ b/setup.py @@ -414,7 +414,7 @@ sources = [ 'adapter_asis.c', 'adapter_binary.c', 'adapter_datetime.c', 'adapter_list.c', 'adapter_pboolean.c', 'adapter_pdecimal.c', - 'adapter_pfloat.c', 'adapter_qstring.c', + 'adapter_pint.c', 'adapter_pfloat.c', 'adapter_qstring.c', 'microprotocols.c', 'microprotocols_proto.c', 'typecast.c', ] @@ -427,7 +427,7 @@ depends = [ 'adapter_asis.h', 'adapter_binary.h', 'adapter_datetime.h', 'adapter_list.h', 'adapter_pboolean.h', 'adapter_pdecimal.h', - 'adapter_pfloat.h', 'adapter_qstring.h', + 'adapter_pint.h', 'adapter_pfloat.h', 'adapter_qstring.h', 'microprotocols.h', 'microprotocols_proto.h', 'typecast.h', 'typecast_binary.h', diff --git a/tests/types_basic.py b/tests/types_basic.py index 1ca668de..709907eb 100755 --- a/tests/types_basic.py +++ b/tests/types_basic.py @@ -275,6 +275,16 @@ class TypesBasicTests(unittest.TestCase): o2 = self.execute("SELECT %s::bytea AS foo", (o1,)) self.assertEqual(b('x'), o2[0]) + def testNegNumber(self): + d1 = self.execute("select -%s;", (decimal.Decimal('-1.0'),)) + self.assertEqual(1, d1) + f1 = self.execute("select -%s;", (-1.0,)) + self.assertEqual(1, f1) + i1 = self.execute("select -%s;", (-1,)) + self.assertEqual(1, i1) + l1 = self.execute("select -%s;", (-1L,)) + self.assertEqual(1, l1) + class AdaptSubclassTest(unittest.TestCase): def test_adapt_subtype(self): From a69facc7f0abf96493a2d269365f1392c0e8143c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 31 May 2011 00:05:50 +0100 Subject: [PATCH 06/47] Adding docs for the planned set_transaction/autocommit features --- doc/src/advanced.rst | 13 ++++---- doc/src/conf.py | 4 +-- doc/src/connection.rst | 76 +++++++++++++++++++++++++++++++++++++++++- doc/src/faq.rst | 4 +-- 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index 3d95cb27..ac16ca9b 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -239,9 +239,8 @@ be sent from Python code simply executing a :sql:`NOTIFY` command in an `~cursor.execute()` call. Because of the way sessions interact with notifications (see |NOTIFY|_ -documentation), you should keep the connection in :ref:`autocommit -` mode if you wish to receive or send notifications in a timely -manner. +documentation), you should keep the connection in `~connection.autocommit` +mode if you wish to receive or send notifications in a timely manner. .. |LISTEN| replace:: :sql:`LISTEN` .. _LISTEN: http://www.postgresql.org/docs/9.0/static/sql-listen.html @@ -373,12 +372,14 @@ When an asynchronous query is being executed, `connection.isexecuting()` returns connection. There are several limitations in using asynchronous connections: the -connection is always in :ref:`autocommit ` mode and it is not -possible to change it using `~connection.set_isolation_level()`. So a +connection is always in `~connection.autocommit` mode and it is not +possible to change it. So a transaction is not implicitly started at the first query and is not possible to use methods `~connection.commit()` and `~connection.rollback()`: you can manually control transactions using `~cursor.execute()` to send database -commands such as :sql:`BEGIN`, :sql:`COMMIT` and :sql:`ROLLBACK`. +commands such as :sql:`BEGIN`, :sql:`COMMIT` and :sql:`ROLLBACK`. Similarly +`set_transaction()` can't be used but it is still possible to invoke the +:sql:`SET` command with the proper :sql:`default_transaction_...` parameter. With asynchronous connections it is also not possible to use `~connection.set_client_encoding()`, `~cursor.executemany()`, :ref:`large diff --git a/doc/src/conf.py b/doc/src/conf.py index 56a07683..db64f864 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -111,10 +111,10 @@ rst_epilog = """ .. _DBAPI: http://www.python.org/dev/peps/pep-0249/ .. _transaction isolation level: - http://www.postgresql.org/docs/9.0/static/transaction-iso.html + http://www.postgresql.org/docs/9.1/static/transaction-iso.html .. _serializable isolation level: - http://www.postgresql.org/docs/9.0/static/transaction-iso.html#XACT-SERIALIZABLE + http://www.postgresql.org/docs/9.1/static/transaction-iso.html#XACT-SERIALIZABLE .. _mx.DateTime: http://www.egenix.com/products/python/mxBase/mxDateTime/ diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 01032587..9564095e 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -327,11 +327,85 @@ The ``connection`` class pair: Transaction; Autocommit pair: Transaction; Isolation level - .. _autocommit: + .. method:: set_transaction(isolation_level=None, readonly=None, deferrable=None, autocommit=None) + + Set one or more parameters for the next transactions or statements in + the current session. See |SET TRANSACTION|_ for further details. + + .. |SET TRANSACTION| replace:: :sql:`SET TRANSACTION` + .. _SET TRANSACTION: http://www.postgresql.org/docs/9.1/static/sql-set-transaction.html + + :param isolation_level: set the `isolation level`_ for the next + transactions/statements. The value should be one of the + :ref:`constants ` defined in the + `~psycopg2.extensions` module. + :param readonly: if `!True`, set the connection to read only; + read/write if `!False`. + :param deferrable: if `!True`, set the connection to deferrable; + non deferrable if `!False`. Only available from PostgreSQL 9.1. + :param autocommit: switch the connection to autocommit mode: not a + PostgreSQL session setting but an alias for setting the + `autocommit` attribute. + + .. _isolation level: + http://www.postgresql.org/docs/9.1/static/transaction-iso.html + + The function must be invoked with no transaction in progress. At every + function invocation, only the parameters whose value is not `!None` are + changed. + + The default for the values are defined by the server configuration: + see values for |default_transaction_isolation|__, + |default_transaction_read_only|__, |default_transaction_deferrable|__. + + .. |default_transaction_isolation| replace:: :sql:`default_transaction_isolation` + .. __: http://www.postgresql.org/docs/9.1/static/runtime-config-client.html#GUC-DEFAULT-TRANSACTION-ISOLATION + .. |default_transaction_read_only| replace:: :sql:`default_transaction_read_only` + .. __: http://www.postgresql.org/docs/9.1/static/runtime-config-client.html#GUC-DEFAULT-TRANSACTION-READ-ONLY + .. |default_transaction_deferrable| replace:: :sql:`default_transaction_deferrable` + .. __: http://www.postgresql.org/docs/9.1/static/runtime-config-client.html#GUC-DEFAULT-TRANSACTION-DEFERRABLE + + .. note:: + + There is currently no builtin method to read the current value for + the parameters: use :sql:`SHOW default_transaction_...` to read + the values from the backend. + + .. versionadded:: 2.4.2 + + + .. attribute:: autocommit + + Read/write attribute: if `!True`, no transaction is handled by the + driver and every statement sent to the backend has immediate effect; + if `!False` a new transaction is started at the first command + execution: the methods `commit()` or `rollback()` must be manually + invoked to terminate the transaction. + + The default is `!False` (manual commit) as per DBAPI specification. + + .. warning:: + + By default, any query execution, including a simple :sql:`SELECT` + will start a transaction: for long-running program, if no further + action is taken, the session will remain "idle in transaction", a + condition non desiderable for several reasons (locks are held by + the session, tables bloat...). For long lived scripts, either + ensure to terminate a transaction as soon as possible or use an + autocommit connection. + + .. versionadded:: 2.4.2 + .. attribute:: isolation_level .. method:: set_isolation_level(level) + .. note:: + + From version 2.4.2, replaced by `set_transaction()` and + `autocommit`, offering finer control on the transaction + characteristics. + Read or set the `transaction isolation level`_ for the current session. The level defines the different phenomena that can happen in the database between concurrent transactions. diff --git a/doc/src/faq.rst b/doc/src/faq.rst index 4ebf15a5..e7fe76fc 100644 --- a/doc/src/faq.rst +++ b/doc/src/faq.rst @@ -22,8 +22,8 @@ Why does `!psycopg2` leave database sessions "idle in transaction"? call one of the transaction closing methods before leaving the connection unused for a long time (which may also be a few seconds, depending on the concurrency level in your database). Alternatively you can use a - connection in :ref:`autocommit ` mode to avoid a new - transaction to be started at the first command. + connection in `~connection.autocommit` mode to avoid a new transaction to + be started at the first command. I receive the error *current transaction is aborted, commands ignored until end of transaction block* and can't do anything else! There was a problem *in the previous* command to the database, which From ea03ffbf76c24e7ba36cc6c94ec2b0c748700c20 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 1 Jun 2011 09:07:02 +0100 Subject: [PATCH 07/47] Added partial implementation for set_transaction autocommit to be implemented yet. --- psycopg/connection.h | 9 +++ psycopg/connection_int.c | 40 ++++++++++ psycopg/connection_type.c | 149 ++++++++++++++++++++++++++++++++++++-- tests/test_connection.py | 146 ++++++++++++++++++++++++++++++++++++- 4 files changed, 336 insertions(+), 8 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index 262e6ac2..979b37fd 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -136,6 +136,8 @@ HIDDEN int conn_connect(connectionObject *self, long int async); HIDDEN void conn_close(connectionObject *self); HIDDEN int conn_commit(connectionObject *self); HIDDEN int conn_rollback(connectionObject *self); +HIDDEN int conn_set(connectionObject *self, const char *param, const char *value); +HIDDEN int conn_set_autocommit(connectionObject *self, int value); HIDDEN int conn_switch_isolation_level(connectionObject *self, int level); HIDDEN int conn_set_client_encoding(connectionObject *self, const char *enc); HIDDEN int conn_poll(connectionObject *self); @@ -154,6 +156,13 @@ HIDDEN PyObject *conn_tpc_recover(connectionObject *self); "in asynchronous mode"); \ return NULL; } +#define EXC_IF_IN_TRANSACTION(self, cmd) \ + if (self->status != CONN_STATUS_READY) { \ + PyErr_Format(ProgrammingError, \ + "%s cannot be used inside a transaction", #cmd); \ + return NULL; \ + } + #define EXC_IF_TPC_NOT_SUPPORTED(self) \ if ((self)->server_version < 80100) { \ PyErr_Format(NotSupportedError, \ diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 22c5bc59..b0ed41df 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -952,6 +952,46 @@ conn_rollback(connectionObject *self) return res; } +/* conn_set - set a guc parameter */ + +int +conn_set(connectionObject *self, const char *param, const char *value) +{ + char query[256]; + PGresult *pgres = NULL; + char *error = NULL; + int res = 1; + + Dprintf("conn_set: setting %s to %s", param, value); + + Py_BEGIN_ALLOW_THREADS; + pthread_mutex_lock(&self->lock); + + if (0 == strcmp(value, "default")) { + sprintf(query, "SET %s TO DEFAULT;", param); + } + else { + sprintf(query, "SET %s TO '%s';", param, value); + } + + res = pq_execute_command_locked(self, query, &pgres, &error, &_save); + + pthread_mutex_unlock(&self->lock); + Py_END_ALLOW_THREADS; + + if (res < 0) { + pq_complete_error(self, &pgres, &error); + } + + return res; +} + +int +conn_set_autocommit(connectionObject *self, int value) +{ + return -1; +} + /* conn_switch_isolation_level - switch isolation level on the connection */ int diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 2d02b86b..299888a1 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -187,6 +187,7 @@ psyco_conn_tpc_begin(connectionObject *self, PyObject *args) EXC_IF_CONN_CLOSED(self); EXC_IF_CONN_ASYNC(self, tpc_begin); EXC_IF_TPC_NOT_SUPPORTED(self); + EXC_IF_IN_TRANSACTION(self, tpc_begin); if (!PyArg_ParseTuple(args, "O", &oxid)) { goto exit; @@ -196,13 +197,6 @@ psyco_conn_tpc_begin(connectionObject *self, PyObject *args) goto exit; } - /* check we are not in a transaction */ - if (self->status != CONN_STATUS_READY) { - PyErr_SetString(ProgrammingError, - "tpc_begin must be called outside a transaction"); - goto exit; - } - /* two phase commit and autocommit make no point */ if (self->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT) { PyErr_SetString(ProgrammingError, @@ -384,6 +378,145 @@ psyco_conn_tpc_recover(connectionObject *self, PyObject *args) #ifdef PSYCOPG_EXTENSIONS + +/* parse a python object into one of the possible isolation level values */ + +static const char * +_psyco_conn_parse_isolevel(PyObject *pyval) +{ + const char **value = NULL; + + static const char *isolevels[] = { + "", /* autocommit */ + "read uncommitted", + "read committed", + "repeatable read", + "serializable", + "default", + NULL }; + + /* parse from one of the level constants */ + if (PyInt_Check(pyval)) { + long level = PyInt_AsLong(pyval); + if (level == -1 && PyErr_Occurred()) { return NULL; } + if (level < 1 || level > 4) { + PyErr_SetString(PyExc_ValueError, + "isolation_level must be between 1 and 4"); + return NULL; + } + value = isolevels + level; + } + + /* parse from the string -- this includes "default" */ + else { + value = isolevels; + while (*(++value)) { + int cmp; + PyObject *pylevel; + if (!(pylevel = Text_FromUTF8(*value))) { return NULL; } + cmp = PyObject_RichCompareBool(pylevel, pyval, Py_EQ); + Py_DECREF(pylevel); + if (-1 == cmp) { return NULL; } + if (cmp) { break; } + } + if (!*value) { + PyErr_SetString(PyExc_ValueError, + "bad value for isolation_level"); /* TODO: emit value */ + } + } + return *value; +} + +/* convert True/False/"default" into a C string */ + +static const char * +_psyco_conn_parse_onoff(PyObject *pyval) +{ + int istrue = PyObject_IsTrue(pyval); + if (-1 == istrue) { return NULL; } + if (istrue) { + int cmp; + PyObject *pydef; + if (!(pydef = Text_FromUTF8("default"))) { return NULL; } + cmp = PyObject_RichCompareBool(pyval, pydef, Py_EQ); + Py_DECREF(pydef); + if (-1 == cmp) { return NULL; } + return cmp ? "default" : "on"; + } + else { + return "off"; + } +} + +/* set_transaction - default transaction characteristics */ + +#define psyco_conn_set_transaction_doc \ +"set_transaction(...) -- Set one or more parameters for the next transactions.\n\n" \ +"Accepted arguments are 'isolation_level', 'readonly', 'deferrable', 'autocommit'." + +static PyObject * +psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *isolation_level = Py_None; + PyObject *readonly = Py_None; + PyObject *deferrable = Py_None; + PyObject *autocommit = Py_None; + + static char *kwlist[] = + {"isolation_level", "readonly", "deferrable", "autocommit", NULL}; + + EXC_IF_CONN_CLOSED(self); + EXC_IF_CONN_ASYNC(self, set_transaction); + EXC_IF_IN_TRANSACTION(self, set_transaction); + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOOO", kwlist, + &isolation_level, &readonly, &deferrable, &autocommit)) { + return NULL; + } + + if (Py_None != isolation_level) { + const char *value = NULL; + if (!(value = _psyco_conn_parse_isolevel(isolation_level))) { + return NULL; + } + if (0 != conn_set(self, "default_transaction_isolation", value)) { + return NULL; + } + } + + if (Py_None != readonly) { + const char *value = NULL; + if (!(value = _psyco_conn_parse_onoff(readonly))) { + return NULL; + } + if (0 != conn_set(self, "default_transaction_read_only", value)) { + return NULL; + } + } + + if (Py_None != deferrable) { + const char *value = NULL; + if (!(value = _psyco_conn_parse_onoff(deferrable))) { + return NULL; + } + if (0 != conn_set(self, "default_transaction_deferrable", value)) { + return NULL; + } + } + + if (Py_None != autocommit) { + int value = PyObject_IsTrue(autocommit); + if (-1 == value) { return NULL; } + if (0 != conn_set_autocommit(self, value)) { + return NULL; + } + } + + Py_INCREF(Py_None); + return Py_None; +} + + /* set_isolation_level method - switch connection isolation level */ #define psyco_conn_set_isolation_level_doc \ @@ -717,6 +850,8 @@ static struct PyMethodDef connectionObject_methods[] = { {"tpc_recover", (PyCFunction)psyco_conn_tpc_recover, METH_NOARGS, psyco_conn_tpc_recover_doc}, #ifdef PSYCOPG_EXTENSIONS + {"set_transaction", (PyCFunction)psyco_conn_set_transaction, + METH_VARARGS|METH_KEYWORDS, psyco_conn_set_transaction_doc}, {"set_isolation_level", (PyCFunction)psyco_conn_set_isolation_level, METH_VARARGS, psyco_conn_set_isolation_level_doc}, {"set_client_encoding", (PyCFunction)psyco_conn_set_client_encoding, diff --git a/tests/test_connection.py b/tests/test_connection.py index 89b104f4..18c9277e 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -25,7 +25,8 @@ import os import time import threading -from testutils import unittest, decorate_all_tests, skip_before_postgres +from testutils import unittest, decorate_all_tests +from testutils import skip_before_postgres, skip_after_postgres from operator import attrgetter import psycopg2 @@ -707,6 +708,149 @@ from testutils import skip_if_tpc_disabled decorate_all_tests(ConnectionTwoPhaseTests, skip_if_tpc_disabled) +class TransactionControlTests(unittest.TestCase): + def setUp(self): + self.conn = psycopg2.connect(dsn) + + def tearDown(self): + if not self.conn.closed: + self.conn.close() + + def test_not_in_transaction(self): + cur = self.conn.cursor() + cur.execute("select 1") + self.assertRaises(psycopg2.ProgrammingError, + self.conn.set_transaction, + psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) + + def test_set_isolation_level(self): + cur = self.conn.cursor() + self.conn.set_transaction( + psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) + cur.execute("SHOW default_transaction_isolation;") + self.assertEqual(cur.fetchone()[0], 'serializable') + self.conn.rollback() + + self.conn.set_transaction( + psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ) + cur.execute("SHOW default_transaction_isolation;") + if self.conn.server_version > 80000: + self.assertEqual(cur.fetchone()[0], 'repeatable read') + else: + self.assertEqual(cur.fetchone()[0], 'serializable') + self.conn.rollback() + + self.conn.set_transaction( + isolation_level=psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) + cur.execute("SHOW default_transaction_isolation;") + self.assertEqual(cur.fetchone()[0], 'read committed') + self.conn.rollback() + + self.conn.set_transaction( + isolation_level=psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED) + cur.execute("SHOW default_transaction_isolation;") + if self.conn.server_version > 80000: + self.assertEqual(cur.fetchone()[0], 'read uncommitted') + else: + self.assertEqual(cur.fetchone()[0], 'read committed') + self.conn.rollback() + + def test_set_isolation_level_str(self): + cur = self.conn.cursor() + self.conn.set_transaction("serializable") + cur.execute("SHOW default_transaction_isolation;") + self.assertEqual(cur.fetchone()[0], 'serializable') + self.conn.rollback() + + self.conn.set_transaction("repeatable read") + cur.execute("SHOW default_transaction_isolation;") + if self.conn.server_version > 80000: + self.assertEqual(cur.fetchone()[0], 'repeatable read') + else: + self.assertEqual(cur.fetchone()[0], 'serializable') + self.conn.rollback() + + self.conn.set_transaction("read committed") + cur.execute("SHOW default_transaction_isolation;") + self.assertEqual(cur.fetchone()[0], 'read committed') + self.conn.rollback() + + self.conn.set_transaction("read uncommitted") + cur.execute("SHOW default_transaction_isolation;") + if self.conn.server_version > 80000: + self.assertEqual(cur.fetchone()[0], 'read uncommitted') + else: + self.assertEqual(cur.fetchone()[0], 'read committed') + self.conn.rollback() + + def test_bad_isolation_level(self): + self.assertRaises(ValueError, self.conn.set_transaction, 0) + self.assertRaises(ValueError, self.conn.set_transaction, 5) + self.assertRaises(ValueError, self.conn.set_transaction, 'whatever') + + def test_set_read_only(self): + cur = self.conn.cursor() + self.conn.set_transaction(readonly=True) + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], 'on') + self.conn.rollback() + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], 'on') + self.conn.rollback() + + cur = self.conn.cursor() + self.conn.set_transaction(readonly=None) + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], 'on') + self.conn.rollback() + + self.conn.set_transaction(readonly=False) + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], 'off') + self.conn.rollback() + + def test_set_default(self): + cur = self.conn.cursor() + cur.execute("SHOW default_transaction_isolation;") + default_isolevel = cur.fetchone()[0] + cur.execute("SHOW default_transaction_read_only;") + default_readonly = cur.fetchone()[0] + self.conn.rollback() + + self.conn.set_transaction(isolation_level='serializable', readonly=True) + self.conn.set_transaction(isolation_level='default', readonly='default') + + cur.execute("SHOW default_transaction_isolation;") + self.assertEqual(cur.fetchone()[0], default_isolevel) + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], default_readonly) + + @skip_before_postgres(9, 1) + def test_set_deferrable(self): + cur = self.conn.cursor() + self.conn.set_transaction(readonly=True, deferrable=True) + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], 'on') + cur.execute("SHOW default_transaction_deferrable;") + self.assertEqual(cur.fetchone()[0], 'on') + self.conn.rollback() + cur.execute("SHOW default_transaction_deferrable;") + self.assertEqual(cur.fetchone()[0], 'on') + self.conn.rollback() + + self.conn.set_transaction(deferrable=False) + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], 'on') + cur.execute("SHOW default_transaction_deferrable;") + self.assertEqual(cur.fetchone()[0], 'off') + self.conn.rollback() + + @skip_after_postgres(9, 1) + def test_set_deferrable_error(self): + self.assertRaises(psycopg2.ProgrammingError, + self.conn.set_transaction, readonly=True, deferrable=True) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 389f2cf1d01a20dda78f09830e13fd3769ad5bb6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 2 Jun 2011 01:16:22 +0100 Subject: [PATCH 08/47] Added autocommit property on connection --- psycopg/connection.h | 2 + psycopg/connection_int.c | 3 +- psycopg/connection_type.c | 42 ++++++++++++++++++ psycopg/pqpath.c | 5 +++ tests/test_connection.py | 92 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 1 deletion(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index 979b37fd..a5aa267a 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -119,6 +119,8 @@ typedef struct { int equote; /* use E''-style quotes for escaped strings */ PyObject *weakreflist; /* list of weak references */ + int autocommit; + } connectionObject; /* C-callable functions in connection_int.c and connection_ext.c */ diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index b0ed41df..5e2ba023 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -989,7 +989,8 @@ conn_set(connectionObject *self, const char *param, const char *value) int conn_set_autocommit(connectionObject *self, int value) { - return -1; + self->autocommit = value; + return 0; } /* conn_switch_isolation_level - switch isolation level on the connection */ diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 299888a1..33a69555 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -517,6 +517,42 @@ psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwa } +#define psyco_conn_autocommit_doc \ +"set or return the autocommit status." + +static PyObject * +psyco_conn_autocommit_get(connectionObject *self) +{ + PyObject *ret; + ret = self->autocommit ? Py_True : Py_False; + Py_INCREF(ret); + return ret; +} + +static PyObject * +_psyco_conn_autocommit_set_checks(connectionObject *self) +{ + /* wrapper to use the EXC_IF macros. + * return NULL in case of error, else whatever */ + EXC_IF_CONN_CLOSED(self); + EXC_IF_CONN_ASYNC(self, autocommit); + EXC_IF_IN_TRANSACTION(self, autocommit); + return Py_None; /* borrowed */ +} + +static int +psyco_conn_autocommit_set(connectionObject *self, PyObject *pyvalue) +{ + int value; + + if (!_psyco_conn_autocommit_set_checks(self)) { return -1; } + if (-1 == (value = PyObject_IsTrue(pyvalue))) { return -1; } + if (0 != conn_set_autocommit(self, value)) { return -1; } + + return 0; +} + + /* set_isolation_level method - switch connection isolation level */ #define psyco_conn_set_isolation_level_doc \ @@ -927,6 +963,12 @@ static struct PyGetSetDef connectionObject_getsets[] = { EXCEPTION_GETTER(IntegrityError), EXCEPTION_GETTER(DataError), EXCEPTION_GETTER(NotSupportedError), +#ifdef PSYCOPG_EXTENSIONS + { "autocommit", + (getter)psyco_conn_autocommit_get, + (setter)psyco_conn_autocommit_set, + psyco_conn_autocommit_doc }, +#endif {NULL} }; #undef EXCEPTION_GETTER diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index a6144229..f945448c 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -417,6 +417,11 @@ pq_begin_locked(connectionObject *conn, PGresult **pgres, char **error, Dprintf("pq_begin_locked: pgconn = %p, isolevel = %ld, status = %d", conn->pgconn, conn->isolation_level, conn->status); + if (conn->autocommit) { + Dprintf("pq_begin_locked: autocommit"); + return 0; + } + if (conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT || conn->status != CONN_STATUS_READY) { Dprintf("pq_begin_locked: transaction in progress"); diff --git a/tests/test_connection.py b/tests/test_connection.py index 18c9277e..90014574 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -851,6 +851,98 @@ class TransactionControlTests(unittest.TestCase): self.conn.set_transaction, readonly=True, deferrable=True) +class AutocommitTests(unittest.TestCase): + def setUp(self): + self.conn = psycopg2.connect(dsn) + + def tearDown(self): + if not self.conn.closed: + self.conn.close() + + def test_default_no_autocommit(self): + self.assert_(not self.conn.autocommit) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + cur = self.conn.cursor() + cur.execute('select 1;') + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_BEGIN) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_INTRANS) + + self.conn.rollback() + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + def test_set_autocommit(self): + self.conn.autocommit = True + self.assert_(self.conn.autocommit) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + cur = self.conn.cursor() + cur.execute('select 1;') + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + self.conn.autocommit = False + self.assert_(not self.conn.autocommit) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + cur.execute('select 1;') + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_BEGIN) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_INTRANS) + + def test_set_intrans_error(self): + cur = self.conn.cursor() + cur.execute('select 1;') + self.assertRaises(psycopg2.ProgrammingError, + setattr, self.conn, 'autocommit', True) + + def test_set_transaction_autocommit(self): + self.conn.set_transaction(autocommit=True) + self.assert_(self.conn.autocommit) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + cur = self.conn.cursor() + cur.execute('select 1;') + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + self.conn.set_transaction(autocommit=False) + self.assert_(not self.conn.autocommit) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + cur.execute('select 1;') + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_BEGIN) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_INTRANS) + self.conn.rollback() + + self.conn.set_transaction('serializable', readonly=True, autocommit=True) + self.assert_(self.conn.autocommit) + cur.execute('select 1;') + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + cur.execute("SHOW default_transaction_isolation;") + self.assertEqual(cur.fetchone()[0], 'serializable') + cur.execute("SHOW default_transaction_read_only;") + self.assertEqual(cur.fetchone()[0], 'on') + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From c2d1f1f2e6832384ca01466cfbefecfa877e6850 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 3 Jun 2011 00:10:24 +0100 Subject: [PATCH 09/47] Dropped isolation level from the connection object Don't issue a SET TRANSACTION ISOLATION LEVEL at every begin: use PG's GUC default, eventually set by set_transaction. Dropped the last query at connection, yay! Method set_isolation_level() and property isolation_level refactored using the new structures, keeping the previous semantic. --- psycopg/connection.h | 18 ++-- psycopg/connection_int.c | 167 ++++++++++++++++++++++++++------------ psycopg/connection_type.c | 71 +++++++++------- psycopg/cursor_type.c | 2 +- psycopg/lobject.h | 2 +- psycopg/lobject_int.c | 2 +- psycopg/lobject_type.c | 4 +- psycopg/pqpath.c | 53 ++++-------- psycopg/python.h | 1 + 9 files changed, 188 insertions(+), 132 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index a5aa267a..d79392bc 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -61,15 +61,6 @@ extern "C" { #define psyco_datestyle "SET DATESTYLE TO 'ISO'" #define psyco_transaction_isolation "SHOW default_transaction_isolation" -/* possible values for isolation_level */ -typedef enum { - ISOLATION_LEVEL_AUTOCOMMIT = 0, - ISOLATION_LEVEL_READ_UNCOMMITTED = 1, - ISOLATION_LEVEL_READ_COMMITTED = 2, - ISOLATION_LEVEL_REPEATABLE_READ = 3, - ISOLATION_LEVEL_SERIALIZABLE = 4, -} conn_isolation_level_t; - extern HIDDEN PyTypeObject connectionType; struct connectionObject_notice { @@ -89,7 +80,6 @@ typedef struct { long int closed; /* 1 means connection has been closed; 2 that something horrible happened */ - long int isolation_level; /* isolation level for this connection */ long int mark; /* number of commits/rollbacks done so far */ int status; /* status of the connection */ XidObject *tpc_xid; /* Transaction ID in two-phase commit */ @@ -123,10 +113,16 @@ typedef struct { } connectionObject; +/* map isolation level values into a numeric const */ +typedef struct { + char *name; + int value; +} IsolationLevel; + /* C-callable functions in connection_int.c and connection_ext.c */ HIDDEN PyObject *conn_text_from_chars(connectionObject *pgconn, const char *str); HIDDEN int conn_get_standard_conforming_strings(PGconn *pgconn); -HIDDEN int conn_get_isolation_level(PGresult *pgres); +HIDDEN int conn_get_isolation_level(connectionObject *self); HIDDEN int conn_get_protocol_version(PGconn *pgconn); HIDDEN int conn_get_server_version(PGconn *pgconn); HIDDEN PGcancel *conn_get_cancel(PGconn *pgconn); diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 5e2ba023..de7083f7 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -34,6 +34,19 @@ #include +/* Mapping from isolation level name to value exposed by Python. + * Only used for backward compatibility by the isolation_level property */ + +const IsolationLevel conn_isolevels[] = { + {"", 0}, /* autocommit */ + {"read uncommitted", 1}, + {"read committed", 2}, + {"repeatable read", 3}, + {"serializable", 4}, + {"default", -1}, + { NULL } +}; + /* Return a new "string" from a char* from the database. * @@ -358,22 +371,60 @@ exit: return rv; } + int -conn_get_isolation_level(PGresult *pgres) +conn_get_isolation_level(connectionObject *self) { - static const char lvl1a[] = "read uncommitted"; - static const char lvl1b[] = "read committed"; - int rv; + PGresult *pgres; + int rv = -1; + char *lname; + const IsolationLevel *level; - char *isolation_level = PQgetvalue(pgres, 0, 0); + /* this may get called by async connections too: here's your result */ + if (self->autocommit) { + return 0; + } - if ((strcmp(lvl1b, isolation_level) == 0) /* most likely */ - || (strcmp(lvl1a, isolation_level) == 0)) - rv = ISOLATION_LEVEL_READ_COMMITTED; - else /* if it's not one of the lower ones, it's SERIALIZABLE */ - rv = ISOLATION_LEVEL_SERIALIZABLE; + Py_BEGIN_ALLOW_THREADS; + pthread_mutex_lock(&self->lock); + Py_BLOCK_THREADS; - CLEARPGRES(pgres); + if (!psyco_green()) { + Py_UNBLOCK_THREADS; + pgres = PQexec(self->pgconn, psyco_transaction_isolation); + Py_BLOCK_THREADS; + } else { + pgres = psyco_exec_green(self, psyco_transaction_isolation); + } + + if (pgres == NULL || PQresultStatus(pgres) != PGRES_TUPLES_OK) { + PyErr_SetString(OperationalError, + "can't fetch default_transaction_isolation"); + goto endlock; + } + + /* find the value for the requested isolation level */ + lname = PQgetvalue(pgres, 0, 0); + level = conn_isolevels; + while ((++level)->name) { + if (0 == strcasecmp(level->name, lname)) { + rv = level->value; + break; + } + } + + if (-1 == rv) { + char msg[256]; + snprintf(msg, sizeof(msg), "unexpected isolation level: '%s'", lname); + PyErr_SetString(OperationalError, msg); + } + +endlock: + IFCLEARPGRES(pgres); + + Py_UNBLOCK_THREADS; + pthread_mutex_unlock(&self->lock); + Py_END_ALLOW_THREADS; return rv; } @@ -477,24 +528,8 @@ conn_setup(connectionObject *self, PGconn *pgconn) CLEARPGRES(pgres); } - if (!green) { - Py_UNBLOCK_THREADS; - pgres = PQexec(pgconn, psyco_transaction_isolation); - Py_BLOCK_THREADS; - } else { - pgres = psyco_exec_green(self, psyco_transaction_isolation); - } - - if (pgres == NULL || PQresultStatus(pgres) != PGRES_TUPLES_OK) { - PyErr_SetString(OperationalError, - "can't fetch default_isolation_level"); - IFCLEARPGRES(pgres); - Py_UNBLOCK_THREADS; - pthread_mutex_unlock(&self->lock); - Py_BLOCK_THREADS; - return -1; - } - self->isolation_level = conn_get_isolation_level(pgres); + /* for reset */ + self->autocommit = 0; Py_UNBLOCK_THREADS; pthread_mutex_unlock(&self->lock); @@ -779,7 +814,7 @@ _conn_poll_setup_async(connectionObject *self) * expected to manage the transactions himself, by sending * (asynchronously) BEGIN and COMMIT statements. */ - self->isolation_level = ISOLATION_LEVEL_AUTOCOMMIT; + self->autocommit = 1; /* If the datestyle is ISO or anything else good, * we can skip the CONN_STATUS_DATESTYLE step. */ @@ -989,7 +1024,14 @@ conn_set(connectionObject *self, const char *param, const char *value) int conn_set_autocommit(connectionObject *self, int value) { + Py_BEGIN_ALLOW_THREADS; + pthread_mutex_lock(&self->lock); + self->autocommit = value; + + pthread_mutex_unlock(&self->lock); + Py_END_ALLOW_THREADS; + return 0; } @@ -998,33 +1040,54 @@ conn_set_autocommit(connectionObject *self, int value) int conn_switch_isolation_level(connectionObject *self, int level) { - PGresult *pgres = NULL; - char *error = NULL; - int res = 0; + int curr_level; - Py_BEGIN_ALLOW_THREADS; - pthread_mutex_lock(&self->lock); - - /* if the current isolation level is equal to the requested one don't switch */ - if (self->isolation_level != level) { - - /* if the current isolation level is > 0 we need to abort the current - transaction before changing; that all folks! */ - if (self->isolation_level != ISOLATION_LEVEL_AUTOCOMMIT) { - res = pq_abort_locked(self, &pgres, &error, &_save); - } - self->isolation_level = level; - - Dprintf("conn_switch_isolation_level: switched to level %d", level); + if (-1 == (curr_level = conn_get_isolation_level(self))) { + return -1; } - pthread_mutex_unlock(&self->lock); - Py_END_ALLOW_THREADS; + if (curr_level == level) { + /* no need to change level */ + return 0; + } - if (res < 0) - pq_complete_error(self, &pgres, &error); + /* Emulate the previous semantic of set_isolation_level() using the + * functions currently available. */ - return res; + /* terminate the current transaction if any */ + pq_abort(self); + + if (level == 0) { + if (0 != conn_set(self, "default_transaction_isolation", "default")) { + return -1; + } + if (0 != conn_set_autocommit(self, 1)) { + return -1; + } + } + else { + /* find the name of the requested level */ + const IsolationLevel *isolevel = conn_isolevels; + while ((++isolevel)->name) { + if (level == isolevel->value) { + break; + } + } + if (!isolevel->name) { + PyErr_SetString(OperationalError, "bad isolation level value"); + return -1; + } + + if (0 != conn_set(self, "default_transaction_isolation", isolevel->name)) { + return -1; + } + if (0 != conn_set_autocommit(self, 0)) { + return -1; + } + } + + Dprintf("conn_switch_isolation_level: switched to level %d", level); + return 0; } /* conn_set_client_encoding - switch client encoding on connection */ diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 33a69555..455abdae 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -198,7 +198,7 @@ psyco_conn_tpc_begin(connectionObject *self, PyObject *args) } /* two phase commit and autocommit make no point */ - if (self->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT) { + if (self->autocommit) { PyErr_SetString(ProgrammingError, "tpc_begin can't be called in autocommit mode"); goto exit; @@ -381,50 +381,51 @@ psyco_conn_tpc_recover(connectionObject *self, PyObject *args) /* parse a python object into one of the possible isolation level values */ +extern const IsolationLevel conn_isolevels[]; + static const char * _psyco_conn_parse_isolevel(PyObject *pyval) { - const char **value = NULL; + const char *rv = NULL; + const IsolationLevel *value = NULL; - static const char *isolevels[] = { - "", /* autocommit */ - "read uncommitted", - "read committed", - "repeatable read", - "serializable", - "default", - NULL }; + Py_INCREF(pyval); /* for ensure_bytes */ /* parse from one of the level constants */ if (PyInt_Check(pyval)) { long level = PyInt_AsLong(pyval); - if (level == -1 && PyErr_Occurred()) { return NULL; } + if (level == -1 && PyErr_Occurred()) { goto exit; } if (level < 1 || level > 4) { PyErr_SetString(PyExc_ValueError, "isolation_level must be between 1 and 4"); - return NULL; + goto exit; } - value = isolevels + level; + rv = conn_isolevels[level].name; } /* parse from the string -- this includes "default" */ else { - value = isolevels; - while (*(++value)) { - int cmp; - PyObject *pylevel; - if (!(pylevel = Text_FromUTF8(*value))) { return NULL; } - cmp = PyObject_RichCompareBool(pylevel, pyval, Py_EQ); - Py_DECREF(pylevel); - if (-1 == cmp) { return NULL; } - if (cmp) { break; } + value = conn_isolevels; + while ((++value)->name) { + if (!(pyval = psycopg_ensure_bytes(pyval))) { + goto exit; + } + if (0 == strcasecmp(value->name, Bytes_AS_STRING(pyval))) { + rv = value->name; + break; + } } - if (!*value) { - PyErr_SetString(PyExc_ValueError, - "bad value for isolation_level"); /* TODO: emit value */ + if (!rv) { + char msg[256]; + snprintf(msg, sizeof(msg), + "bad value for isolation_level: '%s'", Bytes_AS_STRING(pyval)); + PyErr_SetString(PyExc_ValueError, msg); } } - return *value; + +exit: + Py_XDECREF(pyval); + return rv; } /* convert True/False/"default" into a C string */ @@ -553,6 +554,17 @@ psyco_conn_autocommit_set(connectionObject *self, PyObject *pyvalue) } +/* isolation_level - return the current isolation level */ + +static PyObject * +psyco_conn_isolation_level_get(connectionObject *self) +{ + int rv = conn_get_isolation_level(self); + if (-1 == rv) { return NULL; } + return PyInt_FromLong((long)rv); +} + + /* set_isolation_level method - switch connection isolation level */ #define psyco_conn_set_isolation_level_doc \ @@ -920,9 +932,6 @@ static struct PyMemberDef connectionObject_members[] = { #ifdef PSYCOPG_EXTENSIONS {"closed", T_LONG, offsetof(connectionObject, closed), READONLY, "True if the connection is closed."}, - {"isolation_level", T_LONG, - offsetof(connectionObject, isolation_level), READONLY, - "The current isolation level."}, {"encoding", T_STRING, offsetof(connectionObject, encoding), READONLY, "The current client encoding."}, {"notices", T_OBJECT, offsetof(connectionObject, notice_list), READONLY}, @@ -968,6 +977,10 @@ static struct PyGetSetDef connectionObject_getsets[] = { (getter)psyco_conn_autocommit_get, (setter)psyco_conn_autocommit_set, psyco_conn_autocommit_doc }, + { "isolation_level", + (getter)psyco_conn_isolation_level_get, + (setter)NULL, + "The current isolation level." }, #endif {NULL} }; diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index d166de73..8ede845f 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -456,7 +456,7 @@ psyco_curs_execute(cursorObject *self, PyObject *args, PyObject *kwargs) NULL, NULL); return NULL; } - if (self->conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT) { + if (self->conn->autocommit) { psyco_set_error(ProgrammingError, self, "can't use a named cursor outside of transactions", NULL, NULL); return NULL; diff --git a/psycopg/lobject.h b/psycopg/lobject.h index 84fd3b52..f52d85cf 100644 --- a/psycopg/lobject.h +++ b/psycopg/lobject.h @@ -76,7 +76,7 @@ HIDDEN int lobject_close(lobjectObject *self); return NULL; } #define EXC_IF_LOBJ_LEVEL0(self) \ -if (self->conn->isolation_level == 0) { \ +if (self->conn->autocommit) { \ psyco_set_error(ProgrammingError, NULL, \ "can't use a lobject outside of transactions", NULL, NULL); \ return NULL; \ diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 3fe1f86e..e6ad1b6c 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -252,7 +252,7 @@ lobject_close_locked(lobjectObject *self, char **error) break; } - if (self->conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT || + if (self->conn->autocommit || self->conn->mark != self->mark || self->fd == -1) return 0; diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index ba45de2d..a55272ca 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -51,7 +51,7 @@ psyco_lobj_close(lobjectObject *self, PyObject *args) closing the current transaction is equivalent to close all the opened large objects */ if (!lobject_is_closed(self) - && self->conn->isolation_level != ISOLATION_LEVEL_AUTOCOMMIT + && !self->conn->autocommit && self->conn->mark == self->mark) { Dprintf("psyco_lobj_close: closing lobject at %p", self); @@ -331,7 +331,7 @@ lobject_setup(lobjectObject *self, connectionObject *conn, { Dprintf("lobject_setup: init lobject object at %p", self); - if (conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT) { + if (conn->autocommit) { psyco_set_error(ProgrammingError, NULL, "can't use a lobject outside of transactions", NULL, NULL); return -1; diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index f945448c..a188debc 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -406,30 +406,17 @@ int pq_begin_locked(connectionObject *conn, PGresult **pgres, char **error, PyThreadState **tstate) { - const char *query[] = { - NULL, - "BEGIN; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED", - "BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED", - "BEGIN; SET TRANSACTION ISOLATION LEVEL REPEATABLE READ", - "BEGIN; SET TRANSACTION ISOLATION LEVEL SERIALIZABLE"}; int result; - Dprintf("pq_begin_locked: pgconn = %p, isolevel = %ld, status = %d", - conn->pgconn, conn->isolation_level, conn->status); + Dprintf("pq_begin_locked: pgconn = %p, autocommit = %d, status = %d", + conn->pgconn, conn->autocommit, conn->status); - if (conn->autocommit) { - Dprintf("pq_begin_locked: autocommit"); - return 0; - } - - if (conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT - || conn->status != CONN_STATUS_READY) { + if (conn->autocommit || conn->status != CONN_STATUS_READY) { Dprintf("pq_begin_locked: transaction in progress"); return 0; } - result = pq_execute_command_locked(conn, query[conn->isolation_level], - pgres, error, tstate); + result = pq_execute_command_locked(conn, "BEGIN;", pgres, error, tstate); if (result == 0) conn->status = CONN_STATUS_BEGIN; @@ -449,11 +436,10 @@ pq_commit(connectionObject *conn) PGresult *pgres = NULL; char *error = NULL; - Dprintf("pq_commit: pgconn = %p, isolevel = %ld, status = %d", - conn->pgconn, conn->isolation_level, conn->status); + Dprintf("pq_commit: pgconn = %p, autocommit = %d, status = %d", + conn->pgconn, conn->autocommit, conn->status); - if (conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT - || conn->status != CONN_STATUS_BEGIN) { + if (conn->autocommit || conn->status != CONN_STATUS_BEGIN) { Dprintf("pq_commit: no transaction to commit"); return 0; } @@ -485,11 +471,10 @@ pq_abort_locked(connectionObject *conn, PGresult **pgres, char **error, { int retvalue = -1; - Dprintf("pq_abort_locked: pgconn = %p, isolevel = %ld, status = %d", - conn->pgconn, conn->isolation_level, conn->status); + Dprintf("pq_abort_locked: pgconn = %p, autocommit = %d, status = %d", + conn->pgconn, conn->autocommit, conn->status); - if (conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT - || conn->status != CONN_STATUS_BEGIN) { + if (conn->autocommit || conn->status != CONN_STATUS_BEGIN) { Dprintf("pq_abort_locked: no transaction to abort"); return 0; } @@ -514,11 +499,10 @@ pq_abort(connectionObject *conn) PGresult *pgres = NULL; char *error = NULL; - Dprintf("pq_abort: pgconn = %p, isolevel = %ld, status = %d", - conn->pgconn, conn->isolation_level, conn->status); + Dprintf("pq_abort: pgconn = %p, autocommit = %d, status = %d", + conn->pgconn, conn->autocommit, conn->status); - if (conn->isolation_level == ISOLATION_LEVEL_AUTOCOMMIT - || conn->status != CONN_STATUS_BEGIN) { + if (conn->autocommit || conn->status != CONN_STATUS_BEGIN) { Dprintf("pq_abort: no transaction to abort"); return 0; } @@ -554,13 +538,12 @@ pq_reset_locked(connectionObject *conn, PGresult **pgres, char **error, { int retvalue = -1; - Dprintf("pq_reset_locked: pgconn = %p, isolevel = %ld, status = %d", - conn->pgconn, conn->isolation_level, conn->status); + Dprintf("pq_reset_locked: pgconn = %p, autocommit = %d, status = %d", + conn->pgconn, conn->autocommit, conn->status); conn->mark += 1; - if (conn->isolation_level != ISOLATION_LEVEL_AUTOCOMMIT - && conn->status == CONN_STATUS_BEGIN) { + if (!conn->autocommit && conn->status == CONN_STATUS_BEGIN) { retvalue = pq_execute_command_locked(conn, "ABORT", pgres, error, tstate); if (retvalue != 0) return retvalue; } @@ -585,8 +568,8 @@ pq_reset(connectionObject *conn) PGresult *pgres = NULL; char *error = NULL; - Dprintf("pq_reset: pgconn = %p, isolevel = %ld, status = %d", - conn->pgconn, conn->isolation_level, conn->status); + Dprintf("pq_reset: pgconn = %p, autocommit = %d, status = %d", + conn->pgconn, conn->autocommit, conn->status); Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&conn->lock); diff --git a/psycopg/python.h b/psycopg/python.h index fed0303e..7d3f8bae 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -105,6 +105,7 @@ typedef unsigned long Py_uhash_t; #if PY_MAJOR_VERSION > 2 #define PyInt_Type PyLong_Type +#define PyInt_Check PyLong_Check #define PyInt_AsLong PyLong_AsLong #define PyInt_FromLong PyLong_FromLong #define PyInt_FromSsize_t PyLong_FromSsize_t From 4d3c6865eea89c99ed67a3a5765932cff5dda514 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 3 Jun 2011 00:40:54 +0100 Subject: [PATCH 10/47] Use only the isolation levels available on old PG versions --- psycopg/connection_int.c | 7 +++++++ psycopg/connection_type.c | 29 ++++++++++++++++++----------- tests/test_connection.py | 13 +++++++++++-- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index de7083f7..0544957f 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -1042,6 +1042,13 @@ conn_switch_isolation_level(connectionObject *self, int level) { int curr_level; + /* use only supported levels on older PG versions */ + if (self->server_version < 80000) { + if (level == 1 || level == 3) { + ++level; + } + } + if (-1 == (curr_level = conn_get_isolation_level(self))) { return -1; } diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 455abdae..3a5b7feb 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -384,10 +384,9 @@ psyco_conn_tpc_recover(connectionObject *self, PyObject *args) extern const IsolationLevel conn_isolevels[]; static const char * -_psyco_conn_parse_isolevel(PyObject *pyval) +_psyco_conn_parse_isolevel(connectionObject *self, PyObject *pyval) { - const char *rv = NULL; - const IsolationLevel *value = NULL; + const IsolationLevel *isolevel = NULL; Py_INCREF(pyval); /* for ensure_bytes */ @@ -400,22 +399,22 @@ _psyco_conn_parse_isolevel(PyObject *pyval) "isolation_level must be between 1 and 4"); goto exit; } - rv = conn_isolevels[level].name; + + isolevel = conn_isolevels + level; } /* parse from the string -- this includes "default" */ else { - value = conn_isolevels; - while ((++value)->name) { + isolevel = conn_isolevels; + while ((++isolevel)->name) { if (!(pyval = psycopg_ensure_bytes(pyval))) { goto exit; } - if (0 == strcasecmp(value->name, Bytes_AS_STRING(pyval))) { - rv = value->name; + if (0 == strcasecmp(isolevel->name, Bytes_AS_STRING(pyval))) { break; } } - if (!rv) { + if (!isolevel->name) { char msg[256]; snprintf(msg, sizeof(msg), "bad value for isolation_level: '%s'", Bytes_AS_STRING(pyval)); @@ -423,9 +422,17 @@ _psyco_conn_parse_isolevel(PyObject *pyval) } } + /* use only supported levels on older PG versions */ + if (isolevel && self->server_version < 80000) { + if (isolevel->value == 1 || isolevel->value == 3) { + ++isolevel; + } + } + exit: Py_XDECREF(pyval); - return rv; + + return isolevel ? isolevel->name : NULL; } /* convert True/False/"default" into a C string */ @@ -477,7 +484,7 @@ psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwa if (Py_None != isolation_level) { const char *value = NULL; - if (!(value = _psyco_conn_parse_isolevel(isolation_level))) { + if (!(value = _psyco_conn_parse_isolevel(self, isolation_level))) { return NULL; } if (0 != conn_set(self, "default_transaction_isolation", value)) { diff --git a/tests/test_connection.py b/tests/test_connection.py index 90014574..e62e51e3 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -204,14 +204,23 @@ class IsolationLevelsTestCase(unittest.TestCase): conn = self.connect() curs = conn.cursor() - for name, level in ( + levels = ( (None, psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT), ('read uncommitted', psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED), ('read committed', psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED), ('repeatable read', psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ), ('serializable', psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE), - ): + ) + for name, level in levels: conn.set_isolation_level(level) + + # the only values available on prehistoric PG versions + if conn.server_version < 80000: + if level in ( + psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED, + psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ): + name, level = levels[levels.index((name, level)) + 1] + self.assertEqual(conn.isolation_level, level) curs.execute('show transaction_isolation;') From 5748ae14bf060aa69c97449d91253b2ba69dd23e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 3 Jun 2011 00:54:33 +0100 Subject: [PATCH 11/47] Test tweaked to deal with missing usecs in BC timestamps Probably depending on compile time options. On my test db usecs are available from PG 8.4. --- tests/test_dates.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_dates.py b/tests/test_dates.py index db2050ac..27abbc1e 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -417,7 +417,11 @@ class mxDateTimeTests(unittest.TestCase, CommonDatetimeTestsMixin): from mx.DateTime import DateTime value = self.execute('select (%s)::timestamp::text', [DateTime(-41, 1, 1, 13, 30, 29.123456)]) - self.assertEqual(value, '0042-01-01 13:30:29.123456 BC') + # microsecs for BC timestamps look not available in PG < 8.4 + # but more likely it's determined at compile time. + self.assert_(value in ( + '0042-01-01 13:30:29.123456 BC', + '0042-01-01 13:30:29 BC'), value) def test_adapt_timedelta(self): from mx.DateTime import DateTimeDeltaFrom From 530ba7888158a7ee5b27a3c7d9c8f481e6200528 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 3 Jun 2011 01:46:56 +0100 Subject: [PATCH 12/47] Documentation for set_transaction() and autocommit improved --- NEWS | 8 ++++++-- doc/src/connection.rst | 24 ++++++++++++++++-------- doc/src/usage.rst | 4 ++++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/NEWS b/NEWS index 1c439323..226ccda9 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,12 @@ What's new in psycopg 2.4.2 --------------------------- - - Allow using the isolation level "repeatable read" which is distinct - from "serializable" in PostgreSQL 9.1. + - Added 'set_transaction()' method and 'autocommit' property to the + connection. Added support for read-only sessions and, for PostgreSQL + 9.1, for the "repeatable read" isolation level and the "deferrable" + transaction property. + - Psycopg doesn't execute queries at connection time to find the + default isolation level. - Don't build mx.DateTime support if the module can't be imported (ticket #53). diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 9564095e..970b0371 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -327,7 +327,7 @@ The ``connection`` class pair: Transaction; Autocommit pair: Transaction; Isolation level - .. method:: set_transaction(isolation_level=None, readonly=None, deferrable=None, autocommit=None) + .. method:: set_transaction([isolation_level,] [readonly,] [deferrable,] [autocommit]) Set one or more parameters for the next transactions or statements in the current session. See |SET TRANSACTION|_ for further details. @@ -336,9 +336,11 @@ The ``connection`` class .. _SET TRANSACTION: http://www.postgresql.org/docs/9.1/static/sql-set-transaction.html :param isolation_level: set the `isolation level`_ for the next - transactions/statements. The value should be one of the + transactions/statements. The value can be one of the :ref:`constants ` defined in the - `~psycopg2.extensions` module. + `~psycopg2.extensions` module or one of the literal values + ``read uncommitted``, ``read committed``, ``repeatable read``, + ``serializable``. :param readonly: if `!True`, set the connection to read only; read/write if `!False`. :param deferrable: if `!True`, set the connection to deferrable; @@ -347,12 +349,15 @@ The ``connection`` class PostgreSQL session setting but an alias for setting the `autocommit` attribute. + The parameters *isolation_level*, *readonly* and *deferrable* also + accept the string ``default`` as a value: the effect is to reset the + parameter to the server default. + .. _isolation level: http://www.postgresql.org/docs/9.1/static/transaction-iso.html The function must be invoked with no transaction in progress. At every - function invocation, only the parameters whose value is not `!None` are - changed. + function invocation, only the specified parameters are changed. The default for the values are defined by the server configuration: see values for |default_transaction_isolation|__, @@ -382,6 +387,10 @@ The ``connection`` class execution: the methods `commit()` or `rollback()` must be manually invoked to terminate the transaction. + The autocommit mode is useful to execute commands requiring to be run + outside a transaction, such as :sql:`CREATE DATABASE` or + :sql:`VACUUM`. + The default is `!False` (manual commit) as per DBAPI specification. .. warning:: @@ -402,9 +411,8 @@ The ``connection`` class .. note:: - From version 2.4.2, replaced by `set_transaction()` and - `autocommit`, offering finer control on the transaction - characteristics. + From version 2.4.2, `set_transaction()` and `autocommit`, offer + finer control on the transaction characteristics. Read or set the `transaction isolation level`_ for the current session. The level defines the different phenomena that can happen in the diff --git a/doc/src/usage.rst b/doc/src/usage.rst index 4d039dee..de82c624 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -519,6 +519,10 @@ outside any transaction: in order to be able to run these commands from Psycopg, the session must be in autocommit mode. Read the documentation for `connection.set_isolation_level()` to know how to change the commit mode. +.. note:: + + From version 2.4.2 you can use the `~connection.autocommit` property to + switch a connection in autocommit mode. .. index:: From 3aad3d3143a8e5963b06b8905a040521d3c3bcaa Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 3 Jun 2011 09:31:06 +0100 Subject: [PATCH 13/47] Fixed test to run on Python <= 2.5 tuple.index() is not available on these versions. --- tests/test_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index e62e51e3..dc3cfb00 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -204,13 +204,13 @@ class IsolationLevelsTestCase(unittest.TestCase): conn = self.connect() curs = conn.cursor() - levels = ( + levels = [ (None, psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT), ('read uncommitted', psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED), ('read committed', psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED), ('repeatable read', psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ), ('serializable', psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE), - ) + ] for name, level in levels: conn.set_isolation_level(level) From d9c0b8166f9ba17eacb09d5861409ab862447331 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 4 Jun 2011 01:31:36 +0100 Subject: [PATCH 14/47] Process notifies when data is received, not when the result is parsed Notifies process access the connection, is not limited to the result, so There is the possibility of loss of protocol sync in multithread programs. Closes ticket #55. --- NEWS | 2 ++ psycopg/connection_int.c | 2 -- psycopg/pqpath.c | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 5e3aa14a..a14cbf33 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,8 @@ What's new in psycopg 2.4.2 transaction property. - Psycopg doesn't execute queries at connection time to find the default isolation level. + - Fixed bug with multithread code potentially causing loss of sync + with the server communication or lock of the client (ticket #55). - Don't build mx.DateTime support if the module can't be imported (ticket #53). - Fixed escape for negative numbers prefixed by minus operator diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 0544957f..18a0a14f 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -174,8 +174,6 @@ conn_notifies_process(connectionObject *self) PyObject *notify = NULL; PyObject *pid = NULL, *channel = NULL, *payload = NULL; - /* TODO: we are called without the lock! */ - while ((pgn = PQnotifies(self->pgconn)) != NULL) { Dprintf("conn_notifies_process: got NOTIFY from pid %d, msg = %s", diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index a188debc..e1a4f5fb 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -665,11 +665,14 @@ pq_is_busy(connectionObject *conn) res = PQisBusy(conn->pgconn); + Py_BLOCK_THREADS; + conn_notifies_process(conn); + Py_UNBLOCK_THREADS; + pthread_mutex_unlock(&(conn->lock)); Py_END_ALLOW_THREADS; conn_notice_process(conn); - conn_notifies_process(conn); return res; } @@ -781,6 +784,14 @@ pq_execute(cursorObject *curs, const char *query, int async) } return -1; } + + /* Process notifies here instead of when fetching the tuple as we are + * into the same critical section that received the data. Without this + * care, reading notifies may disrupt other thread communications. + * (as in ticket #55). */ + Py_BLOCK_THREADS; + conn_notifies_process(curs->conn); + Py_UNBLOCK_THREADS; } else if (async == 1) { @@ -1370,7 +1381,6 @@ pq_fetch(cursorObject *curs) } conn_notice_process(curs->conn); - conn_notifies_process(curs->conn); /* error checking, close the connection if necessary (some critical errors are not really critical, like a COPY FROM error: if that's the case we From 05659c0d16767f0cba55d0e6acfbe3748aacc719 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 4 Jun 2011 01:49:03 +0100 Subject: [PATCH 15/47] Cleanup of notice processing The function is always called in the context of functions grabbing the connection lock, so just use the same critical section instead of releasing and re-acquiring it. It is not a problem as serious as the notifies process (ticket #55) as the notices are a psycopg structure, not libpq. However the change allows again processing notices/notifies in the same place, which makes sense conceptually, plus we save some lock dance. --- psycopg/connection_int.c | 19 +++++-------------- psycopg/pqpath.c | 30 +++++++++++++++++------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 18a0a14f..24d424df 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -95,6 +95,10 @@ conn_notice_callback(void *args, const char *message) self->notice_pending = notice; } +/* Expose the notices received as Python objects. + * + * The function should be called with the connection lock and the GIL. + */ void conn_notice_process(connectionObject *self) { @@ -105,10 +109,6 @@ conn_notice_process(connectionObject *self) return; } - Py_BEGIN_ALLOW_THREADS; - pthread_mutex_lock(&self->lock); - Py_BLOCK_THREADS; - notice = self->notice_pending; nnotices = PyList_GET_SIZE(self->notice_list); @@ -132,10 +132,6 @@ conn_notice_process(connectionObject *self) 0, nnotices - CONN_NOTICES_LIMIT); } - Py_UNBLOCK_THREADS; - pthread_mutex_unlock(&self->lock); - Py_END_ALLOW_THREADS; - conn_notice_clean(self); } @@ -143,8 +139,6 @@ void conn_notice_clean(connectionObject *self) { struct connectionObject_notice *tmp, *notice; - Py_BEGIN_ALLOW_THREADS; - pthread_mutex_lock(&self->lock); notice = self->notice_pending; @@ -154,11 +148,8 @@ conn_notice_clean(connectionObject *self) free((void*)tmp->message); free(tmp); } - + self->notice_pending = NULL; - - pthread_mutex_unlock(&self->lock); - Py_END_ALLOW_THREADS; } diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index e1a4f5fb..f50b00b4 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -450,11 +450,13 @@ pq_commit(connectionObject *conn) retvalue = pq_execute_command_locked(conn, "COMMIT", &pgres, &error, &_save); + Py_BLOCK_THREADS; + conn_notice_process(conn); + Py_UNBLOCK_THREADS; + pthread_mutex_unlock(&conn->lock); Py_END_ALLOW_THREADS; - conn_notice_process(conn); - if (retvalue < 0) pq_complete_error(conn, &pgres, &error); @@ -512,11 +514,13 @@ pq_abort(connectionObject *conn) retvalue = pq_abort_locked(conn, &pgres, &error, &_save); + Py_BLOCK_THREADS; + conn_notice_process(conn); + Py_UNBLOCK_THREADS; + pthread_mutex_unlock(&conn->lock); Py_END_ALLOW_THREADS; - conn_notice_process(conn); - if (retvalue < 0) pq_complete_error(conn, &pgres, &error); @@ -576,11 +580,13 @@ pq_reset(connectionObject *conn) retvalue = pq_reset_locked(conn, &pgres, &error, &_save); + Py_BLOCK_THREADS; + conn_notice_process(conn); + Py_UNBLOCK_THREADS; + pthread_mutex_unlock(&conn->lock); Py_END_ALLOW_THREADS; - conn_notice_process(conn); - if (retvalue < 0) { pq_complete_error(conn, &pgres, &error); } @@ -667,13 +673,12 @@ pq_is_busy(connectionObject *conn) Py_BLOCK_THREADS; conn_notifies_process(conn); + conn_notice_process(conn); Py_UNBLOCK_THREADS; pthread_mutex_unlock(&(conn->lock)); Py_END_ALLOW_THREADS; - conn_notice_process(conn); - return res; } @@ -693,9 +698,9 @@ pq_is_busy_locked(connectionObject *conn) return -1; } - /* We can't call conn_notice_process/conn_notifies_process because - they try to get the lock. We don't need anyway them because at the end of - the loop we are in (async reading) pq_fetch will be called. */ + /* notices and notifies will be processed at the end of the loop we are in + * (async reading) by pq_fetch. */ + return PQisBusy(conn->pgconn); } @@ -791,6 +796,7 @@ pq_execute(cursorObject *curs, const char *query, int async) * (as in ticket #55). */ Py_BLOCK_THREADS; conn_notifies_process(curs->conn); + conn_notice_process(curs->conn); Py_UNBLOCK_THREADS; } @@ -1380,8 +1386,6 @@ pq_fetch(cursorObject *curs) break; } - conn_notice_process(curs->conn); - /* error checking, close the connection if necessary (some critical errors are not really critical, like a COPY FROM error: if that's the case we raise the exception but we avoid to close the connection) */ From c8ec747903a4d0063d88fdd38cb18b50303daaa7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 4 Jun 2011 14:16:24 +0100 Subject: [PATCH 16/47] Don't fail import if mx.DateTime module is not found at import time A better fix for ticket #53. --- NEWS | 4 ++-- psycopg/adapter_datetime.h | 4 ++-- psycopg/adapter_mxdatetime.c | 8 +++++--- psycopg/psycopgmodule.c | 38 ++++++++++++++++++++++++++--------- psycopg/typecast.c | 15 +++++++------- psycopg/typecast_mxdatetime.c | 8 ++++++-- setup.py | 19 +++++++----------- 7 files changed, 59 insertions(+), 37 deletions(-) diff --git a/NEWS b/NEWS index a14cbf33..2d0d076c 100644 --- a/NEWS +++ b/NEWS @@ -9,8 +9,8 @@ What's new in psycopg 2.4.2 default isolation level. - Fixed bug with multithread code potentially causing loss of sync with the server communication or lock of the client (ticket #55). - - Don't build mx.DateTime support if the module can't be imported - (ticket #53). + - Don't fail import if mx.DateTime module can't be found, even if its + support was built (ticket #53). - Fixed escape for negative numbers prefixed by minus operator (ticket #57). diff --git a/psycopg/adapter_datetime.h b/psycopg/adapter_datetime.h index dd59e9b5..09abd88f 100644 --- a/psycopg/adapter_datetime.h +++ b/psycopg/adapter_datetime.h @@ -45,11 +45,11 @@ typedef struct { } pydatetimeObject; +HIDDEN int psyco_adapter_datetime_init(void); + /* functions exported to psycopgmodule.c */ #ifdef PSYCOPG_DEFAULT_PYDATETIME -HIDDEN int psyco_adapter_datetime_init(void); - HIDDEN PyObject *psyco_Date(PyObject *module, PyObject *args); #define psyco_Date_doc \ "Date(year, month, day) -> new date\n\n" \ diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index 793dfba2..abe73f86 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -26,7 +26,6 @@ #define PSYCOPG_MODULE #include "psycopg/psycopg.h" -/* TODO: check if still compiles ok: I have no mx on this box */ #include "psycopg/adapter_mxdatetime.h" #include "psycopg/microprotocols_proto.h" @@ -34,13 +33,16 @@ #include +/* Return 0 on success, -1 on failure, but don't set an exception */ + int psyco_adapter_mxdatetime_init(void) { Dprintf("psyco_adapter_mxdatetime_init: mx.DateTime init"); - if(mxDateTime_ImportModuleAndAPI()) { - PyErr_SetString(PyExc_ImportError, "mx.DateTime initialization failed"); + if (mxDateTime_ImportModuleAndAPI()) { + Dprintf("psyco_adapter_mxdatetime_init: mx.DateTime initialization failed"); + PyErr_Clear(); return -1; } return 0; diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index c5daefd5..f37a98e7 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -360,10 +360,16 @@ psyco_adapters_init(PyObject *mod) #ifdef HAVE_MXDATETIME /* as above, we use the callable objects from the psycopg module */ - call = PyMapping_GetItemString(mod, "TimestampFromMx"); - microprotocols_add(mxDateTime.DateTime_Type, NULL, call); - call = PyMapping_GetItemString(mod, "TimeFromMx"); - microprotocols_add(mxDateTime.DateTimeDelta_Type, NULL, call); + if (NULL != (call = PyMapping_GetItemString(mod, "TimestampFromMx"))) { + microprotocols_add(mxDateTime.DateTime_Type, NULL, call); + + /* if we found the above, we have this too. */ + call = PyMapping_GetItemString(mod, "TimeFromMx"); + microprotocols_add(mxDateTime.DateTimeDelta_Type, NULL, call); + } + else { + PyErr_Clear(); + } #endif } @@ -792,6 +798,7 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS, psyco_IntervalFromPy_doc}, #ifdef HAVE_MXDATETIME + /* to be deleted if not found at import time */ {"DateFromMx", (PyCFunction)psyco_DateFromMx, METH_VARARGS, psyco_DateFromMx_doc}, {"TimeFromMx", (PyCFunction)psyco_TimeFromMx, @@ -885,12 +892,16 @@ INIT_MODULE(_psycopg)(void) #ifdef HAVE_MXDATETIME Py_TYPE(&mxdatetimeType) = &PyType_Type; if (PyType_Ready(&mxdatetimeType) == -1) goto exit; - if (mxDateTime_ImportModuleAndAPI() != 0) { - Dprintf("initpsycopg: why marc hide mx.DateTime again?!"); - PyErr_SetString(PyExc_ImportError, "can't import mx.DateTime module"); + if (0 != mxDateTime_ImportModuleAndAPI()) { + PyErr_Clear(); + + /* only fail if the mx typacaster should have been the default */ +#ifdef PSYCOPG_DEFAULT_MXDATETIME + PyErr_SetString(PyExc_ImportError, + "can't import mx.DateTime module (requested as default adapter)"); goto exit; +#endif } - if (psyco_adapter_mxdatetime_init()) { goto exit; } #endif /* import python builtin datetime module, if available */ @@ -967,6 +978,16 @@ INIT_MODULE(_psycopg)(void) /* encodings dictionary in module dictionary */ PyModule_AddObject(module, "encodings", psycoEncodings); +#ifdef HAVE_MXDATETIME + /* If we can't find mx.DateTime objects at runtime, + * remove them from the module (and, as consequence, from the adapters). */ + if (0 != psyco_adapter_mxdatetime_init()) { + PyDict_DelItemString(dict, "DateFromMx"); + PyDict_DelItemString(dict, "TimeFromMx"); + PyDict_DelItemString(dict, "TimestampFromMx"); + PyDict_DelItemString(dict, "IntervalFromMx"); + } +#endif /* initialize default set of typecasters */ typecast_init(dict); @@ -999,7 +1020,6 @@ INIT_MODULE(_psycopg)(void) lobjectType.tp_alloc = PyType_GenericAlloc; #endif - #ifdef HAVE_MXDATETIME mxdatetimeType.tp_alloc = PyType_GenericAlloc; #endif diff --git a/psycopg/typecast.c b/psycopg/typecast.c index ba3871e2..56a203dc 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -292,13 +292,14 @@ typecast_init(PyObject *dict) /* register the date/time typecasters with their original names */ #ifdef HAVE_MXDATETIME - if (psyco_typecast_mxdatetime_init()) { return -1; } - for (i = 0; typecast_mxdatetime[i].name != NULL; i++) { - typecastObject *t; - Dprintf("typecast_init: initializing %s", typecast_mxdatetime[i].name); - t = (typecastObject *)typecast_from_c(&(typecast_mxdatetime[i]), dict); - if (t == NULL) return -1; - PyDict_SetItem(dict, t->name, (PyObject *)t); + if (0 == psyco_typecast_mxdatetime_init()) { + for (i = 0; typecast_mxdatetime[i].name != NULL; i++) { + typecastObject *t; + Dprintf("typecast_init: initializing %s", typecast_mxdatetime[i].name); + t = (typecastObject *)typecast_from_c(&(typecast_mxdatetime[i]), dict); + if (t == NULL) return -1; + PyDict_SetItem(dict, t->name, (PyObject *)t); + } } #endif diff --git a/psycopg/typecast_mxdatetime.c b/psycopg/typecast_mxdatetime.c index a3a95fa6..e61224df 100644 --- a/psycopg/typecast_mxdatetime.c +++ b/psycopg/typecast_mxdatetime.c @@ -25,13 +25,17 @@ #include "mxDateTime.h" + +/* Return 0 on success, -1 on failure, but don't set an exception */ + static int psyco_typecast_mxdatetime_init(void) { Dprintf("psyco_typecast_mxdatetime_init: mx.DateTime init"); - if(mxDateTime_ImportModuleAndAPI()) { - PyErr_SetString(PyExc_ImportError, "mx.DateTime initialization failed"); + if (mxDateTime_ImportModuleAndAPI()) { + Dprintf("psyco_typecast_mxdatetime_init: mx.DateTime initialization failed"); + PyErr_Clear(); return -1; } return 0; diff --git a/setup.py b/setup.py index 45b1a90c..6f06ad10 100644 --- a/setup.py +++ b/setup.py @@ -450,18 +450,13 @@ if parser.has_option('build_ext', 'mx_include_dir'): else: mxincludedir = os.path.join(get_python_inc(plat_specific=1), "mx") if os.path.exists(mxincludedir): - # Check if mx.datetime is importable at all: see ticket #53 - try: - import mx.DateTime - except ImportError: - pass - else: - include_dirs.append(mxincludedir) - define_macros.append(('HAVE_MXDATETIME','1')) - sources.append('adapter_mxdatetime.c') - depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) - have_mxdatetime = True - version_flags.append('mx') + # Build the support for mx: we will check at runtime if it can be imported + include_dirs.append(mxincludedir) + define_macros.append(('HAVE_MXDATETIME','1')) + sources.append('adapter_mxdatetime.c') + depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) + have_mxdatetime = True + version_flags.append('mx') # now decide which package will be the default for date/time typecasts if have_pydatetime and (use_pydatetime or not have_mxdatetime): From 62a8ef308a664ee8c6251cfba27da7fe6fef2d8e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 4 Jun 2011 14:21:18 +0100 Subject: [PATCH 17/47] Fixed version check --- lib/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/__init__.py b/lib/__init__.py index 48a9847a..38312100 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -40,20 +40,16 @@ Homepage: http://initd.org/projects/psycopg2 # Import modules needed by _psycopg to allow tools like py2exe to do # their work without bothering about the module dependencies. -# -# TODO: we should probably use the Warnings framework to signal a missing -# module instead of raising an exception (in case we're running a thin -# embedded Python or something even more devious.) import sys, warnings -if sys.version_info[0] >= 2 and sys.version_info[1] >= 3: +if sys.version_info >= (2, 3): try: import datetime as _psycopg_needs_datetime except: warnings.warn( "can't import datetime module probably needed by _psycopg", RuntimeWarning) -if sys.version_info[0] >= 2 and sys.version_info[1] >= 4: +if sys.version_info >= (2, 4): try: import decimal as _psycopg_needs_decimal except: From a0d16fcfb207ae088b4579b5ebe9f7b6e020ef5b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 4 Jun 2011 22:19:56 +0100 Subject: [PATCH 18/47] Avoid a ton of warnings when building on mingw mingw doesn't support visibility hidden even if gcc can. --- psycopg/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psycopg/config.h b/psycopg/config.h index 551cfe48..01e1e562 100644 --- a/psycopg/config.h +++ b/psycopg/config.h @@ -27,7 +27,7 @@ #define PSYCOPG_CONFIG_H 1 /* GCC 4.0 and later have support for specifying symbol visibility */ -#if __GNUC__ >= 4 +#if __GNUC__ >= 4 && !defined(__MINGW32__) # define HIDDEN __attribute__((visibility("hidden"))) #else # define HIDDEN From dcc9e84a68a0aa446b924c308a81605d57965859 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 4 Jun 2011 22:26:21 +0100 Subject: [PATCH 19/47] Don't encode the pg_config path on Python 3 on Windows It can deal with unicode ok, and fails if it gets bytes. --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6f06ad10..f155119d 100644 --- a/setup.py +++ b/setup.py @@ -387,10 +387,10 @@ or with the pg_config option in 'setup.cfg'. ) # Support unicode paths, if this version of Python provides the # necessary infrastructure: - if hasattr(sys, 'getfilesystemencoding'): + if sys.version_info[0] < 3 \ + and hasattr(sys, 'getfilesystemencoding'): pg_config_path = pg_config_path.encode( - sys.getfilesystemencoding() - ) + sys.getfilesystemencoding()) return pg_config_path From cf6a4ec062c1dcde441c8253af70c59f4c5a5dd6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jun 2011 15:36:02 +0100 Subject: [PATCH 20/47] Added pqpath functions to get/set the value for GUC parameters The aim of these function is to allow the connection to make a better use of the pqpath functions instead of using PQexec for these small things. Also, the functions are to be called with the connection lock: this makes composing higher level functions using them easier. --- psycopg/pqpath.c | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ psycopg/pqpath.h | 6 ++++ 2 files changed, 98 insertions(+) diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index f50b00b4..fb0736a5 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -597,6 +597,98 @@ pq_reset(connectionObject *conn) } +/* Get a session parameter. + * + * The function should be called on a locked connection without + * holding the GIL. + * + * The result is a new string allocated with malloc. + */ + +char * +pq_get_guc_locked( + connectionObject *conn, const char *param, + PGresult **pgres, char **error, PyThreadState **tstate) +{ + char query[256]; + int size; + char *rv = NULL; + + Dprintf("pq_get_guc_locked: reading %s", param); + + size = PyOS_snprintf(query, sizeof(query), "SHOW %s;", param); + if (size >= sizeof(query)) { + *error = strdup("SHOW: query too large"); + goto cleanup; + } + + Dprintf("pq_get_guc_locked: pgconn = %p, query = %s", conn->pgconn, query); + + *error = NULL; + if (!psyco_green()) { + *pgres = PQexec(conn->pgconn, query); + } else { + PyEval_RestoreThread(*tstate); + *pgres = psyco_exec_green(conn, query); + *tstate = PyEval_SaveThread(); + } + + if (*pgres == NULL) { + Dprintf("pq_get_guc_locked: PQexec returned NULL"); + if (!PyErr_Occurred()) { + const char *msg; + msg = PQerrorMessage(conn->pgconn); + if (msg && *msg) { *error = strdup(msg); } + } + goto cleanup; + } + if (PQresultStatus(*pgres) != PGRES_TUPLES_OK) { + Dprintf("pq_get_guc_locked: result was not TUPLES_OK (%d)", + PQresultStatus(*pgres)); + goto cleanup; + } + + rv = strdup(PQgetvalue(*pgres, 0, 0)); + CLEARPGRES(*pgres); + +cleanup: + return rv; +} + +/* Set a session parameter. + * + * The function should be called on a locked connection without + * holding the GIL + */ + +int +pq_set_guc_locked( + connectionObject *conn, const char *param, const char *value, + PGresult **pgres, char **error, PyThreadState **tstate) +{ + char query[256]; + int size; + int rv = -1; + + Dprintf("pq_set_guc_locked: setting %s to %s", param, value); + + if (0 == strcmp(value, "default")) { + size = PyOS_snprintf(query, sizeof(query), + "SET %s TO DEFAULT;", param); + } + else { + size = PyOS_snprintf(query, sizeof(query), + "SET %s TO '%s';", param, value); + } + if (size >= sizeof(query)) { + *error = strdup("SET: query too large"); + } + + rv = pq_execute_command_locked(conn, query, pgres, error, tstate); + + return rv; +} + /* Call one of the PostgreSQL tpc-related commands. * * This function should only be called on a locked connection without diff --git a/psycopg/pqpath.h b/psycopg/pqpath.h index 080047c0..bf012ade 100644 --- a/psycopg/pqpath.h +++ b/psycopg/pqpath.h @@ -47,6 +47,12 @@ HIDDEN int pq_abort(connectionObject *conn); HIDDEN int pq_reset_locked(connectionObject *conn, PGresult **pgres, char **error, PyThreadState **tstate); HIDDEN int pq_reset(connectionObject *conn); +HIDDEN char *pq_get_guc_locked(connectionObject *conn, const char *param, + PGresult **pgres, + char **error, PyThreadState **tstate); +HIDDEN int pq_set_guc_locked(connectionObject *conn, const char *param, + const char *value, PGresult **pgres, + char **error, PyThreadState **tstate); HIDDEN int pq_tpc_command_locked(connectionObject *conn, const char *cmd, const char *tid, PGresult **pgres, char **error, From 8f876d4b5d26a4e618d0cdcb2189b5ee3abf97c6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jun 2011 16:22:54 +0100 Subject: [PATCH 21/47] Avoid a deadlock using concurrent green threads on the same connection Use the async_cursor property to store an indication that something is running (even if it is not necessarily a cursor running the query). --- NEWS | 3 +++ doc/src/advanced.rst | 8 +++----- doc/src/usage.rst | 4 ++-- psycopg/connection.h | 5 ++++- psycopg/connection_int.c | 2 +- psycopg/cursor_type.c | 8 ++++---- psycopg/green.c | 15 +++++++++++++++ 7 files changed, 32 insertions(+), 13 deletions(-) diff --git a/NEWS b/NEWS index 2d0d076c..c7b2aaac 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,9 @@ What's new in psycopg 2.4.2 support was built (ticket #53). - Fixed escape for negative numbers prefixed by minus operator (ticket #57). + - Trying to execute concurrent operations on the same connection + through concurrent green thread results in an error instead of a + deadlock. What's new in psycopg 2.4.1 diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index ac16ca9b..c7625b19 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -432,11 +432,9 @@ SQLAlchemy_) to be used in coroutine-based programs. .. warning:: Psycopg connections are not *green thread safe* and can't be used - concurrently by different green threads. Each connection has a lock - used to serialize requests from different cursors to the backend process. - The lock is held for the duration of the command: if the control switched - to a different thread and the latter tried to access the same connection, - the result would be a deadlock. + concurrently by different green threads. Trying to execute more than one + command at time using one cursor per thread will result in an error (or a + deadlock on versions before 2.4.2). Therefore, programmers are advised to either avoid sharing connections between coroutines or to use a library-friendly lock to synchronize shared diff --git a/doc/src/usage.rst b/doc/src/usage.rst index de82c624..efbd1587 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -598,8 +598,8 @@ forking web deploy method such as FastCGI ensure to create the connections .. __: http://www.postgresql.org/docs/9.0/static/libpq-connect.html#LIBPQ-CONNECT -Connections shouldn't be shared either by different green threads: doing so -may result in a deadlock. See :ref:`green-support` for further details. +Connections shouldn't be shared either by different green threads: see +:ref:`green-support` for further details. diff --git a/psycopg/connection.h b/psycopg/connection.h index d79392bc..24b3be37 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -91,7 +91,10 @@ typedef struct { PGconn *pgconn; /* the postgresql connection */ PGcancel *cancel; /* the cancellation structure */ - PyObject *async_cursor; /* weakref to a cursor executing an asynchronous query */ + /* Weakref to the object executing an asynchronous query. The object + * is a cursor for async connections, but it may be something else + * for a green connection. If NULL, the connection is idle. */ + PyObject *async_cursor; int async_status; /* asynchronous execution status */ /* notice processing */ diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 24d424df..6a1d9c5f 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -881,7 +881,7 @@ conn_poll(connectionObject *self) case CONN_STATUS_PREPARED: res = _conn_poll_query(self); - if (res == PSYCO_POLL_OK && self->async_cursor) { + if (res == PSYCO_POLL_OK && self->async && self->async_cursor) { /* An async query has just finished: parse the tuple in the * target cursor. */ cursorObject *curs; diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 8ede845f..c18fb71d 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -739,7 +739,6 @@ psyco_curs_fetchone(cursorObject *self, PyObject *args) PyObject *res; EXC_IF_CURS_CLOSED(self); - EXC_IF_ASYNC_IN_PROGRESS(self, fetchone); if (_psyco_curs_prefetch(self) < 0) return NULL; EXC_IF_NO_TUPLES(self); @@ -747,6 +746,7 @@ psyco_curs_fetchone(cursorObject *self, PyObject *args) char buffer[128]; EXC_IF_NO_MARK(self); + EXC_IF_ASYNC_IN_PROGRESS(self, fetchone); EXC_IF_TPC_PREPARED(self->conn, fetchone); PyOS_snprintf(buffer, 127, "FETCH FORWARD 1 FROM \"%s\"", self->name); if (pq_execute(self, buffer, 0) == -1) return NULL; @@ -853,7 +853,6 @@ psyco_curs_fetchmany(cursorObject *self, PyObject *args, PyObject *kwords) } EXC_IF_CURS_CLOSED(self); - EXC_IF_ASYNC_IN_PROGRESS(self, fetchmany); if (_psyco_curs_prefetch(self) < 0) return NULL; EXC_IF_NO_TUPLES(self); @@ -861,6 +860,7 @@ psyco_curs_fetchmany(cursorObject *self, PyObject *args, PyObject *kwords) char buffer[128]; EXC_IF_NO_MARK(self); + EXC_IF_ASYNC_IN_PROGRESS(self, fetchmany); EXC_IF_TPC_PREPARED(self->conn, fetchone); PyOS_snprintf(buffer, 127, "FETCH FORWARD %d FROM \"%s\"", (int)size, self->name); @@ -924,7 +924,6 @@ psyco_curs_fetchall(cursorObject *self, PyObject *args) PyObject *list, *res; EXC_IF_CURS_CLOSED(self); - EXC_IF_ASYNC_IN_PROGRESS(self, fetchall); if (_psyco_curs_prefetch(self) < 0) return NULL; EXC_IF_NO_TUPLES(self); @@ -932,6 +931,7 @@ psyco_curs_fetchall(cursorObject *self, PyObject *args) char buffer[128]; EXC_IF_NO_MARK(self); + EXC_IF_ASYNC_IN_PROGRESS(self, fetchall); EXC_IF_TPC_PREPARED(self->conn, fetchall); PyOS_snprintf(buffer, 127, "FETCH FORWARD ALL FROM \"%s\"", self->name); if (pq_execute(self, buffer, 0) == -1) return NULL; @@ -1112,7 +1112,6 @@ psyco_curs_scroll(cursorObject *self, PyObject *args, PyObject *kwargs) return NULL; EXC_IF_CURS_CLOSED(self); - EXC_IF_ASYNC_IN_PROGRESS(self, scroll) /* if the cursor is not named we have the full result set and we can do our own calculations to scroll; else we just delegate the scrolling @@ -1141,6 +1140,7 @@ psyco_curs_scroll(cursorObject *self, PyObject *args, PyObject *kwargs) char buffer[128]; EXC_IF_NO_MARK(self); + EXC_IF_ASYNC_IN_PROGRESS(self, scroll) EXC_IF_TPC_PREPARED(self->conn, scroll); if (strcmp(mode, "absolute") == 0) { diff --git a/psycopg/green.c b/psycopg/green.c index c9b6e07f..65578f51 100644 --- a/psycopg/green.c +++ b/psycopg/green.c @@ -152,6 +152,20 @@ psyco_exec_green(connectionObject *conn, const char *command) { PGresult *result = NULL; + /* Check that there is a single concurrently executing query */ + if (conn->async_cursor) { + PyErr_SetString(ProgrammingError, + "a single async query can be executed on the same connection"); + goto end; + } + /* we don't care about which cursor is executing the query, and + * it may also be that no cursor is involved at all and this is + * an internal query. So just store anything in the async_cursor, + * respecting the code expecting it to be a weakref */ + if (!(conn->async_cursor = PyWeakref_NewRef((PyObject*)conn, NULL))) { + goto end; + } + /* Send the query asynchronously */ if (0 == pq_send_query(conn, command)) { goto end; @@ -173,6 +187,7 @@ psyco_exec_green(connectionObject *conn, const char *command) end: conn->async_status = ASYNC_DONE; + Py_CLEAR(conn->async_cursor); return result; } From 869d48b6f0192ceb781e9b8a59588552df6b2500 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jun 2011 16:26:01 +0100 Subject: [PATCH 22/47] Use the pqpath functions to get/set GUC parameters Functions conn_setup(), conn_get_isolation_level(), conn_set_transaction(), conn_switch_isolation_level(), conn_set_client_encoding() reimplemented using the pqpath funtitons. Dropped analogous function in the connection, as it had to take the lock, thus it was hard to build consistent pieces of functionality with it. --- psycopg/connection.h | 5 +- psycopg/connection_int.c | 170 +++++++++++++++++++++----------------- psycopg/connection_type.c | 43 ++++------ 3 files changed, 116 insertions(+), 102 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index d79392bc..30ac5f42 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -59,7 +59,6 @@ extern "C" { later change it, she must know what she's doing... these are the queries we need to issue */ #define psyco_datestyle "SET DATESTYLE TO 'ISO'" -#define psyco_transaction_isolation "SHOW default_transaction_isolation" extern HIDDEN PyTypeObject connectionType; @@ -134,7 +133,9 @@ HIDDEN int conn_connect(connectionObject *self, long int async); HIDDEN void conn_close(connectionObject *self); HIDDEN int conn_commit(connectionObject *self); HIDDEN int conn_rollback(connectionObject *self); -HIDDEN int conn_set(connectionObject *self, const char *param, const char *value); +HIDDEN int conn_set_transaction(connectionObject *self, const char *isolevel, + const char *readonly, const char *deferrable, + int autocommit); HIDDEN int conn_set_autocommit(connectionObject *self, int value); HIDDEN int conn_switch_isolation_level(connectionObject *self, int level); HIDDEN int conn_set_client_encoding(connectionObject *self, const char *enc); diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 24d424df..4170b32f 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -364,7 +364,8 @@ exit: int conn_get_isolation_level(connectionObject *self) { - PGresult *pgres; + PGresult *pgres = NULL; + char *error = NULL; int rv = -1; char *lname; const IsolationLevel *level; @@ -376,24 +377,13 @@ conn_get_isolation_level(connectionObject *self) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); - Py_BLOCK_THREADS; - if (!psyco_green()) { - Py_UNBLOCK_THREADS; - pgres = PQexec(self->pgconn, psyco_transaction_isolation); - Py_BLOCK_THREADS; - } else { - pgres = psyco_exec_green(self, psyco_transaction_isolation); - } - - if (pgres == NULL || PQresultStatus(pgres) != PGRES_TUPLES_OK) { - PyErr_SetString(OperationalError, - "can't fetch default_transaction_isolation"); + if (!(lname = pq_get_guc_locked(self, "default_transaction_isolation", + &pgres, &error, &_save))) { goto endlock; } /* find the value for the requested isolation level */ - lname = PQgetvalue(pgres, 0, 0); level = conn_isolevels; while ((++level)->name) { if (0 == strcasecmp(level->name, lname)) { @@ -401,23 +391,26 @@ conn_get_isolation_level(connectionObject *self) break; } } - if (-1 == rv) { - char msg[256]; - snprintf(msg, sizeof(msg), "unexpected isolation level: '%s'", lname); - PyErr_SetString(OperationalError, msg); + error = malloc(256); + PyOS_snprintf(error, 256, + "unexpected isolation level: '%s'", lname); } -endlock: - IFCLEARPGRES(pgres); + free(lname); - Py_UNBLOCK_THREADS; +endlock: pthread_mutex_unlock(&self->lock); Py_END_ALLOW_THREADS; + if (rv < 0) { + pq_complete_error(self, &pgres, &error); + } + return rv; } + int conn_get_protocol_version(PGconn *pgconn) { @@ -465,8 +458,8 @@ conn_is_datestyle_ok(PGconn *pgconn) int conn_setup(connectionObject *self, PGconn *pgconn) { - PGresult *pgres; - int green; + PGresult *pgres = NULL; + char *error = NULL; self->equote = conn_get_standard_conforming_strings(pgconn); self->server_version = conn_get_server_version(pgconn); @@ -490,31 +483,20 @@ conn_setup(connectionObject *self, PGconn *pgconn) pthread_mutex_lock(&self->lock); Py_BLOCK_THREADS; - green = psyco_green(); - - if (green && (pq_set_non_blocking(self, 1, 1) != 0)) { + if (psyco_green() && (pq_set_non_blocking(self, 1, 1) != 0)) { return -1; } if (!conn_is_datestyle_ok(self->pgconn)) { - if (!green) { - Py_UNBLOCK_THREADS; - Dprintf("conn_connect: exec query \"%s\";", psyco_datestyle); - pgres = PQexec(pgconn, psyco_datestyle); - Py_BLOCK_THREADS; - } else { - pgres = psyco_exec_green(self, psyco_datestyle); - } - - if (pgres == NULL || PQresultStatus(pgres) != PGRES_COMMAND_OK ) { - PyErr_SetString(OperationalError, "can't set datestyle to ISO"); - IFCLEARPGRES(pgres); - Py_UNBLOCK_THREADS; - pthread_mutex_unlock(&self->lock); - Py_BLOCK_THREADS; + int res; + Py_UNBLOCK_THREADS; + res = pq_set_guc_locked(self, "datestyle", "ISO", + &pgres, &error, &_save); + Py_BLOCK_THREADS; + if (res < 0) { + pq_complete_error(self, &pgres, &error); return -1; } - CLEARPGRES(pgres); } /* for reset */ @@ -976,30 +958,53 @@ conn_rollback(connectionObject *self) return res; } -/* conn_set - set a guc parameter */ - int -conn_set(connectionObject *self, const char *param, const char *value) +conn_set_transaction(connectionObject *self, + const char *isolevel, const char *readonly, const char *deferrable, + int autocommit) { - char query[256]; PGresult *pgres = NULL; char *error = NULL; - int res = 1; - - Dprintf("conn_set: setting %s to %s", param, value); + int res = -1; Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); - if (0 == strcmp(value, "default")) { - sprintf(query, "SET %s TO DEFAULT;", param); - } - else { - sprintf(query, "SET %s TO '%s';", param, value); + if (isolevel) { + Dprintf("conn_set_transaction: setting isolation to %s", isolevel); + if ((res = pq_set_guc_locked(self, + "default_transaction_isolation", isolevel, + &pgres, &error, &_save))) { + goto endlock; + } } - res = pq_execute_command_locked(self, query, &pgres, &error, &_save); + if (readonly) { + Dprintf("conn_set_transaction: setting read only to %s", readonly); + if ((res = pq_set_guc_locked(self, + "default_transaction_read_only", readonly, + &pgres, &error, &_save))) { + goto endlock; + } + } + if (deferrable) { + Dprintf("conn_set_transaction: setting deferrable to %s", deferrable); + if ((res = pq_set_guc_locked(self, + "default_transaction_deferrable", deferrable, + &pgres, &error, &_save))) { + goto endlock; + } + } + + if (self->autocommit != autocommit) { + Dprintf("conn_set_transaction: setting autocommit to %d", autocommit); + self->autocommit = autocommit; + } + + res = 0; + +endlock: pthread_mutex_unlock(&self->lock); Py_END_ALLOW_THREADS; @@ -1029,7 +1034,10 @@ conn_set_autocommit(connectionObject *self, int value) int conn_switch_isolation_level(connectionObject *self, int level) { + PGresult *pgres = NULL; + char *error = NULL; int curr_level; + int ret = -1; /* use only supported levels on older PG versions */ if (self->server_version < 80000) { @@ -1050,16 +1058,21 @@ conn_switch_isolation_level(connectionObject *self, int level) /* Emulate the previous semantic of set_isolation_level() using the * functions currently available. */ + Py_BEGIN_ALLOW_THREADS; + pthread_mutex_lock(&self->lock); + /* terminate the current transaction if any */ - pq_abort(self); + if ((ret = pq_abort_locked(self, &pgres, &error, &_save))) { + goto endlock; + } if (level == 0) { - if (0 != conn_set(self, "default_transaction_isolation", "default")) { - return -1; - } - if (0 != conn_set_autocommit(self, 1)) { - return -1; + if ((ret = pq_set_guc_locked(self, + "default_transaction_isolation", "default", + &pgres, &error, &_save))) { + goto endlock; } + self->autocommit = 1; } else { /* find the name of the requested level */ @@ -1070,22 +1083,33 @@ conn_switch_isolation_level(connectionObject *self, int level) } } if (!isolevel->name) { - PyErr_SetString(OperationalError, "bad isolation level value"); - return -1; + ret = -1; + error = strdup("bad isolation level value"); + goto endlock; } - if (0 != conn_set(self, "default_transaction_isolation", isolevel->name)) { - return -1; - } - if (0 != conn_set_autocommit(self, 0)) { - return -1; + if ((ret = pq_set_guc_locked(self, + "default_transaction_isolation", isolevel->name, + &pgres, &error, &_save))) { + goto endlock; } + self->autocommit = 0; } Dprintf("conn_switch_isolation_level: switched to level %d", level); - return 0; + +endlock: + pthread_mutex_unlock(&self->lock); + Py_END_ALLOW_THREADS; + + if (ret < 0) { + pq_complete_error(self, &pgres, &error); + } + + return ret; } + /* conn_set_client_encoding - switch client encoding on connection */ int @@ -1093,7 +1117,6 @@ conn_set_client_encoding(connectionObject *self, const char *enc) { PGresult *pgres = NULL; char *error = NULL; - char query[48]; int res = 1; char *codec = NULL; char *clean_enc = NULL; @@ -1109,16 +1132,14 @@ conn_set_client_encoding(connectionObject *self, const char *enc) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); - /* set encoding, no encoding string is longer than 24 bytes */ - PyOS_snprintf(query, 47, "SET client_encoding = '%s'", clean_enc); - /* abort the current transaction, to set the encoding ouside of transactions */ if ((res = pq_abort_locked(self, &pgres, &error, &_save))) { goto endlock; } - if ((res = pq_execute_command_locked(self, query, &pgres, &error, &_save))) { + if ((res = pq_set_guc_locked(self, "client_encoding", clean_enc, + &pgres, &error, &_save))) { goto endlock; } @@ -1142,7 +1163,6 @@ conn_set_client_encoding(connectionObject *self, const char *enc) self->encoding, self->codec); endlock: - pthread_mutex_unlock(&self->lock); Py_END_ALLOW_THREADS; diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 3a5b7feb..f3a17f93 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -465,11 +465,16 @@ _psyco_conn_parse_onoff(PyObject *pyval) static PyObject * psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwargs) { - PyObject *isolation_level = Py_None; + PyObject *isolevel = Py_None; PyObject *readonly = Py_None; PyObject *deferrable = Py_None; PyObject *autocommit = Py_None; + const char *c_isolevel = NULL; + const char *c_readonly = NULL; + const char *c_deferrable = NULL; + int c_autocommit = self->autocommit; + static char *kwlist[] = {"isolation_level", "readonly", "deferrable", "autocommit", NULL}; @@ -478,46 +483,34 @@ psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwa EXC_IF_IN_TRANSACTION(self, set_transaction); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOOO", kwlist, - &isolation_level, &readonly, &deferrable, &autocommit)) { + &isolevel, &readonly, &deferrable, &autocommit)) { return NULL; } - if (Py_None != isolation_level) { - const char *value = NULL; - if (!(value = _psyco_conn_parse_isolevel(self, isolation_level))) { - return NULL; - } - if (0 != conn_set(self, "default_transaction_isolation", value)) { + if (Py_None != isolevel) { + if (!(c_isolevel = _psyco_conn_parse_isolevel(self, isolevel))) { return NULL; } } if (Py_None != readonly) { - const char *value = NULL; - if (!(value = _psyco_conn_parse_onoff(readonly))) { - return NULL; - } - if (0 != conn_set(self, "default_transaction_read_only", value)) { + if (!(c_readonly = _psyco_conn_parse_onoff(readonly))) { return NULL; } } - if (Py_None != deferrable) { - const char *value = NULL; - if (!(value = _psyco_conn_parse_onoff(deferrable))) { - return NULL; - } - if (0 != conn_set(self, "default_transaction_deferrable", value)) { + if (!(c_deferrable = _psyco_conn_parse_onoff(deferrable))) { return NULL; } } - if (Py_None != autocommit) { - int value = PyObject_IsTrue(autocommit); - if (-1 == value) { return NULL; } - if (0 != conn_set_autocommit(self, value)) { - return NULL; - } + c_autocommit = PyObject_IsTrue(autocommit); + if (-1 == c_autocommit) { return NULL; } + } + + if (0 != conn_set_transaction(self, + c_isolevel, c_readonly, c_deferrable, c_autocommit)) { + return NULL; } Py_INCREF(Py_None); From 709df38d7912ec1ad06f65ebddb2ee097adf66c7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 5 Jun 2011 16:30:37 +0100 Subject: [PATCH 23/47] Don't clobber an eventual Python exception set by a green thread --- psycopg/pqpath.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index f50b00b4..0e5b2490 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -343,12 +343,12 @@ pq_execute_command_locked(connectionObject *conn, const char *query, *tstate = PyEval_SaveThread(); } if (*pgres == NULL) { - const char *msg; - Dprintf("pq_execute_command_locked: PQexec returned NULL"); - msg = PQerrorMessage(conn->pgconn); - if (msg) - *error = strdup(msg); + if (!PyErr_Occurred()) { + const char *msg; + msg = PQerrorMessage(conn->pgconn); + if (msg && *msg) { *error = strdup(msg); } + } goto cleanup; } @@ -361,8 +361,8 @@ pq_execute_command_locked(connectionObject *conn, const char *query, retvalue = 0; IFCLEARPGRES(*pgres); - - cleanup: + +cleanup: return retvalue; } From dd7ee7093a54213cb5e39fe65a085019f11f5593 Mon Sep 17 00:00:00 2001 From: Jason Erickson Date: Sun, 5 Jun 2011 15:20:19 -0600 Subject: [PATCH 24/47] No strcasecmp function with MSVC The MSVC compiler does not have the strcasecmp(x, y) function, which is a case insensitve string compare function. Instead, MSVC has a similar function, lstrcmpi(x, y). Modified config.h to use this function when building with MSVC. --- psycopg/config.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psycopg/config.h b/psycopg/config.h index 01e1e562..2112043b 100644 --- a/psycopg/config.h +++ b/psycopg/config.h @@ -136,6 +136,8 @@ static int pthread_mutex_init(pthread_mutex_t *mutex, void* fake) * in libxml2 code */ #define isinf(x) ((_fpclass(x) == _FPCLASS_PINF) ? 1 \ : ((_fpclass(x) == _FPCLASS_NINF) ? -1 : 0)) + +#define strcasecmp(x, y) lstrcmpi(x, y) #endif #endif From 2a1b2b5713ffefbfbb22e95b7182577e3cd212a2 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 6 Jun 2011 23:41:31 +0100 Subject: [PATCH 25/47] Added script to demonstrate the refcount bug during copy from https://bugzilla.redhat.com/show_bug.cgi?id=711095 --- scripts/ticket58.py | 59 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 scripts/ticket58.py diff --git a/scripts/ticket58.py b/scripts/ticket58.py new file mode 100644 index 00000000..86cabac3 --- /dev/null +++ b/scripts/ticket58.py @@ -0,0 +1,59 @@ +""" +A script to reproduce the race condition described in ticket #58 + +from https://bugzilla.redhat.com/show_bug.cgi?id=711095 + +Results in the error: + + python: Modules/gcmodule.c:277: visit_decref: Assertion `gc->gc.gc_refs != 0' + failed. + +on unpatched library. +""" + +import threading +import gc +import time + +import psycopg2 +from StringIO import StringIO + +done = 0 + +class GCThread(threading.Thread): + # A thread that sits in an infinite loop, forcing the garbage collector + # to run + def run(self): + global done + while not done: + gc.collect() + time.sleep(0.1) # give the other thread a chance to run + +gc_thread = GCThread() + + +# This assumes a pre-existing db named "test", with: +# "CREATE TABLE test (id serial PRIMARY KEY, num integer, data varchar);" + +conn = psycopg2.connect("dbname=test user=postgres") +cur = conn.cursor() + +# Start the other thread, running the GC regularly +gc_thread.start() + +# Now do lots of "cursor.copy_from" calls: +for i in range(1000): + f = StringIO("42\tfoo\n74\tbar\n") + cur.copy_from(f, 'test', columns=('num', 'data')) + # Assuming the other thread gets a chance to run during this call, expect a + # build of python (with assertions enabled) to bail out here with: + # python: Modules/gcmodule.c:277: visit_decref: Assertion `gc->gc.gc_refs != 0' failed. + + +# Terminate the GC thread's loop: +done = 1 + +cur.close() +conn.close() + + From 1888bf41c0dcdde6d6ef825393554121d88a69e1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 6 Jun 2011 23:47:13 +0100 Subject: [PATCH 26/47] Added patch for refcount bug in copy_from By Dave Malcolm. https://bugzilla.redhat.com/show_bug.cgi?id=711095 (slightly edited to increment the refcount before storing the pointer in the cursor). --- NEWS | 2 ++ psycopg/cursor_type.c | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index c7b2aaac..7f0070cd 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,8 @@ What's new in psycopg 2.4.2 support was built (ticket #53). - Fixed escape for negative numbers prefixed by minus operator (ticket #57). + - Fixed refcount issue during copy. Reported and fixed by Dave + Malcolm (ticket #58, Red Hat Bug 711095). - Trying to execute concurrent operations on the same connection through concurrent green thread results in an error instead of a deadlock. diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index c18fb71d..0aa88c6b 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1220,8 +1220,12 @@ _psyco_curs_has_read_check(PyObject* o, void* var) { if (PyObject_HasAttrString(o, "readline") && PyObject_HasAttrString(o, "read")) { - /* It's OK to store a borrowed reference, because it is only held for - * the duration of psyco_curs_copy_from. */ + /* This routine stores a borrowed reference. Although it is only held + * for the duration of psyco_curs_copy_from, nested invocations of + * Py_BEGIN_ALLOW_THREADS could surrender control to another thread, + * which could invoke the garbage collector. We thus need an + * INCREF/DECREF pair if we store this pointer in a GC object, such as + * a cursorObject */ *((PyObject**)var) = o; return 1; } @@ -1311,6 +1315,7 @@ psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs) Dprintf("psyco_curs_copy_from: query = %s", query); self->copysize = bufsize; + Py_INCREF(file); self->copyfile = file; if (pq_execute(self, query, 0) == 1) { @@ -1319,6 +1324,7 @@ psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs) } self->copyfile = NULL; + Py_DECREF(file); exit: PyMem_Free(quoted_delimiter); From b6e710b0fc022e009f51ca644cbcaa48f5207997 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 00:07:59 +0100 Subject: [PATCH 27/47] Fixed refcount bug in copy_to() and copy_expert() methods too --- psycopg/cursor_type.c | 31 ++++++++++++++----------------- scripts/ticket58.py | 18 +++++++++++++++++- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 0aa88c6b..f850ebe3 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1343,8 +1343,6 @@ static int _psyco_curs_has_write_check(PyObject* o, void* var) { if (PyObject_HasAttrString(o, "write")) { - /* It's OK to store a borrowed reference, because it is only held for - * the duration of psyco_curs_copy_to. */ *((PyObject**)var) = o; return 1; } @@ -1430,12 +1428,15 @@ psyco_curs_copy_to(cursorObject *self, PyObject *args, PyObject *kwargs) Dprintf("psyco_curs_copy_to: query = %s", query); self->copysize = 0; + Py_INCREF(file); self->copyfile = file; if (pq_execute(self, query, 0) == 1) { res = Py_None; Py_INCREF(Py_None); } + + Py_DECREF(file); self->copyfile = NULL; exit: @@ -1477,18 +1478,18 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs) EXC_IF_TPC_PREPARED(self->conn, copy_expert); sql = _psyco_curs_validate_sql_basic(self, sql); - - /* Any failure from here forward should 'goto fail' rather than + + /* Any failure from here forward should 'goto exit' rather than 'return NULL' directly. */ - - if (sql == NULL) { goto fail; } + + if (sql == NULL) { goto exit; } /* This validation of file is rather weak, in that it doesn't enforce the assocation between "COPY FROM" -> "read" and "COPY TO" -> "write". However, the error handling in _pq_copy_[in|out] must be able to handle the case where the attempt to call file.read|write fails, so no harm done. */ - + if ( !PyObject_HasAttrString(file, "read") && !PyObject_HasAttrString(file, "write") ) @@ -1496,26 +1497,22 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_TypeError, "file must be a readable file-like" " object for COPY FROM; a writeable file-like object for COPY TO." ); - goto fail; + goto exit; } self->copysize = bufsize; + Py_INCREF(file); self->copyfile = file; /* At this point, the SQL statement must be str, not unicode */ - if (pq_execute(self, Bytes_AS_STRING(sql), 0) != 1) { goto fail; } + if (pq_execute(self, Bytes_AS_STRING(sql), 0) != 1) { goto exit; } res = Py_None; Py_INCREF(res); - goto cleanup; - fail: - if (res != NULL) { - Py_DECREF(res); - res = NULL; - } - /* Fall through to cleanup */ - cleanup: + +exit: self->copyfile = NULL; + Py_XDECREF(file); Py_XDECREF(sql); return res; diff --git a/scripts/ticket58.py b/scripts/ticket58.py index 86cabac3..95520c1e 100644 --- a/scripts/ticket58.py +++ b/scripts/ticket58.py @@ -42,13 +42,29 @@ cur = conn.cursor() gc_thread.start() # Now do lots of "cursor.copy_from" calls: +print "copy_from" for i in range(1000): f = StringIO("42\tfoo\n74\tbar\n") cur.copy_from(f, 'test', columns=('num', 'data')) # Assuming the other thread gets a chance to run during this call, expect a # build of python (with assertions enabled) to bail out here with: # python: Modules/gcmodule.c:277: visit_decref: Assertion `gc->gc.gc_refs != 0' failed. - + +# Also exercise the copy_to code path +print "copy_to" +cur.execute("truncate test") +f = StringIO("42\tfoo\n74\tbar\n") +cur.copy_from(f, 'test', columns=('num', 'data')) +for i in range(1000): + f = StringIO() + cur.copy_to(f, 'test', columns=('num', 'data')) + +# And copy_expert too +print "copy_expert" +cur.execute("truncate test") +for i in range(1000): + f = StringIO("42\tfoo\n74\tbar\n") + cur.copy_expert("copy test to stdout", f) # Terminate the GC thread's loop: done = 1 From 679af4a97528c140ac8e9cd85598f717b3f5cfb9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 01:20:25 +0100 Subject: [PATCH 28/47] Fixed copyfile refcount in copy_expert In case of early error, jumping to exit would have decref'd the borrowed reference to file. Issue spotted by Dave Malcolm, thanks! --- psycopg/cursor_type.c | 11 ++++++----- tests/test_copy.py | 9 +++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index f850ebe3..048ab6d8 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1505,14 +1505,15 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs) self->copyfile = file; /* At this point, the SQL statement must be str, not unicode */ - if (pq_execute(self, Bytes_AS_STRING(sql), 0) != 1) { goto exit; } + if (pq_execute(self, Bytes_AS_STRING(sql), 0) == 1) { + res = Py_None; + Py_INCREF(res); + } - res = Py_None; - Py_INCREF(res); + self->copyfile = NULL; + Py_DECREF(file); exit: - self->copyfile = NULL; - Py_XDECREF(file); Py_XDECREF(sql); return res; diff --git a/tests/test_copy.py b/tests/test_copy.py index 9026abc5..7ec1b767 100755 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -244,6 +244,15 @@ class CopyTests(unittest.TestCase): self.assertEqual(ntests, len(string.ascii_letters)) + def test_copy_expert_file_refcount(self): + class Whatever(object): + pass + + f = Whatever() + curs = self.conn.cursor() + self.assertRaises(TypeError, + curs.copy_expert, 'COPY tcopy (data) FROM STDIN', f) + decorate_all_tests(CopyTests, skip_if_green) From f3526d06303f6ddbd3ebf43191f979021a6f2fcc Mon Sep 17 00:00:00 2001 From: Steve Lacy Date: Mon, 6 Jun 2011 14:01:52 -0700 Subject: [PATCH 29/47] Refactoring of the pg_config detection code in setup.py Pull all state and path searching into it's own class. --- setup.py | 268 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 140 insertions(+), 128 deletions(-) diff --git a/setup.py b/setup.py index f155119d..86a75d6a 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,6 @@ Operating System :: Unix # Note: The setup.py must be compatible with both Python 2 and 3 import os -import os.path import sys import re import subprocess @@ -54,7 +53,6 @@ from distutils.errors import DistutilsFileError from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc from distutils.ccompiler import get_default_compiler -from distutils.dep_util import newer_group from distutils.util import get_platform try: from distutils.msvc9compiler import MSVCCompiler @@ -85,21 +83,137 @@ version_flags = ['dt', 'dec'] PLATFORM_IS_WINDOWS = sys.platform.lower().startswith('win') -def get_pg_config(kind, pg_config): - try: - p = subprocess.Popen([pg_config, "--" + kind], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - except OSError: - raise Warning("Unable to find 'pg_config' file in '%s'" % pg_config) - p.stdin.close() - r = p.stdout.readline().strip() - if not r: - raise Warning(p.stderr.readline()) - if not isinstance(r, str): - r = r.decode('ascii') - return r + +class PostgresConfig(): + def __init__(self): + self.pg_config_exe = self.autodetect_pg_config_path() + if self.pg_config_exe is None: + sys.stderr.write("""\ +Error: pg_config executable not found. + +Please add the directory containing pg_config to the PATH +or specify the full executable path with the option: + + python setup.py build_ext --pg-config /path/to/pg_config build ... + +or with the pg_config option in 'setup.cfg'. +""") + sys.exit(1) + + def query(self, attr_name): + """Spawn the pg_config executable, querying for the given config + name, and return the printed value, sanitized. """ + try: + pg_config_process = subprocess.Popen( + [self.pg_config_exe, "--" + attr_name], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError: + raise Warning("Unable to find 'pg_config' file in '%s'" % + self.pg_config_exe) + pg_config_process.stdin.close() + result = pg_config_process.stdout.readline().strip() + if not result: + raise Warning(pg_config_process.stderr.readline()) + if not isinstance(result, str): + result = result.decode('ascii') + return result + + def autodetect_pg_config_path(self): + """Find and return the path to the pg_config executable.""" + if PLATFORM_IS_WINDOWS: + return self.autodetect_pg_config_path_windows() + else: + return self.autodetect_pg_config_path_posix() + + def autodetect_pg_config_path_posix(self): + """Return pg_config from the current PATH""" + exename = 'pg_config' + for dir in os.environ['PATH'].split(os.pathsep): + fn = os.path.join(dir, exename) + if os.path.isfile(fn): + return fn + return None + + def autodetect_pg_config_path_windows(self): + """Attempt several different ways of finding the pg_config + executable on Windows, and return its full path, if found.""" + # Find the first PostgreSQL installation listed in the registry and + # return the full path to its pg_config utility. + # + # This autodetection is performed *only* if the following conditions + # hold: + # + # 1) The pg_config utility is not already available on the PATH: + if os.popen('pg_config').close() is None: # .close()->None == success + return None + # 2) The user has not specified any of the following settings in + # setup.cfg: + # - pg_config + # - include_dirs + # - library_dirs + for setting_name in ('pg_config', 'include_dirs', 'library_dirs'): + try: + val = parser.get('build_ext', setting_name) + except configparser.NoOptionError: + pass + else: + if val.strip() != '': + return None + # end of guard conditions + + try: + import winreg + except ImportError: + import _winreg as winreg + + pg_inst_base_dir = None + pg_config_path = None + + reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + try: + pg_inst_list_key = winreg.OpenKey(reg, + 'SOFTWARE\\PostgreSQL\\Installations' + ) + except EnvironmentError: + pg_inst_list_key = None + + if pg_inst_list_key is not None: + try: + # Determine the name of the first subkey, if any: + try: + first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) + except EnvironmentError: + first_sub_key_name = None + + if first_sub_key_name is not None: + pg_first_inst_key = winreg.OpenKey(reg, + 'SOFTWARE\\PostgreSQL\\Installations\\' + + first_sub_key_name + ) + try: + pg_inst_base_dir = winreg.QueryValueEx( + pg_first_inst_key, 'Base Directory' + )[0] + finally: + winreg.CloseKey(pg_first_inst_key) + finally: + winreg.CloseKey(pg_inst_list_key) + + if pg_inst_base_dir and os.path.exists(pg_inst_base_dir): + pg_config_path = os.path.join(pg_inst_base_dir, 'bin', + 'pg_config.exe' + ) + # Support unicode paths, if this version of Python provides the + # necessary infrastructure: + if sys.version_info[0] < 3 \ + and hasattr(sys, 'getfilesystemencoding'): + pg_config_path = pg_config_path.encode( + sys.getfilesystemencoding()) + + return pg_config_path + class psycopg_build_ext(build_ext): """Conditionally complement the setup.cfg options file. @@ -126,6 +240,10 @@ class psycopg_build_ext(build_ext): boolean_options = build_ext.boolean_options[:] boolean_options.extend(('use-pydatetime', 'have-ssl', 'static-libpq')) + def __init__(self, *args, **kwargs): + build_ext.__init__(self, *args, **kwargs) + self.pg_config = PostgresConfig() + def initialize_options(self): build_ext.initialize_options(self) self.use_pg_dll = 1 @@ -134,7 +252,6 @@ class psycopg_build_ext(build_ext): self.use_pydatetime = 1 self.have_ssl = have_ssl self.static_libpq = static_libpq - self.pg_config = None def get_compiler(self): """Return the name of the C compiler used to compile extensions. @@ -153,9 +270,6 @@ class psycopg_build_ext(build_ext): name = get_default_compiler() return name - def get_pg_config(self, kind): - return get_pg_config(kind, self.pg_config) - def get_export_symbols(self, ext): # Fix MSVC seeing two of the same export symbols. if self.get_compiler().lower().startswith('msvc'): @@ -248,38 +362,24 @@ class psycopg_build_ext(build_ext): def finalize_options(self): """Complete the build system configuation.""" build_ext.finalize_options(self) - if self.pg_config is None: - self.pg_config = self.autodetect_pg_config_path() - if self.pg_config is None: - sys.stderr.write("""\ -Error: pg_config executable not found. - -Please add the directory containing pg_config to the PATH -or specify the full executable path with the option: - - python setup.py build_ext --pg-config /path/to/pg_config build ... - -or with the pg_config option in 'setup.cfg'. -""") - sys.exit(1) self.include_dirs.append(".") if self.static_libpq: if not self.link_objects: self.link_objects = [] self.link_objects.append( - os.path.join(self.get_pg_config("libdir"), "libpq.a")) + os.path.join(self.pg_config.query("libdir"), "libpq.a")) else: self.libraries.append("pq") try: - self.library_dirs.append(self.get_pg_config("libdir")) - self.include_dirs.append(self.get_pg_config("includedir")) - self.include_dirs.append(self.get_pg_config("includedir-server")) + self.library_dirs.append(self.pg_config.query("libdir")) + self.include_dirs.append(self.pg_config.query("includedir")) + self.include_dirs.append(self.pg_config.query("includedir-server")) try: # Here we take a conservative approach: we suppose that # *at least* PostgreSQL 7.4 is available (this is the only # 7.x series supported by psycopg 2) - pgversion = self.get_pg_config("version").split()[1] + pgversion = self.pg_config.query("version").split()[1] except: pgversion = "7.4.0" @@ -305,94 +405,6 @@ or with the pg_config option in 'setup.cfg'. if hasattr(self, "finalize_" + sys.platform): getattr(self, "finalize_" + sys.platform)() - def autodetect_pg_config_path(self): - if PLATFORM_IS_WINDOWS: - return self.autodetect_pg_config_path_windows() - else: - return self.autodetect_pg_config_path_posix() - - def autodetect_pg_config_path_posix(self): - exename = 'pg_config' - for dir in os.environ['PATH'].split(os.pathsep): - fn = os.path.join(dir, exename) - if os.path.isfile(fn): - return fn - - def autodetect_pg_config_path_windows(self): - # Find the first PostgreSQL installation listed in the registry and - # return the full path to its pg_config utility. - # - # This autodetection is performed *only* if the following conditions - # hold: - # - # 1) The pg_config utility is not already available on the PATH: - if os.popen('pg_config').close() is None: # .close()->None == success - return None - # 2) The user has not specified any of the following settings in - # setup.cfg: - # - pg_config - # - include_dirs - # - library_dirs - for settingName in ('pg_config', 'include_dirs', 'library_dirs'): - try: - val = parser.get('build_ext', settingName) - except configparser.NoOptionError: - pass - else: - if val.strip() != '': - return None - # end of guard conditions - - try: - import winreg - except ImportError: - import _winreg as winreg - - pg_inst_base_dir = None - pg_config_path = None - - reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) - try: - pg_inst_list_key = winreg.OpenKey(reg, - 'SOFTWARE\\PostgreSQL\\Installations' - ) - except EnvironmentError: - pg_inst_list_key = None - - if pg_inst_list_key is not None: - try: - # Determine the name of the first subkey, if any: - try: - first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) - except EnvironmentError: - first_sub_key_name = None - - if first_sub_key_name is not None: - pg_first_inst_key = winreg.OpenKey(reg, - 'SOFTWARE\\PostgreSQL\\Installations\\' - + first_sub_key_name - ) - try: - pg_inst_base_dir = winreg.QueryValueEx( - pg_first_inst_key, 'Base Directory' - )[0] - finally: - winreg.CloseKey(pg_first_inst_key) - finally: - winreg.CloseKey(pg_inst_list_key) - - if pg_inst_base_dir and os.path.exists(pg_inst_base_dir): - pg_config_path = os.path.join(pg_inst_base_dir, 'bin', - 'pg_config.exe' - ) - # Support unicode paths, if this version of Python provides the - # necessary infrastructure: - if sys.version_info[0] < 3 \ - and hasattr(sys, 'getfilesystemencoding'): - pg_config_path = pg_config_path.encode( - sys.getfilesystemencoding()) - - return pg_config_path # let's start with macro definitions (the ones not already in setup.cfg) define_macros = [] From c826446ff839f2a4d0d9d0327b57dfd973f19ff7 Mon Sep 17 00:00:00 2001 From: Steve Lacy Date: Mon, 6 Jun 2011 14:19:27 -0700 Subject: [PATCH 30/47] Clean up a bunch of lint from pylint/pyflakes/pep8 checking - Don't override global variable name "ext" (use "extension" as function argument names) - Improve function naming (get_compiler -> get_compiler_name) - Other misc operator spacing and 80-column violation cleanup. - Remove unneeded import (DistUtilsFileError) --- setup.py | 93 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/setup.py b/setup.py index 86a75d6a..033e57b6 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,6 @@ import sys import re import subprocess from distutils.core import setup, Extension -from distutils.errors import DistutilsFileError from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc from distutils.ccompiler import get_default_compiler @@ -130,10 +129,10 @@ or with the pg_config option in 'setup.cfg'. def autodetect_pg_config_path_posix(self): """Return pg_config from the current PATH""" exename = 'pg_config' - for dir in os.environ['PATH'].split(os.pathsep): - fn = os.path.join(dir, exename) - if os.path.isfile(fn): - return fn + for dir_name in os.environ['PATH'].split(os.pathsep): + fullpath = os.path.join(dir_name, exename) + if os.path.isfile(fullpath): + return fullpath return None def autodetect_pg_config_path_windows(self): @@ -253,7 +252,7 @@ class psycopg_build_ext(build_ext): self.have_ssl = have_ssl self.static_libpq = static_libpq - def get_compiler(self): + def get_compiler_name(self): """Return the name of the C compiler used to compile extensions. If a compiler was not explicitly set (on the command line, for @@ -270,15 +269,15 @@ class psycopg_build_ext(build_ext): name = get_default_compiler() return name - def get_export_symbols(self, ext): + def get_export_symbols(self, extension): # Fix MSVC seeing two of the same export symbols. - if self.get_compiler().lower().startswith('msvc'): + if self.get_compiler_name().lower().startswith('msvc'): return [] else: - return build_ext.get_export_symbols(self, ext) + return build_ext.get_export_symbols(self, extension) - def build_extension(self, ext): - build_ext.build_extension(self, ext) + def build_extension(self, extension): + build_ext.build_extension(self, extension) # For Python versions that use MSVC compiler 2008, re-insert the # manifest into the resulting .pyd file. @@ -288,19 +287,21 @@ class psycopg_build_ext(build_ext): manifest = '_psycopg.vc9.x86.manifest' if platform == 'win-amd64': manifest = '_psycopg.vc9.amd64.manifest' - self.compiler.spawn(['mt.exe', '-nologo', '-manifest', - os.path.join('psycopg', manifest), - '-outputresource:%s;2' % (os.path.join(self.build_lib, 'psycopg2', '_psycopg.pyd'))]) + self.compiler.spawn( + ['mt.exe', '-nologo', '-manifest', + os.path.join('psycopg', manifest), + '-outputresource:%s;2' % ( + os.path.join(self.build_lib, + 'psycopg2', '_psycopg.pyd'))]) def finalize_win32(self): """Finalize build system configuration on win32 platform.""" - import struct sysVer = sys.version_info[:2] # Add compiler-specific arguments: extra_compiler_args = [] - compiler_name = self.get_compiler().lower() + compiler_name = self.get_compiler_name().lower() compiler_is_msvc = compiler_name.startswith('msvc') compiler_is_mingw = compiler_name.startswith('mingw') if compiler_is_mingw: @@ -315,18 +316,18 @@ class psycopg_build_ext(build_ext): extra_compiler_args.append('-fno-strict-aliasing') # Force correct C runtime library linkage: - if sysVer <= (2,3): + if sysVer <= (2, 3): # Yes: 'msvcr60', rather than 'msvcrt', is the correct value # on the line below: self.libraries.append('msvcr60') - elif sysVer in ((2,4), (2,5)): + elif sysVer in ((2, 4), (2, 5)): self.libraries.append('msvcr71') # Beyond Python 2.5, we take our chances on the default C runtime # library, because we don't know what compiler those future # versions of Python will use. - for exten in ext: # ext is a global list of Extension objects - exten.extra_compile_args.extend(extra_compiler_args) + for extension in ext: # ext is a global list of Extension objects + extension.extra_compile_args.extend(extra_compiler_args) # End of add-compiler-specific arguments section. self.libraries.append("ws2_32") @@ -356,8 +357,9 @@ class psycopg_build_ext(build_ext): def finalize_linux2(self): """Finalize build system configuration on GNU/Linux platform.""" # tell piro that GCC is fine and dandy, but not so MS compilers - for ext in self.extensions: - ext.extra_compile_args.append('-Wdeclaration-after-statement') + for extension in self.extensions: + extension.extra_compile_args.append( + '-Wdeclaration-after-statement') def finalize_options(self): """Complete the build system configuation.""" @@ -365,7 +367,8 @@ class psycopg_build_ext(build_ext): self.include_dirs.append(".") if self.static_libpq: - if not self.link_objects: self.link_objects = [] + if not self.link_objects: + self.link_objects = [] self.link_objects.append( os.path.join(self.pg_config.query("libdir"), "libpq.a")) else: @@ -383,7 +386,8 @@ class psycopg_build_ext(build_ext): except: pgversion = "7.4.0" - verre = re.compile(r"(\d+)\.(\d+)(?:(?:\.(\d+))|(devel|(alpha|beta|rc)\d+))") + verre = re.compile( + r"(\d+)\.(\d+)(?:(?:\.(\d+))|(devel|(alpha|beta|rc)\d+))") m = verre.match(pgversion) if m: pgmajor, pgminor, pgpatch = m.group(1, 2, 3) @@ -398,7 +402,7 @@ class psycopg_build_ext(build_ext): define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % (int(pgmajor), int(pgminor), int(pgpatch)))) except Warning: - w = sys.exc_info()[1] # work around py 2/3 different syntax + w = sys.exc_info()[1] # work around py 2/3 different syntax sys.stderr.write("Error: %s\n" % w) sys.exit(1) @@ -411,7 +415,8 @@ define_macros = [] include_dirs = [] # gather information to build the extension module -ext = [] ; data_files = [] +ext = [] +data_files = [] # sources @@ -464,7 +469,7 @@ else: if os.path.exists(mxincludedir): # Build the support for mx: we will check at runtime if it can be imported include_dirs.append(mxincludedir) - define_macros.append(('HAVE_MXDATETIME','1')) + define_macros.append(('HAVE_MXDATETIME', '1')) sources.append('adapter_mxdatetime.c') depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) have_mxdatetime = True @@ -472,18 +477,21 @@ if os.path.exists(mxincludedir): # now decide which package will be the default for date/time typecasts if have_pydatetime and (use_pydatetime or not have_mxdatetime): - define_macros.append(('PSYCOPG_DEFAULT_PYDATETIME','1')) + define_macros.append(('PSYCOPG_DEFAULT_PYDATETIME', '1')) elif have_mxdatetime: - define_macros.append(('PSYCOPG_DEFAULT_MXDATETIME','1')) + define_macros.append(('PSYCOPG_DEFAULT_MXDATETIME', '1')) else: - def e(msg): - sys.stderr.write("error: " + msg + "\n") - e("psycopg requires a datetime module:") - e(" mx.DateTime module not found") - e(" python datetime module not found") - e("Note that psycopg needs the module headers and not just the module") - e("itself. If you installed Python or mx.DateTime from a binary package") - e("you probably need to install its companion -dev or -devel package.") + error_message = """\ +psycopg requires a datetime module: + mx.DateTime module not found + python datetime module not found + +Note that psycopg needs the module headers and not just the module +itself. If you installed Python or mx.DateTime from a binary package +you probably need to install its companion -dev or -devel package.""" + + for line in error_message.split("\n"): + sys.stderr.write("error: " + line) sys.exit(1) # generate a nice version string to avoid confusion when users report bugs @@ -497,9 +505,9 @@ else: PSYCOPG_VERSION_EX = PSYCOPG_VERSION if not PLATFORM_IS_WINDOWS: - define_macros.append(('PSYCOPG_VERSION', '"'+PSYCOPG_VERSION_EX+'"')) + define_macros.append(('PSYCOPG_VERSION', '"' + PSYCOPG_VERSION_EX + '"')) else: - define_macros.append(('PSYCOPG_VERSION', '\\"'+PSYCOPG_VERSION_EX+'\\"')) + define_macros.append(('PSYCOPG_VERSION', '\\"' + PSYCOPG_VERSION_EX + '\\"')) if parser.has_option('build_ext', 'have_ssl'): have_ssl = int(parser.get('build_ext', 'have_ssl')) @@ -537,17 +545,16 @@ setup(name="psycopg2", author="Federico Di Gregorio", author_email="fog@initd.org", url="http://initd.org/psycopg/", - download_url = download_url, + download_url=download_url, license="GPL with exceptions or ZPL", - platforms = ["any"], + platforms=["any"], description=__doc__.split("\n")[0], long_description="\n".join(__doc__.split("\n")[2:]), classifiers=[x for x in classifiers.split("\n") if x], data_files=data_files, - package_dir={'psycopg2':'lib', 'psycopg2.tests': 'tests'}, + package_dir={'psycopg2': 'lib', 'psycopg2.tests': 'tests'}, packages=['psycopg2', 'psycopg2.tests'], cmdclass={ 'build_ext': psycopg_build_ext, 'build_py': build_py, }, ext_modules=ext) - From ef1891539633a867bd7d2fb389bf0603af0c4410 Mon Sep 17 00:00:00 2001 From: Steve Lacy Date: Mon, 6 Jun 2011 14:30:47 -0700 Subject: [PATCH 31/47] Unify the way the MSVC compiler is detected And do it only once in __init__ instead of different ways and in different places. --- setup.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 033e57b6..8a7a4137 100644 --- a/setup.py +++ b/setup.py @@ -53,10 +53,7 @@ from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc from distutils.ccompiler import get_default_compiler from distutils.util import get_platform -try: - from distutils.msvc9compiler import MSVCCompiler -except ImportError: - MSVCCompiler = None + try: from distutils.command.build_py import build_py_2to3 as build_py except ImportError: @@ -242,6 +239,9 @@ class psycopg_build_ext(build_ext): def __init__(self, *args, **kwargs): build_ext.__init__(self, *args, **kwargs) self.pg_config = PostgresConfig() + compiler_name = self.get_compiler_name().lower() + self.compiler_is_msvc = compiler_name.startswith('msvc') + self.compiler_is_mingw = compiler_name.startswith('mingw') def initialize_options(self): build_ext.initialize_options(self) @@ -271,7 +271,7 @@ class psycopg_build_ext(build_ext): def get_export_symbols(self, extension): # Fix MSVC seeing two of the same export symbols. - if self.get_compiler_name().lower().startswith('msvc'): + if self.compiler_is_msvc: return [] else: return build_ext.get_export_symbols(self, extension) @@ -281,7 +281,7 @@ class psycopg_build_ext(build_ext): # For Python versions that use MSVC compiler 2008, re-insert the # manifest into the resulting .pyd file. - if MSVCCompiler and isinstance(self.compiler, MSVCCompiler): + if self.compiler_is_msvc: platform = get_platform() # Default to the x86 manifest manifest = '_psycopg.vc9.x86.manifest' @@ -301,10 +301,7 @@ class psycopg_build_ext(build_ext): # Add compiler-specific arguments: extra_compiler_args = [] - compiler_name = self.get_compiler_name().lower() - compiler_is_msvc = compiler_name.startswith('msvc') - compiler_is_mingw = compiler_name.startswith('mingw') - if compiler_is_mingw: + if self.compiler_is_mingw: # Default MinGW compilation of Python extensions on Windows uses # only -O: extra_compiler_args.append('-O3') @@ -332,7 +329,7 @@ class psycopg_build_ext(build_ext): self.libraries.append("ws2_32") self.libraries.append("advapi32") - if compiler_is_msvc: + if self.compiler_is_msvc: # MSVC requires an explicit "libpq" self.libraries.remove("pq") self.libraries.append("secur32") From 9a7aee3d05fa0df73a2e6bf792b6c3f2bb2a81b9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 08:58:54 +0100 Subject: [PATCH 32/47] Fixed compatibility problem in setup for Python 2.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8a7a4137..ae4d297c 100644 --- a/setup.py +++ b/setup.py @@ -80,7 +80,7 @@ version_flags = ['dt', 'dec'] PLATFORM_IS_WINDOWS = sys.platform.lower().startswith('win') -class PostgresConfig(): +class PostgresConfig: def __init__(self): self.pg_config_exe = self.autodetect_pg_config_path() if self.pg_config_exe is None: From 9b5ac7951365c166535f645be1d612afc0cc1ec2 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 10:46:43 +0100 Subject: [PATCH 33/47] Fixed default size for read copy buffer The original commit stated it should have been 8192. --- psycopg/cursor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psycopg/cursor.h b/psycopg/cursor.h index 09ac12a8..c6ca25b4 100644 --- a/psycopg/cursor.h +++ b/psycopg/cursor.h @@ -64,7 +64,7 @@ struct cursorObject { PyObject *copyfile; /* file-like used during COPY TO/FROM ops */ Py_ssize_t copysize; /* size of the copy buffer during COPY TO/FROM ops */ #define DEFAULT_COPYSIZE 16384 -#define DEFAULT_COPYBUFF 8132 +#define DEFAULT_COPYBUFF 8192 PyObject *tuple_factory; /* factory for result tuples */ PyObject *tzinfo_factory; /* factory for tzinfo objects */ From 6d907df14d57c1bde297e60dd242c8391f79f08d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 10:48:26 +0100 Subject: [PATCH 34/47] Fixed documentation for COPY methods The size parameter in copy_from was undocumented (ticket #59). --- doc/src/cursor.rst | 53 ++++++++++++++++++++++++------------------- psycopg/cursor_type.c | 4 ++-- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/doc/src/cursor.rst b/doc/src/cursor.rst index 3982d94a..d5cf5244 100644 --- a/doc/src/cursor.rst +++ b/doc/src/cursor.rst @@ -444,20 +444,23 @@ The ``cursor`` class The :sql:`COPY` command is a PostgreSQL extension to the SQL standard. As such, its support is a Psycopg extension to the |DBAPI|. - .. method:: copy_from(file, table, sep='\\t', null='\\N', columns=None) - + .. method:: copy_from(file, table, sep='\\t', null='\\N', size=8192, columns=None) + Read data *from* the file-like object *file* appending them to - the table named *table*. *file* must have both - `!read()` and `!readline()` method. See :ref:`copy` for an - overview. + the table named *table*. See :ref:`copy` for an overview. - The optional argument *sep* is the columns separator and - *null* represents :sql:`NULL` values in the file. + :param file: file-like object to read data from. It must have both + `!read()` and `!readline()` methods. + :param table: name of the table to copy data into. + :param sep: columns separator expected in the file. Defaults to a tab. + :param null: textual representation of :sql:`NULL` in the file. + :param size: size of the buffer used to read from the file. + :param columns: iterable with name of the columns to import. + The length and types should match the content of the file to read. + If not specified, it is assumed that the entire table matches the + file structure. - The *columns* argument is a sequence containing the name of the - fields where the read data will be entered. Its length and column - type should match the content of the read file. If not specifies, it - is assumed that the entire table matches the file structure. + Example:: >>> f = StringIO("42\tfoo\n74\tbar\n") >>> cur.copy_from(f, 'test', columns=('num', 'data')) @@ -476,14 +479,17 @@ The ``cursor`` class .. method:: copy_to(file, table, sep='\\t', null='\\N', columns=None) Write the content of the table named *table* *to* the file-like - object *file*. *file* must have a `!write()` method. - See :ref:`copy` for an overview. + object *file*. See :ref:`copy` for an overview. - The optional argument *sep* is the columns separator and - *null* represents :sql:`NULL` values in the file. + :param file: file-like object to write data into. It must have a + `!write()` method. + :param table: name of the table to copy data from. + :param sep: columns separator expected in the file. Defaults to a tab. + :param null: textual representation of :sql:`NULL` in the file. + :param columns: iterable with name of the columns to export. + If not specified, export all the columns. - The *columns* argument is a sequence of field names: if not - `!None` only the specified fields will be included in the dump. + Example:: >>> cur.copy_to(sys.stdout, 'test', sep="|") 1|100|abc'def @@ -499,17 +505,18 @@ The ``cursor`` class from the backend. - .. method:: copy_expert(sql, file [, size]) + .. method:: copy_expert(sql, file, size=8192) Submit a user-composed :sql:`COPY` statement. The method is useful to handle all the parameters that PostgreSQL makes available (see |COPY|__ command documentation). - *file* must be an open, readable file for :sql:`COPY FROM` or an - open, writeable file for :sql:`COPY TO`. The optional *size* - argument, when specified for a :sql:`COPY FROM` statement, will be - passed to *file*\ 's read method to control the read buffer - size. + :param sql: the :sql:`COPY` statement to execute. + :param file: a file-like object; must be a readable file for + :sql:`COPY FROM` or an writeable file for :sql:`COPY TO`. + :param size: size of the read buffer to be used in :sql:`COPY FROM`. + + Example: >>> cur.copy_expert("COPY test TO STDOUT WITH CSV HEADER", sys.stdout) id,num,data diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 048ab6d8..717cf9cc 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1213,7 +1213,7 @@ static int _psyco_curs_copy_columns(PyObject *columns, char *columnlist) /* extension: copy_from - implements COPY FROM */ #define psyco_curs_copy_from_doc \ -"copy_from(file, table, sep='\\t', null='\\N', columns=None) -- Copy table from file." +"copy_from(file, table, sep='\\t', null='\\N', size=8192, columns=None) -- Copy table from file." static int _psyco_curs_has_read_check(PyObject* o, void* var) @@ -1454,7 +1454,7 @@ exit: */ #define psyco_curs_copy_expert_doc \ -"copy_expert(sql, file, size=None) -- Submit a user-composed COPY statement.\n" \ +"copy_expert(sql, file, size=8192) -- Submit a user-composed COPY statement.\n" \ "`file` must be an open, readable file for COPY FROM or an open, writeable\n" \ "file for COPY TO. The optional `size` argument, when specified for a COPY\n" \ "FROM statement, will be passed to file's read method to control the read\n" \ From 57a6cf3144d3278a4a7f3a5fc0553975a4013764 Mon Sep 17 00:00:00 2001 From: Steve Lacy Date: Tue, 7 Jun 2011 09:50:52 -0700 Subject: [PATCH 35/47] Code to find an executable on the current PATH refactored --- setup.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ae4d297c..277be61c 100644 --- a/setup.py +++ b/setup.py @@ -116,6 +116,15 @@ or with the pg_config option in 'setup.cfg'. result = result.decode('ascii') return result + def find_on_path(self, exename, path_directories=None): + if not path_directories: + path_directories = os.environ['PATH'].split(os.pathsep) + for dir_name in path_directories: + fullpath = os.path.join(dir_name, exename) + if os.path.isfile(fullpath): + return fullpath + return None + def autodetect_pg_config_path(self): """Find and return the path to the pg_config executable.""" if PLATFORM_IS_WINDOWS: @@ -125,12 +134,7 @@ or with the pg_config option in 'setup.cfg'. def autodetect_pg_config_path_posix(self): """Return pg_config from the current PATH""" - exename = 'pg_config' - for dir_name in os.environ['PATH'].split(os.pathsep): - fullpath = os.path.join(dir_name, exename) - if os.path.isfile(fullpath): - return fullpath - return None + return self.find_on_path('pg_config') def autodetect_pg_config_path_windows(self): """Attempt several different ways of finding the pg_config From 46dc7e66f806c8b36fba1d721a0abcd50bcf8a1d Mon Sep 17 00:00:00 2001 From: Steve Lacy Date: Tue, 7 Jun 2011 10:53:10 -0700 Subject: [PATCH 36/47] Fix pg_config commandline option broken in a previous change - Make sure to declare self.pg_config in initialize_options. - Don't declare PostgresConfig in __init__, as its scope is limited. - Pass build_ext instance to PostgresConfig to avoid having to call the option parser directly. --- setup.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/setup.py b/setup.py index 277be61c..6aa4f68b 100644 --- a/setup.py +++ b/setup.py @@ -81,8 +81,11 @@ PLATFORM_IS_WINDOWS = sys.platform.lower().startswith('win') class PostgresConfig: - def __init__(self): - self.pg_config_exe = self.autodetect_pg_config_path() + def __init__(self, build_ext): + self.build_ext = build_ext + self.pg_config_exe = self.build_ext.pg_config + if not self.pg_config_exe: + self.pg_config_exe = self.autodetect_pg_config_path() if self.pg_config_exe is None: sys.stderr.write("""\ Error: pg_config executable not found. @@ -148,19 +151,17 @@ or with the pg_config option in 'setup.cfg'. # 1) The pg_config utility is not already available on the PATH: if os.popen('pg_config').close() is None: # .close()->None == success return None + # 2) The user has not specified any of the following settings in # setup.cfg: # - pg_config # - include_dirs # - library_dirs - for setting_name in ('pg_config', 'include_dirs', 'library_dirs'): - try: - val = parser.get('build_ext', setting_name) - except configparser.NoOptionError: - pass - else: - if val.strip() != '': - return None + + if (self.build_ext.pg_config + or self.build_ext.include_dirs + or self.build_ext.library_dirs): + return None # end of guard conditions try: @@ -242,7 +243,6 @@ class psycopg_build_ext(build_ext): def __init__(self, *args, **kwargs): build_ext.__init__(self, *args, **kwargs) - self.pg_config = PostgresConfig() compiler_name = self.get_compiler_name().lower() self.compiler_is_msvc = compiler_name.startswith('msvc') self.compiler_is_mingw = compiler_name.startswith('mingw') @@ -255,6 +255,7 @@ class psycopg_build_ext(build_ext): self.use_pydatetime = 1 self.have_ssl = have_ssl self.static_libpq = static_libpq + self.pg_config = None def get_compiler_name(self): """Return the name of the C compiler used to compile extensions. @@ -366,24 +367,26 @@ class psycopg_build_ext(build_ext): """Complete the build system configuation.""" build_ext.finalize_options(self) + pg_config_helper = PostgresConfig(self) + self.include_dirs.append(".") if self.static_libpq: - if not self.link_objects: + if not hasattr(self, 'link_objects'): self.link_objects = [] self.link_objects.append( - os.path.join(self.pg_config.query("libdir"), "libpq.a")) + os.path.join(pg_config_helper.query("libdir"), "libpq.a")) else: self.libraries.append("pq") try: - self.library_dirs.append(self.pg_config.query("libdir")) - self.include_dirs.append(self.pg_config.query("includedir")) - self.include_dirs.append(self.pg_config.query("includedir-server")) + self.library_dirs.append(pg_config_helper.query("libdir")) + self.include_dirs.append(pg_config_helper.query("includedir")) + self.include_dirs.append(pg_config_helper.query("includedir-server")) try: # Here we take a conservative approach: we suppose that # *at least* PostgreSQL 7.4 is available (this is the only # 7.x series supported by psycopg 2) - pgversion = self.pg_config.query("version").split()[1] + pgversion = pg_config_helper.query("version").split()[1] except: pgversion = "7.4.0" From 575afa2e0e0e98de996c8ac6108cd6e15f3b0ff4 Mon Sep 17 00:00:00 2001 From: Steve Lacy Date: Tue, 7 Jun 2011 11:25:59 -0700 Subject: [PATCH 37/47] Properly detect pg_config.exe on Windows. I'm fairly certain this is correct, submitting so I can pull on my Windows box and do some testing there. --- setup.py | 84 ++++++++++++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 48 deletions(-) diff --git a/setup.py b/setup.py index 6aa4f68b..bd05a04a 100644 --- a/setup.py +++ b/setup.py @@ -133,37 +133,22 @@ or with the pg_config option in 'setup.cfg'. if PLATFORM_IS_WINDOWS: return self.autodetect_pg_config_path_windows() else: - return self.autodetect_pg_config_path_posix() - - def autodetect_pg_config_path_posix(self): - """Return pg_config from the current PATH""" - return self.find_on_path('pg_config') + return self.find_on_path('pg_config') def autodetect_pg_config_path_windows(self): """Attempt several different ways of finding the pg_config executable on Windows, and return its full path, if found.""" - # Find the first PostgreSQL installation listed in the registry and - # return the full path to its pg_config utility. - # - # This autodetection is performed *only* if the following conditions - # hold: - # - # 1) The pg_config utility is not already available on the PATH: - if os.popen('pg_config').close() is None: # .close()->None == success - return None - # 2) The user has not specified any of the following settings in - # setup.cfg: - # - pg_config - # - include_dirs - # - library_dirs + # This code only runs if they have not specified a pg_config option + # in the config file or via the commandline. - if (self.build_ext.pg_config - or self.build_ext.include_dirs - or self.build_ext.library_dirs): - return None - # end of guard conditions + # First, check for pg_config.exe on the PATH, and use that if found. + pg_config_exe = self.find_on_path('pg_config.exe') + if pg_config_exe: + return pg_config_exe + # Now, try looking in the Windows Registry to find a PostgreSQL + # installation, and infer the path from that. try: import winreg except ImportError: @@ -180,32 +165,35 @@ or with the pg_config option in 'setup.cfg'. except EnvironmentError: pg_inst_list_key = None - if pg_inst_list_key is not None: - try: - # Determine the name of the first subkey, if any: - try: - first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) - except EnvironmentError: - first_sub_key_name = None + if not pg_inst_list_key: + # No PostgreSQL installation, as best as we can tell. + return None - if first_sub_key_name is not None: - pg_first_inst_key = winreg.OpenKey(reg, - 'SOFTWARE\\PostgreSQL\\Installations\\' - + first_sub_key_name - ) - try: - pg_inst_base_dir = winreg.QueryValueEx( - pg_first_inst_key, 'Base Directory' - )[0] - finally: - winreg.CloseKey(pg_first_inst_key) - finally: - winreg.CloseKey(pg_inst_list_key) + + try: + # Determine the name of the first subkey, if any: + try: + first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) + except EnvironmentError: + first_sub_key_name = None + + if first_sub_key_name is not None: + pg_first_inst_key = winreg.OpenKey(reg, + 'SOFTWARE\\PostgreSQL\\Installations\\' + + first_sub_key_name + ) + try: + pg_inst_base_dir = winreg.QueryValueEx( + pg_first_inst_key, 'Base Directory' + )[0] + finally: + winreg.CloseKey(pg_first_inst_key) + finally: + winreg.CloseKey(pg_inst_list_key) if pg_inst_base_dir and os.path.exists(pg_inst_base_dir): - pg_config_path = os.path.join(pg_inst_base_dir, 'bin', - 'pg_config.exe' - ) + pg_config_path = os.path.join( + pg_inst_base_dir, 'bin', 'pg_config.exe') # Support unicode paths, if this version of Python provides the # necessary infrastructure: if sys.version_info[0] < 3 \ @@ -285,7 +273,7 @@ class psycopg_build_ext(build_ext): build_ext.build_extension(self, extension) # For Python versions that use MSVC compiler 2008, re-insert the - # manifest into the resulting .pyd file. + # manifest into the resulting .pyd file. if self.compiler_is_msvc: platform = get_platform() # Default to the x86 manifest From d0b97feab3101f32a95a558be1f6b02ef0960347 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 22:15:37 +0100 Subject: [PATCH 38/47] More cleanup in pg_config detection from Windows registry --- setup.py | 59 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index bd05a04a..9dee671a 100644 --- a/setup.py +++ b/setup.py @@ -149,57 +149,56 @@ or with the pg_config option in 'setup.cfg'. # Now, try looking in the Windows Registry to find a PostgreSQL # installation, and infer the path from that. + pg_config_exe = self._get_pg_config_from_registry() + if pg_config_exe: + return pg_config_exe + + return None + + def _get_pg_config_from_registry(self): try: import winreg except ImportError: import _winreg as winreg - pg_inst_base_dir = None - pg_config_path = None - reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) try: pg_inst_list_key = winreg.OpenKey(reg, - 'SOFTWARE\\PostgreSQL\\Installations' - ) + 'SOFTWARE\\PostgreSQL\\Installations') except EnvironmentError: - pg_inst_list_key = None - - if not pg_inst_list_key: # No PostgreSQL installation, as best as we can tell. return None - try: # Determine the name of the first subkey, if any: try: first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) except EnvironmentError: - first_sub_key_name = None + return None + + pg_first_inst_key = winreg.OpenKey(reg, + 'SOFTWARE\\PostgreSQL\\Installations\\' + + first_sub_key_name) + try: + pg_inst_base_dir = winreg.QueryValueEx( + pg_first_inst_key, 'Base Directory')[0] + finally: + winreg.CloseKey(pg_first_inst_key) - if first_sub_key_name is not None: - pg_first_inst_key = winreg.OpenKey(reg, - 'SOFTWARE\\PostgreSQL\\Installations\\' - + first_sub_key_name - ) - try: - pg_inst_base_dir = winreg.QueryValueEx( - pg_first_inst_key, 'Base Directory' - )[0] - finally: - winreg.CloseKey(pg_first_inst_key) finally: winreg.CloseKey(pg_inst_list_key) - if pg_inst_base_dir and os.path.exists(pg_inst_base_dir): - pg_config_path = os.path.join( - pg_inst_base_dir, 'bin', 'pg_config.exe') - # Support unicode paths, if this version of Python provides the - # necessary infrastructure: - if sys.version_info[0] < 3 \ - and hasattr(sys, 'getfilesystemencoding'): - pg_config_path = pg_config_path.encode( - sys.getfilesystemencoding()) + pg_config_path = os.path.join( + pg_inst_base_dir, 'bin', 'pg_config.exe') + if not os.path.exists(pg_config_path): + return None + + # Support unicode paths, if this version of Python provides the + # necessary infrastructure: + if sys.version_info[0] < 3 \ + and hasattr(sys, 'getfilesystemencoding'): + pg_config_path = pg_config_path.encode( + sys.getfilesystemencoding()) return pg_config_path From dc92161dda754d0bf8a5017c5ab3728e9981a891 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 23:28:17 +0100 Subject: [PATCH 39/47] Delay detection of the compiler in setup.py At init time, build_ext is not configured, so neither the --compiler option nor settings in setup.cfg/distutil.cfg is effective. Steve, I told you distutils was a mess :) --- setup.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 9dee671a..4840e585 100644 --- a/setup.py +++ b/setup.py @@ -230,9 +230,6 @@ class psycopg_build_ext(build_ext): def __init__(self, *args, **kwargs): build_ext.__init__(self, *args, **kwargs) - compiler_name = self.get_compiler_name().lower() - self.compiler_is_msvc = compiler_name.startswith('msvc') - self.compiler_is_mingw = compiler_name.startswith('mingw') def initialize_options(self): build_ext.initialize_options(self) @@ -244,6 +241,12 @@ class psycopg_build_ext(build_ext): self.static_libpq = static_libpq self.pg_config = None + def compiler_is_msvc(self): + return self.get_compiler_name().lower().startswith('msvc') + + def compiler_is_mingw(self): + return self.get_compiler_name().lower().startswith('mingw') + def get_compiler_name(self): """Return the name of the C compiler used to compile extensions. @@ -263,7 +266,7 @@ class psycopg_build_ext(build_ext): def get_export_symbols(self, extension): # Fix MSVC seeing two of the same export symbols. - if self.compiler_is_msvc: + if self.compiler_is_msvc(): return [] else: return build_ext.get_export_symbols(self, extension) @@ -273,7 +276,7 @@ class psycopg_build_ext(build_ext): # For Python versions that use MSVC compiler 2008, re-insert the # manifest into the resulting .pyd file. - if self.compiler_is_msvc: + if self.compiler_is_msvc(): platform = get_platform() # Default to the x86 manifest manifest = '_psycopg.vc9.x86.manifest' @@ -293,7 +296,7 @@ class psycopg_build_ext(build_ext): # Add compiler-specific arguments: extra_compiler_args = [] - if self.compiler_is_mingw: + if self.compiler_is_mingw(): # Default MinGW compilation of Python extensions on Windows uses # only -O: extra_compiler_args.append('-O3') @@ -321,7 +324,7 @@ class psycopg_build_ext(build_ext): self.libraries.append("ws2_32") self.libraries.append("advapi32") - if self.compiler_is_msvc: + if self.compiler_is_msvc(): # MSVC requires an explicit "libpq" self.libraries.remove("pq") self.libraries.append("secur32") From 7b017e794455ea8953b3aa2c87998a2def0bb0d7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 7 Jun 2011 23:58:37 +0100 Subject: [PATCH 40/47] Mention Steve and his work in the NEWS file That's Steve's Job! :D --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 7f0070cd..09669107 100644 --- a/NEWS +++ b/NEWS @@ -18,6 +18,8 @@ What's new in psycopg 2.4.2 - Trying to execute concurrent operations on the same connection through concurrent green thread results in an error instead of a deadlock. + - Fixed detection of pg_config on Window. Report and fix, plus some + long needed setup.py cleanup by Steve Lacy: thanks! What's new in psycopg 2.4.1 From 5ee7bac66ba79de7446e1a5c083dff76078dd37f Mon Sep 17 00:00:00 2001 From: Jason Erickson Date: Tue, 7 Jun 2011 21:50:53 -0600 Subject: [PATCH 41/47] No manifest reinsertion into 2.4/2.5 with MSVC Python versions 2.4 and 2.5 for MSVC on Windows do not need to manifest file reinserted into the DLL. The VC compiler for these versions does not have the mt.exe executable to insert the manifest file. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4840e585..df1f6fed 100644 --- a/setup.py +++ b/setup.py @@ -273,10 +273,11 @@ class psycopg_build_ext(build_ext): def build_extension(self, extension): build_ext.build_extension(self, extension) + sysVer = sys.version_info[:2] # For Python versions that use MSVC compiler 2008, re-insert the # manifest into the resulting .pyd file. - if self.compiler_is_msvc(): + if self.compiler_is_msvc() and sysVer not in ((2, 4), (2, 5)): platform = get_platform() # Default to the x86 manifest manifest = '_psycopg.vc9.x86.manifest' From 0a1bbb56cd4cb6d57d66935268032be839b9b2e7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 8 Jun 2011 09:22:35 +0100 Subject: [PATCH 42/47] Dropped redundant semicolons at the end of internal queries For consistency with other queries, and probably we give less work to do to the server parser (a ridiculously tiny amount). --- psycopg/pqpath.c | 12 ++++++------ psycopg/xid_type.c | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 84110f6f..cee5ce4c 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -416,7 +416,7 @@ pq_begin_locked(connectionObject *conn, PGresult **pgres, char **error, return 0; } - result = pq_execute_command_locked(conn, "BEGIN;", pgres, error, tstate); + result = pq_execute_command_locked(conn, "BEGIN", pgres, error, tstate); if (result == 0) conn->status = CONN_STATUS_BEGIN; @@ -616,7 +616,7 @@ pq_get_guc_locked( Dprintf("pq_get_guc_locked: reading %s", param); - size = PyOS_snprintf(query, sizeof(query), "SHOW %s;", param); + size = PyOS_snprintf(query, sizeof(query), "SHOW %s", param); if (size >= sizeof(query)) { *error = strdup("SHOW: query too large"); goto cleanup; @@ -674,11 +674,11 @@ pq_set_guc_locked( if (0 == strcmp(value, "default")) { size = PyOS_snprintf(query, sizeof(query), - "SET %s TO DEFAULT;", param); + "SET %s TO DEFAULT", param); } else { size = PyOS_snprintf(query, sizeof(query), - "SET %s TO '%s';", param, value); + "SET %s TO '%s'", param, value); } if (size >= sizeof(query)) { *error = strdup("SET: query too large"); @@ -714,12 +714,12 @@ pq_tpc_command_locked(connectionObject *conn, const char *cmd, const char *tid, { goto exit; } /* prepare the command to the server */ - buflen = 3 + strlen(cmd) + strlen(etid); /* add space, semicolon, zero */ + buflen = 2 + strlen(cmd) + strlen(etid); /* add space, zero */ if (!(buf = PyMem_Malloc(buflen))) { PyErr_NoMemory(); goto exit; } - if (0 > PyOS_snprintf(buf, buflen, "%s %s;", cmd, etid)) { goto exit; } + if (0 > PyOS_snprintf(buf, buflen, "%s %s", cmd, etid)) { goto exit; } /* run the command and let it handle the error cases */ *tstate = PyEval_SaveThread(); diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index 9e95fd1b..4de46b44 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -663,7 +663,7 @@ xid_recover(PyObject *conn) /* curs.execute(...) */ if (!(tmp = PyObject_CallMethod(curs, "execute", "s", - "SELECT gid, prepared, owner, database FROM pg_prepared_xacts;"))) + "SELECT gid, prepared, owner, database FROM pg_prepared_xacts"))) { goto exit; } From 1a51cfe2743e4355bd43d20e1a8315d574a16427 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 8 Jun 2011 10:59:27 +0100 Subject: [PATCH 43/47] Better error message if deferrable is used in PG < 9.1 --- psycopg/connection_type.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index f3a17f93..13c017e8 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -499,6 +499,12 @@ psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwa } } if (Py_None != deferrable) { + if (self->server_version < 90100) { + PyErr_SetString(ProgrammingError, + "the 'deferrable' setting is only available" + " from PostgreSQL 9.1"); + return NULL; + } if (!(c_deferrable = _psyco_conn_parse_onoff(deferrable))) { return NULL; } From d2b28abcede7788f583ff88571d2a418cec1a4df Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 8 Jun 2011 14:22:11 +0100 Subject: [PATCH 44/47] Method set_transaction() renamed to set_session() In fact it doesn't change "the transaction", as there has to be no transaction when invoked. The effect instead is to execute SET SESSION CHARACTERISTICS. --- NEWS | 2 +- doc/src/advanced.rst | 2 +- doc/src/connection.rst | 4 ++-- psycopg/connection.h | 2 +- psycopg/connection_int.c | 10 ++++---- psycopg/connection_type.c | 18 +++++++-------- tests/test_connection.py | 48 +++++++++++++++++++-------------------- 7 files changed, 43 insertions(+), 43 deletions(-) diff --git a/NEWS b/NEWS index 09669107..dd595968 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,7 @@ What's new in psycopg 2.4.2 --------------------------- - - Added 'set_transaction()' method and 'autocommit' property to the + - Added 'set_session()' method and 'autocommit' property to the connection. Added support for read-only sessions and, for PostgreSQL 9.1, for the "repeatable read" isolation level and the "deferrable" transaction property. diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index c7625b19..c2c3a156 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -378,7 +378,7 @@ transaction is not implicitly started at the first query and is not possible to use methods `~connection.commit()` and `~connection.rollback()`: you can manually control transactions using `~cursor.execute()` to send database commands such as :sql:`BEGIN`, :sql:`COMMIT` and :sql:`ROLLBACK`. Similarly -`set_transaction()` can't be used but it is still possible to invoke the +`~connection.set_session()` can't be used but it is still possible to invoke the :sql:`SET` command with the proper :sql:`default_transaction_...` parameter. With asynchronous connections it is also not possible to use diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 970b0371..a38a560a 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -327,7 +327,7 @@ The ``connection`` class pair: Transaction; Autocommit pair: Transaction; Isolation level - .. method:: set_transaction([isolation_level,] [readonly,] [deferrable,] [autocommit]) + .. method:: set_session([isolation_level,] [readonly,] [deferrable,] [autocommit]) Set one or more parameters for the next transactions or statements in the current session. See |SET TRANSACTION|_ for further details. @@ -411,7 +411,7 @@ The ``connection`` class .. note:: - From version 2.4.2, `set_transaction()` and `autocommit`, offer + From version 2.4.2, `set_session()` and `autocommit`, offer finer control on the transaction characteristics. Read or set the `transaction isolation level`_ for the current session. diff --git a/psycopg/connection.h b/psycopg/connection.h index 14d58a09..7f512a18 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -136,7 +136,7 @@ HIDDEN int conn_connect(connectionObject *self, long int async); HIDDEN void conn_close(connectionObject *self); HIDDEN int conn_commit(connectionObject *self); HIDDEN int conn_rollback(connectionObject *self); -HIDDEN int conn_set_transaction(connectionObject *self, const char *isolevel, +HIDDEN int conn_set_session(connectionObject *self, const char *isolevel, const char *readonly, const char *deferrable, int autocommit); HIDDEN int conn_set_autocommit(connectionObject *self, int value); diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 732b22ee..441d3629 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -959,7 +959,7 @@ conn_rollback(connectionObject *self) } int -conn_set_transaction(connectionObject *self, +conn_set_session(connectionObject *self, const char *isolevel, const char *readonly, const char *deferrable, int autocommit) { @@ -971,7 +971,7 @@ conn_set_transaction(connectionObject *self, pthread_mutex_lock(&self->lock); if (isolevel) { - Dprintf("conn_set_transaction: setting isolation to %s", isolevel); + Dprintf("conn_set_session: setting isolation to %s", isolevel); if ((res = pq_set_guc_locked(self, "default_transaction_isolation", isolevel, &pgres, &error, &_save))) { @@ -980,7 +980,7 @@ conn_set_transaction(connectionObject *self, } if (readonly) { - Dprintf("conn_set_transaction: setting read only to %s", readonly); + Dprintf("conn_set_session: setting read only to %s", readonly); if ((res = pq_set_guc_locked(self, "default_transaction_read_only", readonly, &pgres, &error, &_save))) { @@ -989,7 +989,7 @@ conn_set_transaction(connectionObject *self, } if (deferrable) { - Dprintf("conn_set_transaction: setting deferrable to %s", deferrable); + Dprintf("conn_set_session: setting deferrable to %s", deferrable); if ((res = pq_set_guc_locked(self, "default_transaction_deferrable", deferrable, &pgres, &error, &_save))) { @@ -998,7 +998,7 @@ conn_set_transaction(connectionObject *self, } if (self->autocommit != autocommit) { - Dprintf("conn_set_transaction: setting autocommit to %d", autocommit); + Dprintf("conn_set_session: setting autocommit to %d", autocommit); self->autocommit = autocommit; } diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 13c017e8..e456ce43 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -456,14 +456,14 @@ _psyco_conn_parse_onoff(PyObject *pyval) } } -/* set_transaction - default transaction characteristics */ +/* set_session - set default transaction characteristics */ -#define psyco_conn_set_transaction_doc \ -"set_transaction(...) -- Set one or more parameters for the next transactions.\n\n" \ +#define psyco_conn_set_session_doc \ +"set_session(...) -- Set one or more parameters for the next transactions.\n\n" \ "Accepted arguments are 'isolation_level', 'readonly', 'deferrable', 'autocommit'." static PyObject * -psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwargs) +psyco_conn_set_session(connectionObject *self, PyObject *args, PyObject *kwargs) { PyObject *isolevel = Py_None; PyObject *readonly = Py_None; @@ -479,8 +479,8 @@ psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwa {"isolation_level", "readonly", "deferrable", "autocommit", NULL}; EXC_IF_CONN_CLOSED(self); - EXC_IF_CONN_ASYNC(self, set_transaction); - EXC_IF_IN_TRANSACTION(self, set_transaction); + EXC_IF_CONN_ASYNC(self, set_session); + EXC_IF_IN_TRANSACTION(self, set_session); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|OOOO", kwlist, &isolevel, &readonly, &deferrable, &autocommit)) { @@ -514,7 +514,7 @@ psyco_conn_set_transaction(connectionObject *self, PyObject *args, PyObject *kwa if (-1 == c_autocommit) { return NULL; } } - if (0 != conn_set_transaction(self, + if (0 != conn_set_session(self, c_isolevel, c_readonly, c_deferrable, c_autocommit)) { return NULL; } @@ -904,8 +904,8 @@ static struct PyMethodDef connectionObject_methods[] = { {"tpc_recover", (PyCFunction)psyco_conn_tpc_recover, METH_NOARGS, psyco_conn_tpc_recover_doc}, #ifdef PSYCOPG_EXTENSIONS - {"set_transaction", (PyCFunction)psyco_conn_set_transaction, - METH_VARARGS|METH_KEYWORDS, psyco_conn_set_transaction_doc}, + {"set_session", (PyCFunction)psyco_conn_set_session, + METH_VARARGS|METH_KEYWORDS, psyco_conn_set_session_doc}, {"set_isolation_level", (PyCFunction)psyco_conn_set_isolation_level, METH_VARARGS, psyco_conn_set_isolation_level_doc}, {"set_client_encoding", (PyCFunction)psyco_conn_set_client_encoding, diff --git a/tests/test_connection.py b/tests/test_connection.py index dc3cfb00..ce0bf7e5 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -729,18 +729,18 @@ class TransactionControlTests(unittest.TestCase): cur = self.conn.cursor() cur.execute("select 1") self.assertRaises(psycopg2.ProgrammingError, - self.conn.set_transaction, + self.conn.set_session, psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) def test_set_isolation_level(self): cur = self.conn.cursor() - self.conn.set_transaction( + self.conn.set_session( psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) cur.execute("SHOW default_transaction_isolation;") self.assertEqual(cur.fetchone()[0], 'serializable') self.conn.rollback() - self.conn.set_transaction( + self.conn.set_session( psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ) cur.execute("SHOW default_transaction_isolation;") if self.conn.server_version > 80000: @@ -749,13 +749,13 @@ class TransactionControlTests(unittest.TestCase): self.assertEqual(cur.fetchone()[0], 'serializable') self.conn.rollback() - self.conn.set_transaction( + self.conn.set_session( isolation_level=psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) cur.execute("SHOW default_transaction_isolation;") self.assertEqual(cur.fetchone()[0], 'read committed') self.conn.rollback() - self.conn.set_transaction( + self.conn.set_session( isolation_level=psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED) cur.execute("SHOW default_transaction_isolation;") if self.conn.server_version > 80000: @@ -766,12 +766,12 @@ class TransactionControlTests(unittest.TestCase): def test_set_isolation_level_str(self): cur = self.conn.cursor() - self.conn.set_transaction("serializable") + self.conn.set_session("serializable") cur.execute("SHOW default_transaction_isolation;") self.assertEqual(cur.fetchone()[0], 'serializable') self.conn.rollback() - self.conn.set_transaction("repeatable read") + self.conn.set_session("repeatable read") cur.execute("SHOW default_transaction_isolation;") if self.conn.server_version > 80000: self.assertEqual(cur.fetchone()[0], 'repeatable read') @@ -779,12 +779,12 @@ class TransactionControlTests(unittest.TestCase): self.assertEqual(cur.fetchone()[0], 'serializable') self.conn.rollback() - self.conn.set_transaction("read committed") + self.conn.set_session("read committed") cur.execute("SHOW default_transaction_isolation;") self.assertEqual(cur.fetchone()[0], 'read committed') self.conn.rollback() - self.conn.set_transaction("read uncommitted") + self.conn.set_session("read uncommitted") cur.execute("SHOW default_transaction_isolation;") if self.conn.server_version > 80000: self.assertEqual(cur.fetchone()[0], 'read uncommitted') @@ -793,13 +793,13 @@ class TransactionControlTests(unittest.TestCase): self.conn.rollback() def test_bad_isolation_level(self): - self.assertRaises(ValueError, self.conn.set_transaction, 0) - self.assertRaises(ValueError, self.conn.set_transaction, 5) - self.assertRaises(ValueError, self.conn.set_transaction, 'whatever') + self.assertRaises(ValueError, self.conn.set_session, 0) + self.assertRaises(ValueError, self.conn.set_session, 5) + self.assertRaises(ValueError, self.conn.set_session, 'whatever') def test_set_read_only(self): cur = self.conn.cursor() - self.conn.set_transaction(readonly=True) + self.conn.set_session(readonly=True) cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'on') self.conn.rollback() @@ -808,12 +808,12 @@ class TransactionControlTests(unittest.TestCase): self.conn.rollback() cur = self.conn.cursor() - self.conn.set_transaction(readonly=None) + self.conn.set_session(readonly=None) cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'on') self.conn.rollback() - self.conn.set_transaction(readonly=False) + self.conn.set_session(readonly=False) cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'off') self.conn.rollback() @@ -826,8 +826,8 @@ class TransactionControlTests(unittest.TestCase): default_readonly = cur.fetchone()[0] self.conn.rollback() - self.conn.set_transaction(isolation_level='serializable', readonly=True) - self.conn.set_transaction(isolation_level='default', readonly='default') + self.conn.set_session(isolation_level='serializable', readonly=True) + self.conn.set_session(isolation_level='default', readonly='default') cur.execute("SHOW default_transaction_isolation;") self.assertEqual(cur.fetchone()[0], default_isolevel) @@ -837,7 +837,7 @@ class TransactionControlTests(unittest.TestCase): @skip_before_postgres(9, 1) def test_set_deferrable(self): cur = self.conn.cursor() - self.conn.set_transaction(readonly=True, deferrable=True) + self.conn.set_session(readonly=True, deferrable=True) cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'on') cur.execute("SHOW default_transaction_deferrable;") @@ -847,7 +847,7 @@ class TransactionControlTests(unittest.TestCase): self.assertEqual(cur.fetchone()[0], 'on') self.conn.rollback() - self.conn.set_transaction(deferrable=False) + self.conn.set_session(deferrable=False) cur.execute("SHOW default_transaction_read_only;") self.assertEqual(cur.fetchone()[0], 'on') cur.execute("SHOW default_transaction_deferrable;") @@ -857,7 +857,7 @@ class TransactionControlTests(unittest.TestCase): @skip_after_postgres(9, 1) def test_set_deferrable_error(self): self.assertRaises(psycopg2.ProgrammingError, - self.conn.set_transaction, readonly=True, deferrable=True) + self.conn.set_session, readonly=True, deferrable=True) class AutocommitTests(unittest.TestCase): @@ -915,8 +915,8 @@ class AutocommitTests(unittest.TestCase): self.assertRaises(psycopg2.ProgrammingError, setattr, self.conn, 'autocommit', True) - def test_set_transaction_autocommit(self): - self.conn.set_transaction(autocommit=True) + def test_set_session_autocommit(self): + self.conn.set_session(autocommit=True) self.assert_(self.conn.autocommit) self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) self.assertEqual(self.conn.get_transaction_status(), @@ -928,7 +928,7 @@ class AutocommitTests(unittest.TestCase): self.assertEqual(self.conn.get_transaction_status(), psycopg2.extensions.TRANSACTION_STATUS_IDLE) - self.conn.set_transaction(autocommit=False) + self.conn.set_session(autocommit=False) self.assert_(not self.conn.autocommit) self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) self.assertEqual(self.conn.get_transaction_status(), @@ -940,7 +940,7 @@ class AutocommitTests(unittest.TestCase): psycopg2.extensions.TRANSACTION_STATUS_INTRANS) self.conn.rollback() - self.conn.set_transaction('serializable', readonly=True, autocommit=True) + self.conn.set_session('serializable', readonly=True, autocommit=True) self.assert_(self.conn.autocommit) cur.execute('select 1;') self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) From d76d136b4f3bb2a88a64ad83e8f26fb2b486a604 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 8 Jun 2011 14:38:57 +0100 Subject: [PATCH 45/47] Introductory docs section on transaction control improved Added big fat warning about idle in transaction and reference to set_session(). --- doc/src/connection.rst | 2 +- doc/src/usage.rst | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index a38a560a..2763e066 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -396,7 +396,7 @@ The ``connection`` class .. warning:: By default, any query execution, including a simple :sql:`SELECT` - will start a transaction: for long-running program, if no further + will start a transaction: for long-running programs, if no further action is taken, the session will remain "idle in transaction", a condition non desiderable for several reasons (locks are held by the session, tables bloat...). For long lived scripts, either diff --git a/doc/src/usage.rst b/doc/src/usage.rst index efbd1587..5f6c5b1c 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -489,7 +489,7 @@ rounded to the nearest minute, with an error of up to 30 seconds. versions use `psycopg2.extras.register_tstz_w_secs()`. -.. index:: Transaction, Begin, Commit, Rollback, Autocommit +.. index:: Transaction, Begin, Commit, Rollback, Autocommit, Read only .. _transactions-control: @@ -503,7 +503,7 @@ The following database commands will be executed in the context of the same transaction -- not only the commands issued by the first cursor, but the ones issued by all the cursors created by the same connection. Should any command fail, the transaction will be aborted and no further command will be executed -until a call to the `connection.rollback()` method. +until a call to the `~connection.rollback()` method. The connection is responsible to terminate its transaction, calling either the `~connection.commit()` or `~connection.rollback()` method. Committed @@ -516,13 +516,23 @@ It is possible to set the connection in *autocommit* mode: this way all the commands executed will be immediately committed and no rollback is possible. A few commands (e.g. :sql:`CREATE DATABASE`, :sql:`VACUUM`...) require to be run outside any transaction: in order to be able to run these commands from -Psycopg, the session must be in autocommit mode. Read the documentation for -`connection.set_isolation_level()` to know how to change the commit mode. +Psycopg, the session must be in autocommit mode: you can use the +`~connection.autocommit` property (`~connection.set_isolation_level()` in +older versions). -.. note:: +.. warning:: - From version 2.4.2 you can use the `~connection.autocommit` property to - switch a connection in autocommit mode. + By default even a simple :sql:`SELECT` will start a transaction: in + long-running programs, if no further action is taken, the session will + remain "idle in transaction", a condition non desiderable for several + reasons (locks are held by the session, tables bloat...). For long lived + scripts, either ensure to terminate a transaction as soon as possible or + use an autocommit connection. + +A few other transaction properties can be set session-wide by the +`!connection`: for instance it is possible to have read-only transactions or +change the isolation level. See the `~connection.set_session()` method for all +the details. .. index:: From 3ec9677978a77b9d4ea7e218479e5a0586658281 Mon Sep 17 00:00:00 2001 From: Federico Di Gregorio Date: Sun, 12 Jun 2011 21:40:31 +0200 Subject: [PATCH 46/47] Aligned casing of isolation levels with PostgreSQL documentation --- doc/src/connection.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 2763e066..f9a1b868 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -339,8 +339,8 @@ The ``connection`` class transactions/statements. The value can be one of the :ref:`constants ` defined in the `~psycopg2.extensions` module or one of the literal values - ``read uncommitted``, ``read committed``, ``repeatable read``, - ``serializable``. + ``READ UNCOMMITTED``, ``READ COMMITTED``, ``REPEATABLE READ``, + ``SERIALIZABLE``. :param readonly: if `!True`, set the connection to read only; read/write if `!False`. :param deferrable: if `!True`, set the connection to deferrable; @@ -350,7 +350,7 @@ The ``connection`` class `autocommit` attribute. The parameters *isolation_level*, *readonly* and *deferrable* also - accept the string ``default`` as a value: the effect is to reset the + accept the string ``DEFAULT`` as a value: the effect is to reset the parameter to the server default. .. _isolation level: From f8a5dabdc11364237003f76adadffcd239521138 Mon Sep 17 00:00:00 2001 From: Federico Di Gregorio Date: Sun, 12 Jun 2011 21:40:44 +0200 Subject: [PATCH 47/47] Preparing release 2.4.2 --- ZPsycopgDA/DA.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ZPsycopgDA/DA.py b/ZPsycopgDA/DA.py index 7a681e42..f5154371 100644 --- a/ZPsycopgDA/DA.py +++ b/ZPsycopgDA/DA.py @@ -16,7 +16,7 @@ # their work without bothering about the module dependencies. -ALLOWED_PSYCOPG_VERSIONS = ('2.4-beta1', '2.4-beta2', '2.4', '2.4.1') +ALLOWED_PSYCOPG_VERSIONS = ('2.4-beta1', '2.4-beta2', '2.4', '2.4.1', '2.4.2') import sys import time diff --git a/setup.py b/setup.py index df1f6fed..e03876a8 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ except ImportError: # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. -PSYCOPG_VERSION = '2.4.2.dev0' +PSYCOPG_VERSION = '2.4.2' version_flags = ['dt', 'dec']