From 23d279945ffb3eae3bea69856b20e8271108ebd0 Mon Sep 17 00:00:00 2001 From: mrmilosz Date: Sat, 24 May 2014 23:47:09 -0400 Subject: [PATCH 01/18] cursor.callproc now also accepts dict for PostgreSQL 9+ "named notation" --- psycopg/cursor_type.c | 56 ++++++++++++++++++++++++++++++++++--------- psycopg/python.h | 2 ++ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index cd8d5ca3..7992ce4b 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1024,6 +1024,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) PyObject *parameters = Py_None; PyObject *operation = NULL; PyObject *res = NULL; + PyObject *parameter_names = NULL; if (!PyArg_ParseTuple(args, "s#|O", &procname, &procname_len, ¶meters @@ -1045,19 +1046,52 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) } /* allocate some memory, build the SQL and create a PyString from it */ - sl = procname_len + 17 + nparameters*3 - (nparameters ? 1 : 0); - sql = (char*)PyMem_Malloc(sl); - if (sql == NULL) { - PyErr_NoMemory(); - goto exit; - } - sprintf(sql, "SELECT * FROM %s(", procname); - for(i=0; i 0 && PyDict_Check(parameters)) { + /* for a dict, we put the parameter names into the SQL */ + parameter_names = PyDict_Keys(parameters); + + /* first we need to figure out how much space we need for the SQL */ + sl = procname_len + 17 + nparameters*5 - (nparameters ? 1 : 0); + for(i=0; i 2 From 1205bf9c2b123a0fbc81352af330ac2344251c62 Mon Sep 17 00:00:00 2001 From: mrmilosz Date: Sun, 25 May 2014 04:03:07 -0400 Subject: [PATCH 02/18] callproc using a dict now has a type check to make sure the keys are strings. --- psycopg/cursor_type.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 7992ce4b..16339971 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1024,6 +1024,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) PyObject *parameters = Py_None; PyObject *operation = NULL; PyObject *res = NULL; + PyObject *parameter_name = NULL; PyObject *parameter_names = NULL; if (!PyArg_ParseTuple(args, "s#|O", @@ -1054,7 +1055,12 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* first we need to figure out how much space we need for the SQL */ sl = procname_len + 17 + nparameters*5 - (nparameters ? 1 : 0); for(i=0; i Date: Wed, 28 May 2014 01:05:17 -0400 Subject: [PATCH 03/18] callproc using a dict now uses connection encoding and sanitizes parameter names --- psycopg/cursor_type.c | 57 ++++++++++++++++++++++++++++++++++--------- psycopg/python.h | 2 -- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 16339971..113e0a54 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1025,6 +1025,10 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) PyObject *operation = NULL; PyObject *res = NULL; PyObject *parameter_name = NULL; + PyObject *parameter_name_bytes = NULL; + char *parameter_name_cstr = NULL; + char *parameter_name_cstr_sanitized = NULL; + char **parameter_name_cstr_sanitized_CACHE = NULL; PyObject *parameter_names = NULL; if (!PyArg_ParseTuple(args, "s#|O", @@ -1048,40 +1052,71 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* allocate some memory, build the SQL and create a PyString from it */ + /* a dict requires special handling: we put the parameter names into the SQL */ if (nparameters > 0 && PyDict_Check(parameters)) { - /* for a dict, we put the parameter names into the SQL */ + parameter_names = PyDict_Keys(parameters); - /* first we need to figure out how much space we need for the SQL */ - sl = procname_len + 17 + nparameters*5 - (nparameters ? 1 : 0); + /* first we need to ensure the dict's keys are text */ for(i=0; iconn->pgconn, parameter_name_cstr, strlen(parameter_name_cstr)); + Py_DECREF(parameter_name); + + /* must add the length of the sanitized string to the length of the SQL string */ + sl += strlen(parameter_name_cstr_sanitized); + + parameter_name_cstr_sanitized_CACHE[i] = parameter_name_cstr_sanitized; + } + + Py_DECREF(parameter_names); + sql = (char*)PyMem_Malloc(sl); if (sql == NULL) { - PyErr_NoMemory(); - goto exit; + PyErr_NoMemory(); + for(i=0; i 2 From e9bb4a86f93836f8f6c1233c45507e58c159ef1f Mon Sep 17 00:00:00 2001 From: mrmilosz Date: Wed, 28 May 2014 01:27:56 -0400 Subject: [PATCH 04/18] cursor.callproc: added a missing memory check --- psycopg/cursor_type.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 113e0a54..5f70ce61 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1072,6 +1072,13 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* we will throw the sanitized C strings into a cache to not redo the work later */ parameter_name_cstr_sanitized_CACHE = PyMem_New(char *, nparameters); + if (parameter_name_cstr_sanitized_CACHE == NULL) { + PyErr_NoMemory(); + PyMem_Del(parameter_name_cstr_sanitized_CACHE); + Py_DECREF(parameter_names); + goto exit; + } + for(i=0; i Date: Thu, 29 May 2014 04:31:43 -0400 Subject: [PATCH 05/18] callproc: now more compliant with local coding standards. --- psycopg/cursor_type.c | 192 ++++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 90 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 5f70ce61..d6875a79 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1024,17 +1024,18 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) PyObject *parameters = Py_None; PyObject *operation = NULL; PyObject *res = NULL; - PyObject *parameter_name = NULL; - PyObject *parameter_name_bytes = NULL; - char *parameter_name_cstr = NULL; - char *parameter_name_cstr_sanitized = NULL; - char **parameter_name_cstr_sanitized_CACHE = NULL; - PyObject *parameter_names = NULL; - if (!PyArg_ParseTuple(args, "s#|O", - &procname, &procname_len, ¶meters - )) - { goto exit; } + int using_dict; + PyObject *pname = NULL; + PyObject *bpname = NULL; + PyObject *pnames = NULL; + char *cpname = NULL; + char **scpnames = NULL; + + if (!PyArg_ParseTuple(args, "s#|O", &procname, &procname_len, + ¶meters)) { + goto exit; + } EXC_IF_CURS_CLOSED(self); EXC_IF_ASYNC_IN_PROGRESS(self, callproc); @@ -1042,7 +1043,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (self->name != NULL) { psyco_set_error(ProgrammingError, self, - "can't call .callproc() on named cursors"); + "can't call .callproc() on named cursors"); goto exit; } @@ -1050,106 +1051,117 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (-1 == (nparameters = PyObject_Length(parameters))) { goto exit; } } - /* allocate some memory, build the SQL and create a PyString from it */ + using_dict = nparameters > 0 && PyDict_Check(parameters); - /* a dict requires special handling: we put the parameter names into the SQL */ - if (nparameters > 0 && PyDict_Check(parameters)) { - - parameter_names = PyDict_Keys(parameters); - - /* first we need to ensure the dict's keys are text */ - for(i=0; iconn->pgconn, parameter_name_cstr, strlen(parameter_name_cstr)); - Py_DECREF(parameter_name); - - /* must add the length of the sanitized string to the length of the SQL string */ - sl += strlen(parameter_name_cstr_sanitized); - - parameter_name_cstr_sanitized_CACHE[i] = parameter_name_cstr_sanitized; - } - - Py_DECREF(parameter_names); - - sql = (char*)PyMem_Malloc(sl); - if (sql == NULL) { - PyErr_NoMemory(); - for(i=0; iconn->pgconn, cpname, + strlen(cpname)))) { + PyErr_SetString(PyExc_RuntimeError, + "libpq failed to escape identifier!"); + goto exit; + } + + sl += strlen(scpnames[i]); + } + + sql = (char*)PyMem_Malloc(sl); + if (sql == NULL) { + PyErr_NoMemory(); + goto exit; + } + + sprintf(sql, "SELECT * FROM %s(", procname); + for (i = 0; i < nparameters; i++) { + strcat(sql, scpnames[i]); + strcat(sql, ":=%s,"); + } + sql[sl-2] = ')'; + sql[sl-1] = '\0'; + + if (!(parameters = PyDict_Values(parameters))) { + PyErr_SetString(PyExc_RuntimeError, + "built-in 'values' failed on a Dict!"); + goto exit; + } } - /* a list (or None) is a little bit simpler */ + /* a list (or None, or empty data structure) is a little bit simpler */ else { - sl = procname_len + 17 + nparameters*3 - (nparameters ? 1 : 0); + sl = procname_len + 17 + nparameters * 3 - (nparameters ? 1 : 0); - sql = (char*)PyMem_Malloc(sl); - if (sql == NULL) { - PyErr_NoMemory(); - goto exit; - } + sql = (char*)PyMem_Malloc(sl); + if (sql == NULL) { + PyErr_NoMemory(); + goto exit; + } - sprintf(sql, "SELECT * FROM %s(", procname); - for(i=0; iconn->async, 0)) { - Py_INCREF(parameters); + self->conn->async, 0)) { + /* In the dict case, the parameters are already a new reference */ + if (!using_dict) { + Py_INCREF(parameters); + } res = parameters; } exit: + if (scpnames != NULL) { + for (i = 0; i < nparameters; i++) { + if (scpnames[i] != NULL) { + PQfreemem(scpnames[i]); + } + } + } + PyMem_Del(scpnames); + Py_XDECREF(pnames); Py_XDECREF(operation); PyMem_Free((void*)sql); return res; From 37a80e9de8d6df13d9e43e4106b62cf74308c5d9 Mon Sep 17 00:00:00 2001 From: mrmilosz Date: Thu, 29 May 2014 05:09:09 -0400 Subject: [PATCH 06/18] callproc: checking for libpq 9.0+ on compile. yes: use PQescapeIdentifier. no: error --- psycopg/cursor_type.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index d6875a79..3dbdbdc4 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1026,11 +1026,13 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) PyObject *res = NULL; int using_dict; +#if PG_VERSION_HEX >= 0x090000 PyObject *pname = NULL; PyObject *bpname = NULL; PyObject *pnames = NULL; char *cpname = NULL; char **scpnames = NULL; +#endif if (!PyArg_ParseTuple(args, "s#|O", &procname, &procname_len, ¶meters)) { @@ -1055,6 +1057,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* A Dict is complicated. The parameter names go into the query */ if (using_dict) { +#if PG_VERSION_HEX >= 0x090000 if (!(pnames = PyDict_Keys(parameters))) { PyErr_SetString(PyExc_RuntimeError, "built-in 'keys' failed on a Dict!"); @@ -1119,6 +1122,11 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) "built-in 'values' failed on a Dict!"); goto exit; } +#else + PyErr_SetString(PyExc_NotImplementedError, + "named parameters require psycopg2 compiled against libpq 9.0+"); + goto exit; +#endif } /* a list (or None, or empty data structure) is a little bit simpler */ @@ -1153,6 +1161,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) } exit: +#if PG_VERSION_HEX >= 0x090000 if (scpnames != NULL) { for (i = 0; i < nparameters; i++) { if (scpnames[i] != NULL) { @@ -1162,6 +1171,7 @@ exit: } PyMem_Del(scpnames); Py_XDECREF(pnames); +#endif Py_XDECREF(operation); PyMem_Free((void*)sql); return res; From c205f140a00f030bb2b7da943aba02653454d633 Mon Sep 17 00:00:00 2001 From: mrmilosz Date: Fri, 30 May 2014 20:34:19 -0400 Subject: [PATCH 07/18] callproc: tests, docs, and comment/error-reporting touchups. --- doc/src/cursor.rst | 17 ++++++++++------ psycopg/cursor_type.c | 36 +++++++++++++++++++++++---------- tests/test_cursor.py | 37 ++++++++++++++++++++++++++++++++++ tests/test_psycopg2_dbapi20.py | 23 +++++++++++++++++++++ 4 files changed, 96 insertions(+), 17 deletions(-) diff --git a/doc/src/cursor.rst b/doc/src/cursor.rst index 73bb5375..9df65865 100644 --- a/doc/src/cursor.rst +++ b/doc/src/cursor.rst @@ -201,12 +201,17 @@ The ``cursor`` class Call a stored database procedure with the given name. The sequence of parameters must contain one entry for each argument that the procedure - expects. The result of the call is returned as modified copy of the - input sequence. Input parameters are left untouched, output and - input/output parameters replaced with possibly new values. - - The procedure may also provide a result set as output. This must then - be made available through the standard |fetch*|_ methods. + expects. Overloaded procedures are supported. Named parameters can be + used with a PostgreSQL 9.0+ client by supplying the sequence of + parameters as a Dict. + + This function is, at present, not DBAPI-compliant. The return value is + supposed to consist of the sequence of parameters with modified output + and input/output parameters. In future versions, the DBAPI-compliant + return value may be implemented, but for now the function returns None. + + The procedure may provide a result set as output. This is then made + available through the standard |fetch*|_ methods. .. method:: mogrify(operation [, parameters]) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 3dbdbdc4..8214d359 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1028,6 +1028,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) int using_dict; #if PG_VERSION_HEX >= 0x090000 PyObject *pname = NULL; + PyObject *spname = NULL; PyObject *bpname = NULL; PyObject *pnames = NULL; char *cpname = NULL; @@ -1055,7 +1056,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) using_dict = nparameters > 0 && PyDict_Check(parameters); - /* A Dict is complicated. The parameter names go into the query */ + /* a Dict is complicated; the parameter names go into the query */ if (using_dict) { #if PG_VERSION_HEX >= 0x090000 if (!(pnames = PyDict_Keys(parameters))) { @@ -1073,33 +1074,46 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) memset(scpnames, 0, sizeof(char *) * nparameters); - /* Each parameter has to be processed. It's a few steps. */ + /* each parameter has to be processed; it's a few steps. */ for (i = 0; i < nparameters; i++) { + /* all errors are RuntimeErrors as they should never occur */ + if (!(pname = PyList_GetItem(pnames, i))) { PyErr_SetString(PyExc_RuntimeError, "built-in 'values' did not return List!"); goto exit; } - if (!(bpname = psycopg_ensure_bytes(pname))) { - PyErr_SetString(PyExc_TypeError, - "argument 2 must have only string keys if Dict"); + if (!(spname = PyObject_Str(pname))) { + PyErr_SetString(PyExc_RuntimeError, + "built-in 'str' failed!"); + goto exit; + } + + /* this is the only function here that returns a new reference */ + if (!(bpname = psycopg_ensure_bytes(spname))) { + PyErr_SetString(PyExc_RuntimeError, + "failed to get Bytes from text!"); goto exit; } if (!(cpname = Bytes_AsString(bpname))) { + Py_XDECREF(bpname); PyErr_SetString(PyExc_RuntimeError, - "failed to get Bytes from String!"); + "failed to get cstr from Bytes!"); goto exit; } if (!(scpnames[i] = PQescapeIdentifier(self->conn->pgconn, cpname, strlen(cpname)))) { + Py_XDECREF(bpname); PyErr_SetString(PyExc_RuntimeError, "libpq failed to escape identifier!"); goto exit; } + Py_XDECREF(bpname); + sl += strlen(scpnames[i]); } @@ -1153,11 +1167,11 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (0 <= _psyco_curs_execute(self, operation, parameters, self->conn->async, 0)) { - /* In the dict case, the parameters are already a new reference */ - if (!using_dict) { - Py_INCREF(parameters); - } - res = parameters; + if (using_dict) { + Py_DECREF(parameters); + } + /* return None from this until it's DBAPI compliant... */ + res = Py_None; } exit: diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 970cc37d..00d19dfb 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -490,6 +490,43 @@ class CursorTests(ConnectingTestCase): cur = self.conn.cursor() self.assertRaises(TypeError, cur.callproc, 'lower', 42) + # It would be inappropriate to test callproc's named parameters in the + # DBAPI2.0 test section because they are a psycopg2 extension. + @skip_before_postgres(9, 0) + def test_callproc_dict(self): + # This parameter name tests for injection and quote escaping + paramname = ''' + Robert'); drop table "students" -- + '''.strip() + escaped_paramname = '"%s"' % paramname.replace('"', '""') + procname = 'pg_temp.randall' + + cur = self.conn.cursor() + + # Set up the temporary function + cur.execute(''' + CREATE FUNCTION %s(%s INT) + RETURNS INT AS + 'SELECT $1 * $1' + LANGUAGE SQL + ''' % (procname, escaped_paramname)); + + # Make sure callproc works right + cur.callproc(procname, { paramname: 2 }) + self.assertEquals(cur.fetchone()[0], 4) + + # Make sure callproc fails right + failing_cases = [ + ({ paramname: 2, 'foo': 'bar' }, psycopg2.ProgrammingError), + ({ paramname: '2' }, psycopg2.ProgrammingError), + ({ paramname: 'two' }, psycopg2.ProgrammingError), + ({ 'bjørn': 2 }, psycopg2.ProgrammingError), + ({ 3: 2 }, psycopg2.ProgrammingError), + ({ self: 2 }, psycopg2.ProgrammingError), + ] + for parameter_sequence, exception in failing_cases: + self.assertRaises(exception, cur.callproc, procname, parameter_sequence) + self.conn.rollback() def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/tests/test_psycopg2_dbapi20.py b/tests/test_psycopg2_dbapi20.py index 744d3224..28ea6690 100755 --- a/tests/test_psycopg2_dbapi20.py +++ b/tests/test_psycopg2_dbapi20.py @@ -36,6 +36,29 @@ class Psycopg2Tests(dbapi20.DatabaseAPI20Test): connect_kw_args = {'dsn': dsn} lower_func = 'lower' # For stored procedure test + def test_callproc(self): + # Until DBAPI 2.0 compliance, callproc should return None or it's just + # misleading. Therefore, we will skip the return value test for + # callproc and only perform the fetch test. + # + # For what it's worth, the DBAPI2.0 test_callproc doesn't actually + # test for DBAPI2.0 compliance! It doesn't check for modified OUT and + # IN/OUT parameters in the return values! + con = self._connect() + try: + cur = con.cursor() + if self.lower_func and hasattr(cur,'callproc'): + cur.callproc(self.lower_func,('FOO',)) + r = cur.fetchall() + self.assertEqual(len(r),1,'callproc produced no result set') + self.assertEqual(len(r[0]),1, + 'callproc produced invalid result set' + ) + self.assertEqual(r[0][0],'foo', + 'callproc produced invalid results' + ) + finally: + con.close() def test_setoutputsize(self): # psycopg2's setoutputsize() is a no-op From 04ce14b251bbbd1a6b48f26e58135e5e53d4080d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 01:25:22 +0200 Subject: [PATCH 08/18] Avoid clobbering the exceptions raised by other calls --- psycopg/cursor_type.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 8214d359..835f278b 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1060,8 +1060,6 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (using_dict) { #if PG_VERSION_HEX >= 0x090000 if (!(pnames = PyDict_Keys(parameters))) { - PyErr_SetString(PyExc_RuntimeError, - "built-in 'keys' failed on a Dict!"); goto exit; } @@ -1079,36 +1077,26 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* all errors are RuntimeErrors as they should never occur */ if (!(pname = PyList_GetItem(pnames, i))) { - PyErr_SetString(PyExc_RuntimeError, - "built-in 'values' did not return List!"); goto exit; } if (!(spname = PyObject_Str(pname))) { - PyErr_SetString(PyExc_RuntimeError, - "built-in 'str' failed!"); goto exit; } /* this is the only function here that returns a new reference */ if (!(bpname = psycopg_ensure_bytes(spname))) { - PyErr_SetString(PyExc_RuntimeError, - "failed to get Bytes from text!"); goto exit; } if (!(cpname = Bytes_AsString(bpname))) { Py_XDECREF(bpname); - PyErr_SetString(PyExc_RuntimeError, - "failed to get cstr from Bytes!"); goto exit; } if (!(scpnames[i] = PQescapeIdentifier(self->conn->pgconn, cpname, strlen(cpname)))) { Py_XDECREF(bpname); - PyErr_SetString(PyExc_RuntimeError, - "libpq failed to escape identifier!"); goto exit; } @@ -1132,8 +1120,6 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) sql[sl-1] = '\0'; if (!(parameters = PyDict_Values(parameters))) { - PyErr_SetString(PyExc_RuntimeError, - "built-in 'values' failed on a Dict!"); goto exit; } #else From a3eed9c9f530f62b6d4c595b0f66ab16c5a4978d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 01:32:58 +0200 Subject: [PATCH 09/18] Added guard on params with no length on callproc --- psycopg/cursor_type.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 835f278b..effb2e66 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1058,7 +1058,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* a Dict is complicated; the parameter names go into the query */ if (using_dict) { -#if PG_VERSION_HEX >= 0x090000 +#if PG_VERSION_NUM >= 90000 if (!(pnames = PyDict_Keys(parameters))) { goto exit; } From d297976d6d01273745e8e1f18b435d84e84984fc Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 02:18:05 +0200 Subject: [PATCH 10/18] Raise TypeError if the dict in callproc param contains non-strings Check-and-conversion chain fixed and simplified. 'spname' was a reference leak. --- psycopg/cursor_type.c | 38 ++++++++++++-------------------------- tests/test_cursor.py | 6 +++--- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index effb2e66..b65b7d22 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1028,8 +1028,6 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) int using_dict; #if PG_VERSION_HEX >= 0x090000 PyObject *pname = NULL; - PyObject *spname = NULL; - PyObject *bpname = NULL; PyObject *pnames = NULL; char *cpname = NULL; char **scpnames = NULL; @@ -1076,37 +1074,24 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) for (i = 0; i < nparameters; i++) { /* all errors are RuntimeErrors as they should never occur */ - if (!(pname = PyList_GetItem(pnames, i))) { + if (!(pname = PyList_GetItem(pnames, i))) { goto exit; } + Py_INCREF(pname); /* was borrowed */ + + /* this also makes a check for keys being strings */ + if (!(pname = psycopg_ensure_bytes(pname))) { goto exit; } + if (!(cpname = Bytes_AsString(pname))) { goto exit; } + + if (!(scpnames[i] = PQescapeIdentifier( + self->conn->pgconn, cpname, strlen(cpname)))) { goto exit; } - if (!(spname = PyObject_Str(pname))) { - goto exit; - } - - /* this is the only function here that returns a new reference */ - if (!(bpname = psycopg_ensure_bytes(spname))) { - goto exit; - } - - if (!(cpname = Bytes_AsString(bpname))) { - Py_XDECREF(bpname); - goto exit; - } - - if (!(scpnames[i] = PQescapeIdentifier(self->conn->pgconn, cpname, - strlen(cpname)))) { - Py_XDECREF(bpname); - goto exit; - } - - Py_XDECREF(bpname); + Py_CLEAR(pname); sl += strlen(scpnames[i]); } - sql = (char*)PyMem_Malloc(sl); - if (sql == NULL) { + if (!(sql = (char*)PyMem_Malloc(sl))) { PyErr_NoMemory(); goto exit; } @@ -1170,6 +1155,7 @@ exit: } } PyMem_Del(scpnames); + Py_XDECREF(pname); Py_XDECREF(pnames); #endif Py_XDECREF(operation); diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 00d19dfb..00143ee9 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -520,9 +520,9 @@ class CursorTests(ConnectingTestCase): ({ paramname: 2, 'foo': 'bar' }, psycopg2.ProgrammingError), ({ paramname: '2' }, psycopg2.ProgrammingError), ({ paramname: 'two' }, psycopg2.ProgrammingError), - ({ 'bjørn': 2 }, psycopg2.ProgrammingError), - ({ 3: 2 }, psycopg2.ProgrammingError), - ({ self: 2 }, psycopg2.ProgrammingError), + ({ u'bj\xc3rn': 2 }, psycopg2.ProgrammingError), + ({ 3: 2 }, TypeError), + ({ self: 2 }, TypeError), ] for parameter_sequence, exception in failing_cases: self.assertRaises(exception, cur.callproc, procname, parameter_sequence) From 7302f348bc7ec7121e04aae70eb8e7705b971334 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 02:32:53 +0200 Subject: [PATCH 11/18] Added test with objects without length as callproc param --- tests/test_cursor.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 00143ee9..552b29c0 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -528,6 +528,11 @@ class CursorTests(ConnectingTestCase): self.assertRaises(exception, cur.callproc, procname, parameter_sequence) self.conn.rollback() + def test_callproc_badparam(self): + cur = self.conn.cursor() + self.assertRaises(TypeError, cur.callproc, 'lower', 42) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 021f6d22ad5995a24439befdf5acde2590556cf4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 12:44:04 +0200 Subject: [PATCH 12/18] More straightforward param refcount handling in callproc --- psycopg/cursor_type.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index b65b7d22..06a43f18 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1022,6 +1022,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) char *sql = NULL; Py_ssize_t procname_len, i, nparameters = 0, sl = 0; PyObject *parameters = Py_None; + PyObject *pvals = NULL; PyObject *operation = NULL; PyObject *res = NULL; @@ -1057,9 +1058,8 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* a Dict is complicated; the parameter names go into the query */ if (using_dict) { #if PG_VERSION_NUM >= 90000 - if (!(pnames = PyDict_Keys(parameters))) { - goto exit; - } + if (!(pnames = PyDict_Keys(parameters))) { goto exit; } + if (!(pvals = PyDict_Values(parameters))) { goto exit; } sl = procname_len + 17 + nparameters * 5 - (nparameters ? 1 : 0); @@ -1104,9 +1104,6 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) sql[sl-2] = ')'; sql[sl-1] = '\0'; - if (!(parameters = PyDict_Values(parameters))) { - goto exit; - } #else PyErr_SetString(PyExc_NotImplementedError, "named parameters require psycopg2 compiled against libpq 9.0+"); @@ -1116,6 +1113,9 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* a list (or None, or empty data structure) is a little bit simpler */ else { + Py_INCREF(parameters); + pvals = parameters; + sl = procname_len + 17 + nparameters * 3 - (nparameters ? 1 : 0); sql = (char*)PyMem_Malloc(sl); @@ -1136,11 +1136,9 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) goto exit; } - if (0 <= _psyco_curs_execute(self, operation, parameters, - self->conn->async, 0)) { - if (using_dict) { - Py_DECREF(parameters); - } + if (0 <= _psyco_curs_execute( + self, operation, pvals, self->conn->async, 0)) { + /* return None from this until it's DBAPI compliant... */ res = Py_None; } @@ -1159,6 +1157,7 @@ exit: Py_XDECREF(pnames); #endif Py_XDECREF(operation); + Py_XDECREF(pvals); PyMem_Free((void*)sql); return res; } From 4003b7c977fba9897da21446707671157a67e6ad Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 12:45:39 +0200 Subject: [PATCH 13/18] Fixed callproc return value refcount Temporary anyway: I want to go back returning a list (or dict). --- psycopg/cursor_type.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 06a43f18..d10bfb82 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1139,8 +1139,9 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (0 <= _psyco_curs_execute( self, operation, pvals, self->conn->async, 0)) { - /* return None from this until it's DBAPI compliant... */ - res = Py_None; + /* return None from this until it's DBAPI compliant... */ + Py_INCREF(Py_None); + res = Py_None; } exit: From 54e5349f531df58b08243f74c03ce13ab8708263 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 13:17:51 +0200 Subject: [PATCH 14/18] Set an exception in case of PQescapeIdentifier error Ifdeffed surface reduced. --- psycopg/cursor_type.c | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index d10bfb82..fd8fbbee 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1015,6 +1015,34 @@ exit: #define psyco_curs_callproc_doc \ "callproc(procname, parameters=None) -- Execute stored procedure." +/* Call PQescapeIdentifier. + * + * In case of error set a Python exception. + * + * TODO: this function can become more generic and go into utils + */ +static char * +_escape_identifier(PGconn *pgconn, const char *str, size_t length) +{ + char *rv = NULL; + +#if PG_VERSION_NUM >= 90000 + rv = PQescapeIdentifier(pgconn, str, length); + if (!rv) { + char *msg; + if (!(msg = PQerrorMessage(pgconn))) { + msg = "no message provided"; + } + PyErr_Format(InterfaceError, "failed to escape identifier: %s", msg); + } +#else + PyErr_Format(PyExc_NotImplementedError, + "named parameters require psycopg2 compiled against libpq 9.0+"); +#endif + + return rv; +} + static PyObject * psyco_curs_callproc(cursorObject *self, PyObject *args) { @@ -1022,17 +1050,15 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) char *sql = NULL; Py_ssize_t procname_len, i, nparameters = 0, sl = 0; PyObject *parameters = Py_None; - PyObject *pvals = NULL; PyObject *operation = NULL; PyObject *res = NULL; int using_dict; -#if PG_VERSION_HEX >= 0x090000 PyObject *pname = NULL; PyObject *pnames = NULL; + PyObject *pvals = NULL; char *cpname = NULL; char **scpnames = NULL; -#endif if (!PyArg_ParseTuple(args, "s#|O", &procname, &procname_len, ¶meters)) { @@ -1057,7 +1083,6 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) /* a Dict is complicated; the parameter names go into the query */ if (using_dict) { -#if PG_VERSION_NUM >= 90000 if (!(pnames = PyDict_Keys(parameters))) { goto exit; } if (!(pvals = PyDict_Values(parameters))) { goto exit; } @@ -1081,7 +1106,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (!(pname = psycopg_ensure_bytes(pname))) { goto exit; } if (!(cpname = Bytes_AsString(pname))) { goto exit; } - if (!(scpnames[i] = PQescapeIdentifier( + if (!(scpnames[i] = _escape_identifier( self->conn->pgconn, cpname, strlen(cpname)))) { goto exit; } @@ -1103,12 +1128,6 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) } sql[sl-2] = ')'; sql[sl-1] = '\0'; - -#else - PyErr_SetString(PyExc_NotImplementedError, - "named parameters require psycopg2 compiled against libpq 9.0+"); - goto exit; -#endif } /* a list (or None, or empty data structure) is a little bit simpler */ @@ -1145,7 +1164,6 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) } exit: -#if PG_VERSION_HEX >= 0x090000 if (scpnames != NULL) { for (i = 0; i < nparameters; i++) { if (scpnames[i] != NULL) { @@ -1156,7 +1174,6 @@ exit: PyMem_Del(scpnames); Py_XDECREF(pname); Py_XDECREF(pnames); -#endif Py_XDECREF(operation); Py_XDECREF(pvals); PyMem_Free((void*)sql); From 92109e4bbad9cdcd86dd9bf2f40bae5c26202a0d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 13:41:45 +0200 Subject: [PATCH 15/18] Correctly handle an empty error message from PQescapeIdentifier --- psycopg/cursor_type.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index fd8fbbee..0a785c05 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1030,7 +1030,8 @@ _escape_identifier(PGconn *pgconn, const char *str, size_t length) rv = PQescapeIdentifier(pgconn, str, length); if (!rv) { char *msg; - if (!(msg = PQerrorMessage(pgconn))) { + msg = PQerrorMessage(pgconn); + if (!msg || !msg[0]) { msg = "no message provided"; } PyErr_Format(InterfaceError, "failed to escape identifier: %s", msg); From 0772d187e914929bf2666a1f6acac91639c18765 Mon Sep 17 00:00:00 2001 From: mrmilosz Date: Sun, 13 Dec 2015 01:10:03 -0500 Subject: [PATCH 16/18] Return input tuple in cur.callproc, factor code to use PQescapeIdentifier in single place --- psycopg/cursor_type.c | 43 +++++++++-------------------------------- psycopg/psycopg.h | 1 + psycopg/psycopgmodule.c | 9 +-------- psycopg/utils.c | 33 +++++++++++++++++++++++++++++-- 4 files changed, 42 insertions(+), 44 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 0a785c05..e205ba2a 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1015,35 +1015,6 @@ exit: #define psyco_curs_callproc_doc \ "callproc(procname, parameters=None) -- Execute stored procedure." -/* Call PQescapeIdentifier. - * - * In case of error set a Python exception. - * - * TODO: this function can become more generic and go into utils - */ -static char * -_escape_identifier(PGconn *pgconn, const char *str, size_t length) -{ - char *rv = NULL; - -#if PG_VERSION_NUM >= 90000 - rv = PQescapeIdentifier(pgconn, str, length); - if (!rv) { - char *msg; - msg = PQerrorMessage(pgconn); - if (!msg || !msg[0]) { - msg = "no message provided"; - } - PyErr_Format(InterfaceError, "failed to escape identifier: %s", msg); - } -#else - PyErr_Format(PyExc_NotImplementedError, - "named parameters require psycopg2 compiled against libpq 9.0+"); -#endif - - return rv; -} - static PyObject * psyco_curs_callproc(cursorObject *self, PyObject *args) { @@ -1107,7 +1078,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (!(pname = psycopg_ensure_bytes(pname))) { goto exit; } if (!(cpname = Bytes_AsString(pname))) { goto exit; } - if (!(scpnames[i] = _escape_identifier( + if (!(scpnames[i] = psycopg_escape_identifier( self->conn->pgconn, cpname, strlen(cpname)))) { goto exit; } @@ -1158,10 +1129,14 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (0 <= _psyco_curs_execute( self, operation, pvals, self->conn->async, 0)) { - - /* return None from this until it's DBAPI compliant... */ - Py_INCREF(Py_None); - res = Py_None; + /* The dict case is outside DBAPI scope anyway, so simply return None */ + if (using_dict) { + Py_INCREF(Py_None); + res = Py_None; + } + else { + res = pvals; + } } exit: diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index eb406fd2..1a16e8af 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -124,6 +124,7 @@ RAISES HIDDEN PyObject *psyco_set_error(PyObject *exc, cursorObject *curs, const HIDDEN char *psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, char *to, Py_ssize_t *tolen); HIDDEN char *psycopg_escape_identifier_easy(const char *from, Py_ssize_t len); +HIDDEN char *psycopg_escape_identifier(PGconn *pgconn, const char *str, size_t length); HIDDEN int psycopg_strdup(char **to, const char *from, Py_ssize_t len); HIDDEN int psycopg_is_text_file(PyObject *f); diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index cf70a4ad..0ecfcc9c 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -59,7 +59,6 @@ HIDDEN PyObject *pyDateTimeModuleP = NULL; HIDDEN PyObject *psycoEncodings = NULL; - #ifdef PSYCOPG_DEBUG HIDDEN int psycopg_debug_enabled = 0; #endif @@ -175,7 +174,6 @@ exit: static PyObject * psyco_quote_ident(PyObject *self, PyObject *args, PyObject *kwargs) { -#if PG_VERSION_NUM >= 90000 PyObject *ident = NULL, *obj = NULL, *result = NULL; connectionObject *conn; const char *str; @@ -203,9 +201,8 @@ psyco_quote_ident(PyObject *self, PyObject *args, PyObject *kwargs) str = Bytes_AS_STRING(ident); - quoted = PQescapeIdentifier(conn->pgconn, str, strlen(str)); + quoted = psycopg_escape_identifier(conn->pgconn, str, strlen(str)); if (!quoted) { - PyErr_NoMemory(); goto exit; } result = conn_text_from_chars(conn, quoted); @@ -215,10 +212,6 @@ exit: Py_XDECREF(ident); return result; -#else - PyErr_SetString(NotSupportedError, "PQescapeIdentifier not available in libpq < 9.0"); - return NULL; -#endif } /** type registration **/ diff --git a/psycopg/utils.c b/psycopg/utils.c index ec8e47c8..e224a0d5 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -95,8 +95,8 @@ psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, * The returned string doesn't include quotes. * * WARNING: this function is not so safe to allow untrusted input: it does no - * check for multibyte chars. Such a function should be built on - * PQescapeIdentifier, which is only available from PostgreSQL 9.0. + * check for multibyte chars. Functions otherwise reliant on PostgreSQL 9.0 + * and above should use the below function psycopg_escape_identifier instead. */ char * psycopg_escape_identifier_easy(const char *from, Py_ssize_t len) @@ -124,6 +124,35 @@ psycopg_escape_identifier_easy(const char *from, Py_ssize_t len) return rv; } + +/* Call PostgreSQL 9.0+ function PQescapeIdentifier. + * + * In case of error set a Python exception. + */ +char * +psycopg_escape_identifier(PGconn *pgconn, const char *str, size_t length) +{ + char *rv = NULL; + +#if PG_VERSION_NUM >= 90000 + rv = PQescapeIdentifier(pgconn, str, length); + if (!rv) { + char *msg; + msg = PQerrorMessage(pgconn); + if (!msg || !msg[0]) { + msg = "no message provided"; + } + PyErr_Format(InterfaceError, "failed to escape identifier: %s", msg); + } +#else + PyErr_Format(PyExc_NotImplementedError, + "PQescapeIdentifier requires psycopg2 compiled against libpq 9.0+"); +#endif + + return rv; +} + + /* Duplicate a string. * * Allocate a new buffer on the Python heap containing the new string. From d13521a6ce1f92956dcca2b68e4e703008580c7b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 26 Dec 2016 03:39:28 +0100 Subject: [PATCH 17/18] Mention named callproc in news, fixed docs. --- NEWS | 1 + doc/src/cursor.rst | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 67883d74..6ffa66a9 100644 --- a/NEWS +++ b/NEWS @@ -24,6 +24,7 @@ New features: adapter is deprecated (:tickets:`#317, #343, #387`). - Added `~psycopg2.extensions.quote_ident()` function (:ticket:`#359`). - Added `~connection.get_dsn_parameters()` connection method (:ticket:`#364`). +- `~cursor.callproc()` now accepts a dictionary of parameters (:ticket:`#381`). Other changes: diff --git a/doc/src/cursor.rst b/doc/src/cursor.rst index 974e1a2b..aee6b465 100644 --- a/doc/src/cursor.rst +++ b/doc/src/cursor.rst @@ -202,8 +202,7 @@ The ``cursor`` class Call a stored database procedure with the given name. The sequence of parameters must contain one entry for each argument that the procedure expects. Overloaded procedures are supported. Named parameters can be - used with a PostgreSQL 9.0+ client by supplying the sequence of - parameters as a Dict. + used by supplying the parameters as a dictionary. This function is, at present, not DBAPI-compliant. The return value is supposed to consist of the sequence of parameters with modified output @@ -213,6 +212,8 @@ The ``cursor`` class The procedure may provide a result set as output. This is then made available through the standard |fetch*|_ methods. + .. versionchanged:: 2.7 + added support for named arguments. .. method:: mogrify(operation [, parameters]) From ffeb7001ebfaab34613ce604a509dfa1de193b80 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 26 Dec 2016 04:12:18 +0100 Subject: [PATCH 18/18] Fixed refcount problems in named callproc --- psycopg/cursor_type.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 66580ad5..baa5b8f7 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1080,6 +1080,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) if (!(scpnames[i] = psycopg_escape_identifier( self->conn, cpname, 0))) { + Py_CLEAR(pname); goto exit; } @@ -1131,12 +1132,12 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) self, operation, pvals, self->conn->async, 0)) { /* The dict case is outside DBAPI scope anyway, so simply return None */ if (using_dict) { - Py_INCREF(Py_None); res = Py_None; } else { res = pvals; } + Py_INCREF(res); } exit: