From 6c57e4a648cbd5ff81db425978e007f5961e2e77 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 10:16:07 +0200 Subject: [PATCH 1/8] Add parse_dsn module function Calls PQconninfoParse to parse the dsn into a list of keyword and value structs, then constructs a dictionary from that. Can be useful when one needs to alter some part of the the connection string reliably, but doesn't want to get into all the details of parsing a dsn string: quoting, URL format, etc. --- doc/src/module.rst | 12 ++++++++++++ lib/__init__.py | 2 +- psycopg/psycopgmodule.c | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/doc/src/module.rst b/doc/src/module.rst index 8de9f87e..36073a23 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -78,6 +78,7 @@ The module interface respects the standard defined in the |DBAPI|_. .. seealso:: + - `parse_dsn` - libpq `connection string syntax`__ - libpq supported `connection parameters`__ - libpq supported `environment variables`__ @@ -91,6 +92,17 @@ The module interface respects the standard defined in the |DBAPI|_. The parameters *connection_factory* and *async* are Psycopg extensions to the |DBAPI|. +.. function:: parse_dsn(dsn) + + Parse connection string into a dictionary of keywords and values. + + Uses libpq's ``PQconninfoParse`` to parse the string according to + accepted format(s) and check for supported keywords. + + Example:: + + >>> psycopg2.parse_dsn('dbname=test user=postgres password=secret') + {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} .. data:: apilevel diff --git a/lib/__init__.py b/lib/__init__.py index cf8c06ae..27b9d172 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -56,7 +56,7 @@ from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, Programm from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError -from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle +from psycopg2._psycopg import _connect, parse_dsn, apilevel, threadsafety, paramstyle from psycopg2._psycopg import __version__ from psycopg2 import tz diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 61e2de57..d8f893cc 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -112,6 +112,44 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) return conn; } +#define psyco_parse_dsn_doc \ +"parse_dsn(dsn) -- Parse database connection string.\n\n" + +static PyObject * +psyco_parse_dsn(PyObject *self, PyObject *args) +{ + char *dsn, *err; + PQconninfoOption *options = NULL, *o; + PyObject *res = NULL, *value; + + if (!PyArg_ParseTuple(args, "s", &dsn)) { + return NULL; + } + + options = PQconninfoParse(dsn, &err); + if (!options) { + PyErr_Format(PyExc_RuntimeError, "PQconninfoParse: %s: %s", dsn, err); + PQfreemem(err); + return NULL; + } + + res = PyDict_New(); + for (o = options; o->keyword != NULL; o++) { + if (o->val != NULL) { + value = PyString_FromString(o->val); + if (value == NULL || PyDict_SetItemString(res, o->keyword, value) != 0) { + Py_DECREF(res); + res = NULL; + break; + } + } + } + + PQconninfoFree(options); + + return res; +} + /** type registration **/ #define psyco_register_type_doc \ "register_type(obj, conn_or_curs) -> None -- register obj with psycopg type system\n\n" \ @@ -695,6 +733,8 @@ error: static PyMethodDef psycopgMethods[] = { {"_connect", (PyCFunction)psyco_connect, METH_VARARGS|METH_KEYWORDS, psyco_connect_doc}, + {"parse_dsn", (PyCFunction)psyco_parse_dsn, + METH_VARARGS, psyco_parse_dsn_doc}, {"adapt", (PyCFunction)psyco_microprotocols_adapt, METH_VARARGS, psyco_microprotocols_adapt_doc}, From 6a2f21aa14eec258178b6a90255ebfe7bbe0acdc Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 15:11:12 +0200 Subject: [PATCH 2/8] Move parse_dsn to extensions, add tests --- doc/src/extensions.rst | 11 +++++++++++ doc/src/module.rst | 14 +------------- lib/__init__.py | 2 +- lib/extensions.py | 2 +- psycopg/psycopgmodule.c | 38 +++++++++++++++++++++++++------------- tests/test_connection.py | 31 +++++++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 28 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index dea10417..3cdd4c4f 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -12,6 +12,17 @@ The module contains a few objects and function extending the minimum set of functionalities defined by the |DBAPI|_. +.. function:: parse_dsn(dsn) + + Parse connection string into a dictionary of keywords and values. + + Uses libpq's ``PQconninfoParse`` to parse the string according to + accepted format(s) and check for supported keywords. + + Example:: + + >>> psycopg2.extensions.parse_dsn('dbname=test user=postgres password=secret') + {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} .. class:: connection(dsn, async=False) diff --git a/doc/src/module.rst b/doc/src/module.rst index 36073a23..ad19aa7b 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -78,7 +78,7 @@ The module interface respects the standard defined in the |DBAPI|_. .. seealso:: - - `parse_dsn` + - `~psycopg2.extensions.parse_dsn` - libpq `connection string syntax`__ - libpq supported `connection parameters`__ - libpq supported `environment variables`__ @@ -92,18 +92,6 @@ The module interface respects the standard defined in the |DBAPI|_. The parameters *connection_factory* and *async* are Psycopg extensions to the |DBAPI|. -.. function:: parse_dsn(dsn) - - Parse connection string into a dictionary of keywords and values. - - Uses libpq's ``PQconninfoParse`` to parse the string according to - accepted format(s) and check for supported keywords. - - Example:: - - >>> psycopg2.parse_dsn('dbname=test user=postgres password=secret') - {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} - .. data:: apilevel String constant stating the supported DB API level. For `psycopg2` is diff --git a/lib/__init__.py b/lib/__init__.py index 27b9d172..cf8c06ae 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -56,7 +56,7 @@ from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, Programm from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError -from psycopg2._psycopg import _connect, parse_dsn, apilevel, threadsafety, paramstyle +from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle from psycopg2._psycopg import __version__ from psycopg2 import tz diff --git a/lib/extensions.py b/lib/extensions.py index 216d8ad2..f951c519 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -56,7 +56,7 @@ try: except ImportError: pass -from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid +from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid, parse_dsn from psycopg2._psycopg import string_types, binary_types, new_type, new_array_type, register_type from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics, Column diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index d8f893cc..f36fbf42 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -112,13 +112,12 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) return conn; } -#define psyco_parse_dsn_doc \ -"parse_dsn(dsn) -- Parse database connection string.\n\n" +#define psyco_parse_dsn_doc "parse_dsn(dsn) -> dict" static PyObject * psyco_parse_dsn(PyObject *self, PyObject *args) { - char *dsn, *err; + char *dsn, *err = NULL; PQconninfoOption *options = NULL, *o; PyObject *res = NULL, *value; @@ -127,20 +126,33 @@ psyco_parse_dsn(PyObject *self, PyObject *args) } options = PQconninfoParse(dsn, &err); - if (!options) { - PyErr_Format(PyExc_RuntimeError, "PQconninfoParse: %s: %s", dsn, err); - PQfreemem(err); + if (options == NULL) { + if (err != NULL) { + PyErr_Format(ProgrammingError, "error parsing the dsn: %s", err); + PQfreemem(err); + } else { + PyErr_SetString(OperationalError, "PQconninfoParse() failed"); + } return NULL; } res = PyDict_New(); - for (o = options; o->keyword != NULL; o++) { - if (o->val != NULL) { - value = PyString_FromString(o->val); - if (value == NULL || PyDict_SetItemString(res, o->keyword, value) != 0) { - Py_DECREF(res); - res = NULL; - break; + if (res != NULL) { + for (o = options; o->keyword != NULL; o++) { + if (o->val != NULL) { + value = Text_FromUTF8(o->val); + if (value == NULL) { + Py_DECREF(res); + res = NULL; + break; + } + if (PyDict_SetItemString(res, o->keyword, value) != 0) { + Py_DECREF(value); + Py_DECREF(res); + res = NULL; + break; + } + Py_DECREF(value); } } } diff --git a/tests/test_connection.py b/tests/test_connection.py index 340693e2..eb80d4c4 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -270,6 +270,37 @@ class ConnectionTests(ConnectingTestCase): self.assert_(c.closed, "connection failed so it must be closed") self.assert_('foobar' not in c.dsn, "password was not obscured") + def test_parse_dsn(self): + from psycopg2 import ProgrammingError + from psycopg2.extensions import parse_dsn + + self.assertEqual(parse_dsn('dbname=test user=tester password=secret'), + dict(user='tester', password='secret', dbname='test'), + "simple DSN parsed") + + self.assertEqual(parse_dsn("dbname='test 2' user=tester password=secret"), + dict(user='tester', password='secret', dbname='test 2'), + "DSN with quoting parsed") + + self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), + dict(user='tester', password='secret', dbname='test'), + "simple URI dsn parsed") + + # Can't really use assertRaisesRegexp() here since we need to + # make sure that secret is *not* exposed in the error messgage + # (and it also requires python >= 2.7). + raised = False + try: + # unterminated quote after dbname: + parse_dsn("dbname='test 2 user=tester password=secret") + except ProgrammingError, e: + raised = True + self.assertTrue(e.message.find('secret') < 0, + "DSN was not exposed in error message") + except e: + self.fail("unexpected error condition: " + repr(e)) + self.assertTrue(raised, "ProgrammingError raised due to invalid DSN") + class IsolationLevelsTestCase(ConnectingTestCase): From 3200cd77bf07c5de3375dece678eb58f714956b9 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 15:18:03 +0200 Subject: [PATCH 3/8] One more parse_dsn test for unquoted space --- tests/test_connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_connection.py b/tests/test_connection.py index eb80d4c4..a5fabf9f 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -278,6 +278,9 @@ class ConnectionTests(ConnectingTestCase): dict(user='tester', password='secret', dbname='test'), "simple DSN parsed") + self.assertRaises(ProgrammingError, parse_dsn, + "dbname=test 2 user=tester password=secret") + self.assertEqual(parse_dsn("dbname='test 2' user=tester password=secret"), dict(user='tester', password='secret', dbname='test 2'), "DSN with quoting parsed") From d6041271bc7fc539d46198b2603cf7fba2f98bd4 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 2 Jun 2015 14:02:29 +0200 Subject: [PATCH 4/8] Separate parse_dsn test on URI, for libpq >= 9.2 --- tests/test_connection.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index a5fabf9f..6a060ab2 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -32,7 +32,7 @@ import psycopg2.errorcodes import psycopg2.extensions from testutils import unittest, decorate_all_tests, skip_if_no_superuser -from testutils import skip_before_postgres, skip_after_postgres +from testutils import skip_before_postgres, skip_after_postgres, skip_before_libpq from testutils import ConnectingTestCase, skip_if_tpc_disabled from testutils import skip_if_windows from testconfig import dsn, dbname @@ -285,10 +285,6 @@ class ConnectionTests(ConnectingTestCase): dict(user='tester', password='secret', dbname='test 2'), "DSN with quoting parsed") - self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), - dict(user='tester', password='secret', dbname='test'), - "simple URI dsn parsed") - # Can't really use assertRaisesRegexp() here since we need to # make sure that secret is *not* exposed in the error messgage # (and it also requires python >= 2.7). @@ -304,6 +300,27 @@ class ConnectionTests(ConnectingTestCase): self.fail("unexpected error condition: " + repr(e)) self.assertTrue(raised, "ProgrammingError raised due to invalid DSN") + @skip_before_libpq(9, 2) + def test_parse_dsn_uri(self): + from psycopg2 import ProgrammingError + from psycopg2.extensions import parse_dsn + + self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), + dict(user='tester', password='secret', dbname='test'), + "valid URI dsn parsed") + + raised = False + try: + # extra '=' after port value + parse_dsn('postgresql://tester:secret@/test?port=1111=x') + except ProgrammingError, e: + raised = True + self.assertTrue(e.message.find('secret') < 0, + "URI was not exposed in error message") + except e: + self.fail("unexpected error condition: " + repr(e)) + self.assertTrue(raised, "ProgrammingError raised due to invalid URI") + class IsolationLevelsTestCase(ConnectingTestCase): From d3bbd19ccb26123a310c7e4c9869410ecf5f6144 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 11:52:42 +0100 Subject: [PATCH 5/8] Separate parse_dsn test in a test case of their own --- tests/test_connection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_connection.py b/tests/test_connection.py index 950c8056..cd60b170 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -307,6 +307,8 @@ class ConnectionTests(ConnectingTestCase): self.assert_(c.closed, "connection failed so it must be closed") self.assert_('foobar' not in c.dsn, "password was not obscured") + +class ParseDsnTestCase(ConnectingTestCase): def test_parse_dsn(self): from psycopg2 import ProgrammingError from psycopg2.extensions import parse_dsn From 71d96293ab61b8554c9f608905c70aa44a6c008d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 12:00:33 +0100 Subject: [PATCH 6/8] Fixed parse_dsn tests on Python 3 On Python 3 there is no Exception.message attribute. --- tests/test_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index cd60b170..4cf50622 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -333,7 +333,7 @@ class ParseDsnTestCase(ConnectingTestCase): parse_dsn("dbname='test 2 user=tester password=secret") except ProgrammingError, e: raised = True - self.assertTrue(e.message.find('secret') < 0, + self.assertTrue(str(e).find('secret') < 0, "DSN was not exposed in error message") except e: self.fail("unexpected error condition: " + repr(e)) @@ -354,7 +354,7 @@ class ParseDsnTestCase(ConnectingTestCase): parse_dsn('postgresql://tester:secret@/test?port=1111=x') except ProgrammingError, e: raised = True - self.assertTrue(e.message.find('secret') < 0, + self.assertTrue(str(e).find('secret') < 0, "URI was not exposed in error message") except e: self.fail("unexpected error condition: " + repr(e)) From 5afeee3613de284fd00c3e8b2228bdbe94d74572 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 13:20:11 +0100 Subject: [PATCH 7/8] Added unicode support to parse_dsn Also added support for the argument as a keyword. --- psycopg/psycopgmodule.c | 51 +++++++++++++++++++++------------------- tests/test_connection.py | 26 +++++++++++++++++--- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 0736d260..737a7811 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -115,17 +115,21 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) #define psyco_parse_dsn_doc "parse_dsn(dsn) -> dict" static PyObject * -psyco_parse_dsn(PyObject *self, PyObject *args) +psyco_parse_dsn(PyObject *self, PyObject *args, PyObject *kwargs) { - char *dsn, *err = NULL; + char *err = NULL; PQconninfoOption *options = NULL, *o; - PyObject *res = NULL, *value; + PyObject *dict = NULL, *res = NULL, *dsn; - if (!PyArg_ParseTuple(args, "s", &dsn)) { + static char *kwlist[] = {"dsn", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kwlist, &dsn)) { return NULL; } - options = PQconninfoParse(dsn, &err); + Py_INCREF(dsn); /* for ensure_bytes */ + if (!(dsn = psycopg_ensure_bytes(dsn))) { goto exit; } + + options = PQconninfoParse(Bytes_AS_STRING(dsn), &err); if (options == NULL) { if (err != NULL) { PyErr_Format(ProgrammingError, "error parsing the dsn: %s", err); @@ -133,31 +137,30 @@ psyco_parse_dsn(PyObject *self, PyObject *args) } else { PyErr_SetString(OperationalError, "PQconninfoParse() failed"); } - return NULL; + goto exit; } - res = PyDict_New(); - if (res != NULL) { - for (o = options; o->keyword != NULL; o++) { - if (o->val != NULL) { - value = Text_FromUTF8(o->val); - if (value == NULL) { - Py_DECREF(res); - res = NULL; - break; - } - if (PyDict_SetItemString(res, o->keyword, value) != 0) { - Py_DECREF(value); - Py_DECREF(res); - res = NULL; - break; - } + if (!(dict = PyDict_New())) { goto exit; } + for (o = options; o->keyword != NULL; o++) { + if (o->val != NULL) { + PyObject *value; + if (!(value = Text_FromUTF8(o->val))) { goto exit; } + if (PyDict_SetItemString(dict, o->keyword, value) != 0) { Py_DECREF(value); + goto exit; } + Py_DECREF(value); } } - PQconninfoFree(options); + /* success */ + res = dict; + dict = NULL; + +exit: + PQconninfoFree(options); /* safe on null */ + Py_XDECREF(dict); + Py_XDECREF(dsn); return res; } @@ -759,7 +762,7 @@ static PyMethodDef psycopgMethods[] = { {"_connect", (PyCFunction)psyco_connect, METH_VARARGS|METH_KEYWORDS, psyco_connect_doc}, {"parse_dsn", (PyCFunction)psyco_parse_dsn, - METH_VARARGS, psyco_parse_dsn_doc}, + METH_VARARGS|METH_KEYWORDS, psyco_parse_dsn_doc}, {"adapt", (PyCFunction)psyco_microprotocols_adapt, METH_VARARGS, psyco_microprotocols_adapt_doc}, diff --git a/tests/test_connection.py b/tests/test_connection.py index 4cf50622..ee742580 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -23,6 +23,7 @@ # License for more details. import os +import sys import time import threading from operator import attrgetter @@ -341,7 +342,6 @@ class ParseDsnTestCase(ConnectingTestCase): @skip_before_libpq(9, 2) def test_parse_dsn_uri(self): - from psycopg2 import ProgrammingError from psycopg2.extensions import parse_dsn self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), @@ -351,8 +351,8 @@ class ParseDsnTestCase(ConnectingTestCase): raised = False try: # extra '=' after port value - parse_dsn('postgresql://tester:secret@/test?port=1111=x') - except ProgrammingError, e: + parse_dsn(dsn='postgresql://tester:secret@/test?port=1111=x') + except psycopg2.ProgrammingError, e: raised = True self.assertTrue(str(e).find('secret') < 0, "URI was not exposed in error message") @@ -360,6 +360,26 @@ class ParseDsnTestCase(ConnectingTestCase): self.fail("unexpected error condition: " + repr(e)) self.assertTrue(raised, "ProgrammingError raised due to invalid URI") + def test_unicode_value(self): + from psycopg2.extensions import parse_dsn + snowman = u"\u2603" + d = parse_dsn('dbname=' + snowman) + if sys.version_info[0] < 3: + self.assertEqual(d['dbname'], snowman.encode('utf8')) + else: + self.assertEqual(d['dbname'], snowman) + + def test_unicode_key(self): + from psycopg2.extensions import parse_dsn + snowman = u"\u2603" + self.assertRaises(psycopg2.ProgrammingError, parse_dsn, + snowman + '=' + snowman) + + def test_bad_param(self): + from psycopg2.extensions import parse_dsn + self.assertRaises(TypeError, parse_dsn, None) + self.assertRaises(TypeError, parse_dsn, 42) + class IsolationLevelsTestCase(ConnectingTestCase): From 58918801d65751270b370d97e6c958fcb23193ab Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 13:26:11 +0100 Subject: [PATCH 8/8] Added parse_dsn() to news file --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 287e5fa4..fd4fc6ba 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,7 @@ What's new in psycopg 2.7 New features: +- Added `~psycopg2.extensions.parse_dsn()` function (:ticket:`#321`). - Added `~psycopg2.__libpq_version__` and `~psycopg2.extensions.libpq_version()` to inspect the version of the ``libpq`` library the module was compiled/loaded with