From 634fc004fbc70d87148c112ddae6a6722bc93bdc Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 00:29:58 +0100 Subject: [PATCH 1/5] Added wishful test suite for jsonb type --- tests/test_types_extras.py | 91 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 4625995d..46a34217 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -1069,6 +1069,97 @@ class JsonTestCase(ConnectingTestCase): self.assert_(s.endswith("'")) +def skip_if_no_jsonb_type(f): + return skip_before_postgres(9, 4)(f) + +class JsonbTestCase(ConnectingTestCase): + @staticmethod + def myloads(s): + import json + rv = json.loads(s) + rv['test'] = 1 + return rv + + def test_default_cast(self): + curs = self.conn.cursor() + + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None}) + + curs.execute("""select array['{"a": 100.0, "b": null}']::jsonb[]""") + self.assertEqual(curs.fetchone()[0], [{'a': 100.0, 'b': None}]) + + def test_register_on_connection(self): + psycopg2.extras.register_json(self.conn, loads=self.myloads, name='jsonb') + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1}) + + def test_register_on_cursor(self): + curs = self.conn.cursor() + psycopg2.extras.register_json(curs, loads=self.myloads, name='jsonb') + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1}) + + def test_register_globally(self): + old = psycopg2.extensions.string_types.get(3802) + olda = psycopg2.extensions.string_types.get(3807) + try: + new, newa = psycopg2.extras.register_json(self.conn, + loads=self.myloads, globally=True, name='jsonb') + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1}) + finally: + psycopg2.extensions.string_types.pop(new.values[0]) + psycopg2.extensions.string_types.pop(newa.values[0]) + if old: + psycopg2.extensions.register_type(old) + if olda: + psycopg2.extensions.register_type(olda) + + def test_loads(self): + json = psycopg2.extras.json + loads = lambda x: json.loads(x, parse_float=Decimal) + psycopg2.extras.register_json(self.conn, loads=loads, name='jsonb') + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], Decimal)) + self.assertEqual(data['a'], Decimal('100.0')) + # sure we are not manling json too? + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], float)) + self.assertEqual(data['a'], 100.0) + + def test_register_default(self): + curs = self.conn.cursor() + + loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) + psycopg2.extras.register_default_jsonb(curs, loads=loads) + + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], Decimal)) + self.assertEqual(data['a'], Decimal('100.0')) + + curs.execute("""select array['{"a": 100.0, "b": null}']::jsonb[]""") + data = curs.fetchone()[0] + self.assert_(isinstance(data[0]['a'], Decimal)) + self.assertEqual(data[0]['a'], Decimal('100.0')) + + def test_null(self): + curs = self.conn.cursor() + curs.execute("""select NULL::jsonb""") + self.assertEqual(curs.fetchone()[0], None) + curs.execute("""select NULL::jsonb[]""") + self.assertEqual(curs.fetchone()[0], None) + +decorate_all_tests(JsonbTestCase, skip_if_no_json_module) +decorate_all_tests(JsonbTestCase, skip_if_no_jsonb_type) + + class RangeTestCase(unittest.TestCase): def test_noparam(self): from psycopg2.extras import Range From 6bca443e37cd0cb5e37352eaf2e8aafc832324ee Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 00:43:33 +0100 Subject: [PATCH 2/5] Added name param to register_json() --- doc/src/extras.rst | 3 +++ lib/_json.py | 28 +++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 7fab3384..0abd3354 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -248,6 +248,9 @@ from :sql:`json` into :py:class:`~decimal.Decimal` you can use:: .. autofunction:: register_json + .. versionchanged:: 2.5.4 + added the *name* parameter to enable :sql:`jsonb` support. + .. autofunction:: register_default_json diff --git a/lib/_json.py b/lib/_json.py index 3a4361e8..047551e5 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -102,7 +102,7 @@ class Json(object): def register_json(conn_or_curs=None, globally=False, loads=None, - oid=None, array_oid=None): + oid=None, array_oid=None, name='json'): """Create and register typecasters converting :sql:`json` type to Python objects. :param conn_or_curs: a connection or cursor used to find the :sql:`json` @@ -118,17 +118,19 @@ def register_json(conn_or_curs=None, globally=False, loads=None, queried on *conn_or_curs* :param array_oid: the OID of the :sql:`json[]` array type if known; if not, it will be queried on *conn_or_curs* + :param name: the name of the data type to look for in *conn_or_curs* The connection or cursor passed to the function will be used to query the - database and look for the OID of the :sql:`json` type. No query is - performed if *oid* and *array_oid* are provided. Raise - `~psycopg2.ProgrammingError` if the type is not found. + database and look for the OID of the :sql:`json` type (or an alternative + type if *name* if provided). No query is performed if *oid* and *array_oid* + are provided. Raise `~psycopg2.ProgrammingError` if the type is not found. """ if oid is None: - oid, array_oid = _get_json_oids(conn_or_curs) + oid, array_oid = _get_json_oids(conn_or_curs, name) - JSON, JSONARRAY = _create_json_typecasters(oid, array_oid, loads) + JSON, JSONARRAY = _create_json_typecasters( + oid, array_oid, loads=loads, name=name.upper()) register_type(JSON, not globally and conn_or_curs or None) @@ -149,7 +151,7 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) -def _create_json_typecasters(oid, array_oid, loads=None): +def _create_json_typecasters(oid, array_oid, loads=None, name='JSON'): """Create typecasters for json data type.""" if loads is None: if json is None: @@ -162,15 +164,15 @@ def _create_json_typecasters(oid, array_oid, loads=None): return None return loads(s) - JSON = new_type((oid, ), 'JSON', typecast_json) + JSON = new_type((oid, ), name, typecast_json) if array_oid is not None: - JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + JSONARRAY = new_array_type((array_oid, ), "%sARRAY" % name, JSON) else: JSONARRAY = None return JSON, JSONARRAY -def _get_json_oids(conn_or_curs): +def _get_json_oids(conn_or_curs, name='json'): # lazy imports from psycopg2.extensions import STATUS_IN_TRANSACTION from psycopg2.extras import _solve_conn_curs @@ -185,8 +187,8 @@ def _get_json_oids(conn_or_curs): # get the oid for the hstore curs.execute( - "SELECT t.oid, %s FROM pg_type t WHERE t.typname = 'json';" - % typarray) + "SELECT t.oid, %s FROM pg_type t WHERE t.typname = %%s;" + % typarray, (name,)) r = curs.fetchone() # revert the status of the connection as before the command @@ -194,7 +196,7 @@ def _get_json_oids(conn_or_curs): conn.rollback() if not r: - raise conn.ProgrammingError("json data type not found") + raise conn.ProgrammingError("%s data type not found" % name) return r From 9d547469b8a3f2a9a554edc5218797349d1a4472 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 00:54:49 +0100 Subject: [PATCH 3/5] Add register_default_jsonb() and register the type --- lib/_json.py | 16 ++++++++++++++++ lib/extensions.py | 6 ++++-- lib/extras.py | 3 ++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/_json.py b/lib/_json.py index 047551e5..eef1436e 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -47,6 +47,10 @@ else: JSON_OID = 114 JSONARRAY_OID = 199 +# oids from PostgreSQL 9.4 +JSONB_OID = 3802 +JSONBARRAY_OID = 3807 + class Json(object): """ An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to @@ -151,6 +155,18 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) +def register_default_jsonb(conn_or_curs=None, globally=False, loads=None): + """ + Create and register :sql:`jsonb` typecasters for PostgreSQL 9.4 and following. + + As in `register_default_json()`, the function allows to register a + customized *loads* function for the :sql:`jsonb` type at its known oid for + PostgreSQL 9.4 and following versions. All the parameters have the same + meaning of `register_json()`. + """ + return register_json(conn_or_curs=conn_or_curs, globally=globally, + loads=loads, oid=JSONB_OID, array_oid=JSONBARRAY_OID, name='jsonb') + def _create_json_typecasters(oid, array_oid, loads=None, name='JSON'): """Create typecasters for json data type.""" if loads is None: diff --git a/lib/extensions.py b/lib/extensions.py index f210da4f..71a92b93 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -152,20 +152,22 @@ class NoneAdapter(object): # Create default json typecasters for PostgreSQL 9.2 oids -from psycopg2._json import register_default_json +from psycopg2._json import register_default_json, register_default_jsonb try: JSON, JSONARRAY = register_default_json() + JSONB, JSONBARRAY = register_default_jsonb() except ImportError: pass -del register_default_json +del register_default_json, register_default_jsonb # Create default Range typecasters from psycopg2. _range import Range del Range + # Add the "cleaned" version of the encodings to the key. # When the encoding is set its name is cleaned up from - and _ and turned # uppercase, so an encoding not respecting these rules wouldn't be found in the diff --git a/lib/extras.py b/lib/extras.py index b21e223d..a873c4ef 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -965,7 +965,8 @@ def register_composite(name, conn_or_curs, globally=False, factory=None): # expose the json adaptation stuff into the module -from psycopg2._json import json, Json, register_json, register_default_json +from psycopg2._json import json, Json, register_json +from psycopg2._json import register_default_json, register_default_jsonb # Expose range-related objects From f40ad93a3784c49ecfaa1eb6450311c0117b74da Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 01:32:19 +0100 Subject: [PATCH 4/5] Added jsonb docs --- doc/src/extras.rst | 40 +++++++++++++++++++++++++--------------- doc/src/faq.rst | 14 ++++++++++++++ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 0abd3354..36ef0132 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -160,23 +160,27 @@ JSON_ adaptation ^^^^^^^^^^^^^^^^ .. versionadded:: 2.5 +.. versionchanged:: 2.5.4 + added |jsonb| support. In previous versions |jsonb| values are returned + as strings. See :ref:`the FAQ ` for a workaround. -Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ type. -With PostgreSQL 9.2 adaptation is available out-of-the-box. To use JSON data -with previous database versions (either with the `9.1 json extension`__, but -even if you want to convert text fields to JSON) you can use -`register_json()`. +Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ and +|jsonb| types. With PostgreSQL 9.2 and following versions adaptation is +available out-of-the-box. To use JSON data with previous database versions +(either with the `9.1 json extension`__, but even if you want to convert text +fields to JSON) you can use the `register_json()` function. .. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html -The Python library used to convert Python objects to JSON depends on the -language version: with Python 2.6 and following the :py:mod:`json` module from -the standard library is used; with previous versions the `simplejson`_ module -is used if available. Note that the last `!simplejson` version supporting -Python 2.4 is the 2.0.9. +The Python library used by default to convert Python objects to JSON and to +parse data from the database depends on the language version: with Python 2.6 +and following the :py:mod:`json` module from the standard library is used; +with previous versions the `simplejson`_ module is used if available. Note +that the last `!simplejson` version supporting Python 2.4 is the 2.0.9. .. _JSON: http://www.json.org/ .. |pgjson| replace:: :sql:`json` +.. |jsonb| replace:: :sql:`jsonb` .. _pgjson: http://www.postgresql.org/docs/current/static/datatype-json.html .. _simplejson: http://pypi.python.org/pypi/simplejson/ @@ -186,8 +190,8 @@ the `Json` adapter:: curs.execute("insert into mytable (jsondata) values (%s)", [Json({'a': 100})]) -Reading from the database, |pgjson| values will be automatically converted to -Python objects. +Reading from the database, |pgjson| and |jsonb| values will be automatically +converted to Python objects. .. note:: @@ -233,9 +237,11 @@ or you can subclass it overriding the `~Json.dumps()` method:: [MyJson({'a': 100})]) Customizing the conversion from PostgreSQL to Python can be done passing a -custom `!loads()` function to `register_json()` (or `register_default_json()` -for PostgreSQL 9.2). For example, if you want to convert the float values -from :sql:`json` into :py:class:`~decimal.Decimal` you can use:: +custom `!loads()` function to `register_json()`. For the builtin data types +(|pgjson| from PostgreSQL 9.2, |jsonb| from PostgreSQL 9.4) use +`register_default_json()` and `register_default_jsonb()`. For example, if you +want to convert the float values from :sql:`json` into +:py:class:`~decimal.Decimal` you can use:: loads = lambda x: json.loads(x, parse_float=Decimal) psycopg2.extras.register_json(conn, loads=loads) @@ -253,6 +259,10 @@ from :sql:`json` into :py:class:`~decimal.Decimal` you can use:: .. autofunction:: register_default_json +.. autofunction:: register_default_jsonb + + .. versionadded:: 2.5.4 + .. index:: diff --git a/doc/src/faq.rst b/doc/src/faq.rst index fe675231..0646cdff 100644 --- a/doc/src/faq.rst +++ b/doc/src/faq.rst @@ -137,6 +137,20 @@ Psycopg automatically converts PostgreSQL :sql:`json` data into Python objects. See :ref:`adapt-json` for further details. +.. _faq-jsonb-adapt: +.. cssclass:: faq + +Psycopg converts :sql:`json` values into Python objects but :sql:`jsonb` values are returned as strings. Can :sql:`jsonb` be converted automatically? + Automatic conversion of :sql:`jsonb` values is supported from Psycopg + release 2.5.4. For previous versions you can register the :sql:`json` + typecaster on the :sql:`jsonb` oids (which are known and not suppsed to + change in future PostgreSQL versions):: + + psycopg2.extras.register_json(oid=3802, array_oid=3807, globally=True) + + See :ref:`adapt-json` for further details. + + .. _faq-bytea-9.0: .. cssclass:: faq From e225aad042917651cc32a0e198bf0e575ed50d26 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 01:58:28 +0100 Subject: [PATCH 5/5] Habemus jsonb --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 9418d52a..54c1217f 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,7 @@ Bug fixes: What's new in psycopg 2.5.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Added :sql:`jsonb` support for PostgreSQL 9.4. - Fixed segfault if COPY statements are executed instead of using the proper methods (:ticket:`#219`). - Don't ignore silently the `cursor.callproc` argument without a length.