From b8e7f022560343c744e2576da478597a61f01575 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 04:12:20 +0100 Subject: [PATCH 01/11] Added Json adapter --- doc/src/extras.rst | 27 +++++++++++++ lib/extras.py | 55 +++++++++++++++++++++++++ tests/test_types_extras.py | 83 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 161 insertions(+), 4 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index f3f10b12..26fdb126 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -128,6 +128,33 @@ Additional data types --------------------- +.. _adapt-json: + +.. index:: + pair: JSON; Data types + pair: JSON; Adaptation + +JSON adaptation +^^^^^^^^^^^^^^^ + +.. versionadded:: 2.4.6 + +Psycopg can use an underlying JSON_ module implementation to adapt Python +objects to and from the PostgreSQL |pgjson|_ data type. The library used +depends on the Python version: with Python 2.6 and following the +:py:mod:`json` module from the standard library is used; with previous +versions the `simplejson`_ module is be used if available. Note that the last +`!simplejson` version supporting Python 2.4 is the 2.0.9. + +.. _JSON: http://www.json.org/ +.. |pgjson| replace:: :sql:`json` +.. _pgjson: http://www.postgresql.org/docs/current/static/datatype-json.html +.. _simplejson: http://pypi.python.org/pypi/simplejson/ + +.. autoclass:: Json + + + .. _adapt-hstore: .. index:: diff --git a/lib/extras.py b/lib/extras.py index 696a9af6..f3aabb10 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -967,4 +967,59 @@ def register_composite(name, conn_or_curs, globally=False): return caster +# import the best json implementation available +if sys.version_info[:2] >= (2,6): + import json +else: + try: + import simplejson as json + except ImportError: + json = None + + +class Json(object): + """A wrapper to adapt a Python object to :sql:`json` data type. + + `!Json` can be used to wrap any object supported by the underlying + `!json` module. Any keyword argument will be passed to the + underlying :py:func:`json.dumps()` function, allowing extension and + customization. :: + + curs.execute("insert into mytable (jsondata) values (%s)", + (Json({'a': 100}),)) + + .. note:: + + You can use `~psycopg2.extensions.register_adapter()` to adapt Python + dictionaries to JSON:: + + psycopg2.extensions.register_adapter(dict, + psycopg2.extras.Json) + + This setting is global though, so it is not compatible with the use of + `register_hstore()`. Any other object supported by the `!json` library + used by Psycopg can be registered the same way, but this will clobber + the default adaptation rule, so be careful to unwanted side effects. + + """ + def __init__(self, adapted, **kwargs): + self.adapted = adapted + self.kwargs = kwargs + + def __conform__(self, proto): + if proto is _ext.ISQLQuote: + return self + + def getquoted(self): + s = json.dumps(self.adapted, **self.kwargs) + return _ext.QuotedString(s).getquoted() + + +# clobber the above class if json is not available +if json is None: + class Json(Json): + def __init__(self, adapted): + raise ImportError("no json module available") + + __all__ = filter(lambda k: not k.startswith('_'), locals().keys()) diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 49be3908..25a003f5 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -14,12 +14,9 @@ # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. -try: - import decimal -except: - pass import re import sys +from decimal import Decimal from datetime import date from testutils import unittest, skip_if_no_uuid, skip_before_postgres @@ -784,6 +781,84 @@ class AdaptTypeTestCase(unittest.TestCase): return oid +def skip_if_no_json_module(f): + """Skip a test if no Python json module is available""" + def skip_if_no_json_module_(self): + if psycopg2.extras.json is None: + return self.skipTest("json module not available") + + return f(self) + + return skip_if_no_json_module_ + +def skip_if_no_json_type(f): + """Skip a test if PostgreSQL json type is not available""" + def skip_if_no_json_type_(self): + curs = self.conn.cursor() + curs.execute("select oid from pg_type where typname = 'json'") + if not curs.fetchone(): + return self.skipTest("json not available in test database") + + return f(self) + + return skip_if_no_json_type_ + +class JsonTestCase(unittest.TestCase): + def setUp(self): + self.conn = psycopg2.connect(dsn) + + def tearDown(self): + self.conn.close() + + def test_module_not_available(self): + from psycopg2.extras import json, Json + if json is not None: + return self.skipTest("json module is available") + + self.assertRaises(ImportError, Json, None) + + @skip_if_no_json_module + def test_adapt(self): + from psycopg2.extras import json, Json + + objs = [None, "te'xt", 123, 123.45, + u'\xe0\u20ac', ['a', 100], {'a': 100} ] + + curs = self.conn.cursor() + for obj in enumerate(objs): + self.assertEqual(curs.mogrify("%s", (Json(obj),)), + psycopg2.extensions.QuotedString(json.dumps(obj)).getquoted()) + + @skip_if_no_json_module + def test_adapt_extended(self): + """Json passes through kw arguments to dumps""" + from psycopg2.extras import json, Json + + class DecimalEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder.default(self, obj) + + curs = self.conn.cursor() + obj = Decimal('123.45') + self.assertEqual(curs.mogrify("%s", (Json(obj, cls=DecimalEncoder),)), + b("'123.45'")) + + @skip_if_no_json_module + def test_register_on_dict(self): + from psycopg2.extras import Json + psycopg2.extensions.register_adapter(dict, Json) + + try: + curs = self.conn.cursor() + obj = {'a': 123} + self.assertEqual(curs.mogrify("%s", (obj,)), + b("""'{"a": 123}'""")) + finally: + del psycopg2.extensions.adapters[dict, psycopg2.extensions.ISQLQuote] + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 024f0dbada97fa5f9718f7d622dba232ac4ca65a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 04:26:35 +0100 Subject: [PATCH 02/11] Added json typecaster --- doc/src/extras.rst | 2 + lib/_json.py | 187 +++++++++++++++++++++++++++++++++++++ lib/extensions.py | 14 +++ lib/extras.py | 61 ++---------- tests/test_types_extras.py | 92 ++++++++++++++++++ 5 files changed, 301 insertions(+), 55 deletions(-) create mode 100644 lib/_json.py diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 26fdb126..be621faf 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -153,6 +153,8 @@ versions the `simplejson`_ module is be used if available. Note that the last .. autoclass:: Json +.. autofunction:: register_json + .. _adapt-hstore: diff --git a/lib/_json.py b/lib/_json.py new file mode 100644 index 00000000..ec33a6aa --- /dev/null +++ b/lib/_json.py @@ -0,0 +1,187 @@ +"""Implementation of the JSON adaptation objects + +This module exists to avoid a circular import problem: pyscopg2.extras depends +on psycopg2.extension, so I can't create the default JSON typecasters in +extensions importing register_json from extras. +""" + +# psycopg/extras.py - miscellaneous extra goodies for psycopg +# +# Copyright (C) 2012 Daniele Varrazzo +# +# psycopg2 is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# In addition, as a special exception, the copyright holders give +# permission to link this program with the OpenSSL library (or with +# modified versions of OpenSSL that use the same license as OpenSSL), +# and distribute linked combinations including the two. +# +# You must obey the GNU Lesser General Public License in all respects for +# all of the code used other than OpenSSL. +# +# psycopg2 is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +import sys + +from psycopg2._psycopg import ISQLQuote, QuotedString +from psycopg2._psycopg import new_type, new_array_type, register_type + + +# import the best json implementation available +if sys.version_info[:2] >= (2,6): + import json +else: + try: + import simplejson as json + except ImportError: + json = None + + +class Json(object): + """A wrapper to adapt a Python object to :sql:`json` data type. + + `!Json` can be used to wrap any object supported by the underlying + `!json` module. Raise `!ImportError` if no module is available. + + Any keyword argument will be passed to the underlying + :py:func:`json.dumps()` function, allowing extension and customization. :: + + curs.execute("insert into mytable (jsondata) values (%s)", + (Json({'a': 100}),)) + + .. note:: + + You can use `~psycopg2.extensions.register_adapter()` to adapt Python + dictionaries to JSON:: + + psycopg2.extensions.register_adapter(dict, + psycopg2.extras.Json) + + This setting is global though, so it is not compatible with the use of + `register_hstore()`. Any other object supported by the `!json` library + used by Psycopg can be registered the same way, but this will clobber + the default adaptation rule, so be careful to unwanted side effects. + + """ + def __init__(self, adapted, **kwargs): + self.adapted = adapted + self.kwargs = kwargs + + def __conform__(self, proto): + if proto is ISQLQuote: + return self + + def getquoted(self): + s = json.dumps(self.adapted, **self.kwargs) + return QuotedString(s).getquoted() + + +# clobber the above class if json is not available +if json is None: + class Json(Json): + def __init__(self, adapted): + raise ImportError("no json module available") + + +def register_json(conn_or_curs, globally=False, loads=None, + oid=None, array_oid=None): + """Create and register typecasters converting :sql:`json` type to Python objects. + + :param conn_or_curs: a connection or cursor used to find the :sql:`json` + and :sql:`json[]` oids; the typecasters are registered in a scope + limited to this object, unless *globally* is set to `!True`. It can be + `!None` if the oids are provided + :param globally: if `!False` register the typecasters only on + *conn_or_curs*, otherwise register them globally + :param loads: the function used to parse the data into a Python object. If + `!None` use `!json.loads()`, where `!json` is the module chosen + according to the Python version (see above) + :param oid: the OID of the :sql:`json` type if known; If not, it will be + queried on *conn_or_curs* + :param array_oid: the OID of the :sql:`json[]` array type if known; + if not, it will be queried on *conn_or_curs* + + Using the function is required to convert :sql:`json` data in PostgreSQL + versions before 9.2. Since 9.2 the oids are hardcoded so a default + typecaster is already registered. The :sql:`json` type is available as + `extension for PostgreSQL 9.1`__. + + .. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html + + Another use of the function is to adapt :sql:`json` using a customized + load function. For example, if you want to convert the float values in the + :sql:`json` into :py:class:`~decimal.Decimal` you can use:: + + loads = lambda x: json.loads(x, parse_float=Decimal) + psycopg2.extras.register_json(conn, loads=loads) + + The connection or cursor passed to the function will be used to query the + database and look for the OID of the :sql:`json` type. No query is + performed if *oid* and *array_oid* are provided. Raise + `~psycopg2.ProgrammingError` if the type is not found. + + """ + if oid is None: + oid, array_oid = _get_json_oids(conn_or_curs) + + JSON, JSONARRAY = create_json_typecasters(oid, array_oid, loads) + + register_type(JSON, not globally and conn_or_curs or None) + + if JSONARRAY is not None: + register_type(JSONARRAY, not globally and conn_or_curs or None) + + return JSON, JSONARRAY + +def create_json_typecasters(oid, array_oid, loads=None): + """Create typecasters for json data type.""" + if loads is None: + if json is None: + raise ImportError("no json module available") + else: + loads = json.loads + + def typecast_json(s, cur): + return loads(s) + + JSON = new_type((oid, ), 'JSON', typecast_json) + JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + + return JSON, JSONARRAY + +def _get_json_oids(conn_or_curs): + # lazy imports + from psycopg2.extensions import STATUS_IN_TRANSACTION + from psycopg2.extras import _solve_conn_curs + + conn, curs = _solve_conn_curs(conn_or_curs) + + # Store the transaction status of the connection to revert it after use + conn_status = conn.status + + # column typarray not available before PG 8.3 + typarray = conn.server_version >= 80300 and "typarray" or "NULL" + + # get the oid for the hstore + curs.execute( + "SELECT t.oid, %s FROM pg_type t WHERE t.typname = 'json';" + % typarray) + r = curs.fetchone() + + # revert the status of the connection as before the command + if (conn_status != STATUS_IN_TRANSACTION and not conn.autocommit): + conn.rollback() + + if not r: + raise conn.ProgrammingError("json data type not found") + + return r + + + diff --git a/lib/extensions.py b/lib/extensions.py index bb8e3dd6..066373c2 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -150,6 +150,20 @@ class NoneAdapter(object): return _null +# Create default json typecasters for PostgreSQL 9.2 oids +from psycopg2._json import create_json_typecasters + +try: + JSON, JSONARRAY = create_json_typecasters(114, 199) +except ImportError: + pass +else: + register_type(JSON) + register_type(JSONARRAY) + +del create_json_typecasters + + # Add the "cleaned" version of the encodings to the key. # When the encoding is set its name is cleaned up from - and _ and turned # uppercase, so an encoding not respecting these rules wouldn't be found in the diff --git a/lib/extras.py b/lib/extras.py index f3aabb10..643214e2 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -573,6 +573,9 @@ def wait_select(conn): def _solve_conn_curs(conn_or_curs): """Return the connection and a DBAPI cursor from a connection or cursor.""" + if conn_or_curs is None: + raise psycopg2.ProgrammingError("no connection or cursor provided") + if hasattr(conn_or_curs, 'execute'): conn = conn_or_curs.connection curs = conn.cursor(cursor_factory=_cursor) @@ -742,7 +745,6 @@ def register_hstore(conn_or_curs, globally=False, unicode=False, 'hstore'::regtype::oid`. Analogously you can obtain a value for *array_oid* using a query such as :sql:`SELECT 'hstore[]'::regtype::oid`. - Note that, when passing a dictionary from Python to the database, both strings and unicode keys and values are supported. Dictionaries returned from the database have keys/values according to the *unicode* parameter. @@ -967,59 +969,8 @@ def register_composite(name, conn_or_curs, globally=False): return caster -# import the best json implementation available -if sys.version_info[:2] >= (2,6): - import json -else: - try: - import simplejson as json - except ImportError: - json = None - - -class Json(object): - """A wrapper to adapt a Python object to :sql:`json` data type. - - `!Json` can be used to wrap any object supported by the underlying - `!json` module. Any keyword argument will be passed to the - underlying :py:func:`json.dumps()` function, allowing extension and - customization. :: - - curs.execute("insert into mytable (jsondata) values (%s)", - (Json({'a': 100}),)) - - .. note:: - - You can use `~psycopg2.extensions.register_adapter()` to adapt Python - dictionaries to JSON:: - - psycopg2.extensions.register_adapter(dict, - psycopg2.extras.Json) - - This setting is global though, so it is not compatible with the use of - `register_hstore()`. Any other object supported by the `!json` library - used by Psycopg can be registered the same way, but this will clobber - the default adaptation rule, so be careful to unwanted side effects. - - """ - def __init__(self, adapted, **kwargs): - self.adapted = adapted - self.kwargs = kwargs - - def __conform__(self, proto): - if proto is _ext.ISQLQuote: - return self - - def getquoted(self): - s = json.dumps(self.adapted, **self.kwargs) - return _ext.QuotedString(s).getquoted() - - -# clobber the above class if json is not available -if json is None: - class Json(Json): - def __init__(self, adapted): - raise ImportError("no json module available") - +# expose the json adaptation stuff into the module +from psycopg2._json import json, Json, register_json __all__ = filter(lambda k: not k.startswith('_'), locals().keys()) + diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 25a003f5..bc32cd3b 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -859,6 +859,98 @@ class JsonTestCase(unittest.TestCase): del psycopg2.extensions.adapters[dict, psycopg2.extensions.ISQLQuote] + def test_type_not_available(self): + curs = self.conn.cursor() + curs.execute("select oid from pg_type where typname = 'json'") + if curs.fetchone(): + return self.skipTest("json available in test database") + + self.assertRaises(psycopg2.ProgrammingError, + psycopg2.extras.register_json, self.conn) + + @skip_if_no_json_module + @skip_before_postgres(9, 2) + def test_default_cast(self): + curs = self.conn.cursor() + + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None}) + + curs.execute("""select array['{"a": 100.0, "b": null}']::json[]""") + self.assertEqual(curs.fetchone()[0], [{'a': 100.0, 'b': None}]) + + @skip_if_no_json_module + @skip_if_no_json_type + def test_register_on_connection(self): + psycopg2.extras.register_json(self.conn) + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None}) + + @skip_if_no_json_module + @skip_if_no_json_type + def test_register_on_cursor(self): + curs = self.conn.cursor() + psycopg2.extras.register_json(curs) + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None}) + + @skip_if_no_json_module + @skip_if_no_json_type + def test_register_globally(self): + old = psycopg2.extensions.string_types.get(114) + olda = psycopg2.extensions.string_types.get(199) + try: + new, newa = psycopg2.extras.register_json(self.conn, globally=True) + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None}) + finally: + psycopg2.extensions.string_types.pop(new.values[0]) + psycopg2.extensions.string_types.pop(newa.values[0]) + if old: + psycopg2.extensions.register_type(old) + if olda: + psycopg2.extensions.register_type(olda) + + @skip_if_no_json_module + @skip_if_no_json_type + def test_loads(self): + json = psycopg2.extras.json + loads = lambda x: json.loads(x, parse_float=Decimal) + psycopg2.extras.register_json(self.conn, loads=loads) + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], Decimal)) + self.assertEqual(data['a'], Decimal('100.0')) + + @skip_if_no_json_module + @skip_if_no_json_type + def test_no_conn_curs(self): + from psycopg2._json import _get_json_oids + oid, array_oid = _get_json_oids(self.conn) + + old = psycopg2.extensions.string_types.get(114) + olda = psycopg2.extensions.string_types.get(199) + loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) + try: + new, newa = psycopg2.extras.register_json(None, + loads=loads, oid=oid, array_oid=array_oid) + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], Decimal)) + self.assertEqual(data['a'], Decimal('100.0')) + finally: + psycopg2.extensions.string_types.pop(new.values[0]) + psycopg2.extensions.string_types.pop(newa.values[0]) + if old: + psycopg2.extensions.register_type(old) + if olda: + psycopg2.extensions.register_type(olda) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 26d71b4cba31a62bc648bec353aab299adb391a6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 15:31:28 +0100 Subject: [PATCH 03/11] First parameter of register_json defaults to None --- lib/_json.py | 2 +- tests/test_types_extras.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/_json.py b/lib/_json.py index ec33a6aa..09d854b7 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -89,7 +89,7 @@ if json is None: raise ImportError("no json module available") -def register_json(conn_or_curs, globally=False, loads=None, +def register_json(conn_or_curs=None, globally=False, loads=None, oid=None, array_oid=None): """Create and register typecasters converting :sql:`json` type to Python objects. diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index bc32cd3b..b06973a1 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -935,7 +935,7 @@ class JsonTestCase(unittest.TestCase): olda = psycopg2.extensions.string_types.get(199) loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) try: - new, newa = psycopg2.extras.register_json(None, + new, newa = psycopg2.extras.register_json( loads=loads, oid=oid, array_oid=array_oid) curs = self.conn.cursor() curs.execute("""select '{"a": 100.0, "b": null}'::json""") From d963b478e27557dab74028b4fd7caf5ebe5f9f44 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 15:49:00 +0100 Subject: [PATCH 04/11] Added register_default_json() function Register a typecaster for PostgreSQL 9.2 json. --- doc/src/extras.rst | 2 ++ lib/_json.py | 21 +++++++++++++++++++-- lib/extensions.py | 9 +++------ lib/extras.py | 2 +- tests/test_types_extras.py | 18 ++++++++++++++++++ 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index be621faf..8459b199 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -155,6 +155,8 @@ versions the `simplejson`_ module is be used if available. Note that the last .. autofunction:: register_json +.. autofunction:: register_default_json + .. _adapt-hstore: diff --git a/lib/_json.py b/lib/_json.py index 09d854b7..3d211482 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -43,6 +43,10 @@ else: json = None +# oids from PostgreSQL 9.2 +JSON_OID = 114 +JSONARRAY_OID = 199 + class Json(object): """A wrapper to adapt a Python object to :sql:`json` data type. @@ -130,7 +134,7 @@ def register_json(conn_or_curs=None, globally=False, loads=None, if oid is None: oid, array_oid = _get_json_oids(conn_or_curs) - JSON, JSONARRAY = create_json_typecasters(oid, array_oid, loads) + JSON, JSONARRAY = _create_json_typecasters(oid, array_oid, loads) register_type(JSON, not globally and conn_or_curs or None) @@ -139,7 +143,20 @@ def register_json(conn_or_curs=None, globally=False, loads=None, return JSON, JSONARRAY -def create_json_typecasters(oid, array_oid, loads=None): +def register_default_json(conn_or_curs=None, globally=False, loads=None): + """ + Create and register :sql:`json` typecasters for PostgreSQL 9.2 and following. + + Since PostgreSQL 9.2 :sql:`json` is a builtin type, hence its oid is known + and fixed. This function allows specifying a customized *loads* function + for the default :sql:`json` type without querying the database. + All the parameters have the same meaning of `register_json()`. + """ + return register_json(conn_or_curs=conn_or_curs, globally=globally, + loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) + + +def _create_json_typecasters(oid, array_oid, loads=None): """Create typecasters for json data type.""" if loads is None: if json is None: diff --git a/lib/extensions.py b/lib/extensions.py index 066373c2..94dcfb76 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -151,17 +151,14 @@ class NoneAdapter(object): # Create default json typecasters for PostgreSQL 9.2 oids -from psycopg2._json import create_json_typecasters +from psycopg2._json import register_default_json try: - JSON, JSONARRAY = create_json_typecasters(114, 199) + JSON, JSONARRAY = register_default_json() except ImportError: pass -else: - register_type(JSON) - register_type(JSONARRAY) -del create_json_typecasters +del register_default_json # Add the "cleaned" version of the encodings to the key. diff --git a/lib/extras.py b/lib/extras.py index 643214e2..e14e1c4a 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -970,7 +970,7 @@ def register_composite(name, conn_or_curs, globally=False): # expose the json adaptation stuff into the module -from psycopg2._json import json, Json, register_json +from psycopg2._json import json, Json, register_json, register_default_json __all__ = filter(lambda k: not k.startswith('_'), locals().keys()) diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index b06973a1..93c54d49 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -950,6 +950,24 @@ class JsonTestCase(unittest.TestCase): if olda: psycopg2.extensions.register_type(olda) + @skip_if_no_json_module + @skip_before_postgres(9, 2) + def test_register_default(self): + curs = self.conn.cursor() + + loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) + psycopg2.extras.register_default_json(curs, loads=loads) + + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], Decimal)) + self.assertEqual(data['a'], Decimal('100.0')) + + curs.execute("""select array['{"a": 100.0, "b": null}']::json[]""") + data = curs.fetchone()[0] + self.assert_(isinstance(data[0]['a'], Decimal)) + self.assertEqual(data[0]['a'], Decimal('100.0')) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 7386b8327cf4b77666a67e150ce660bf4ff489b6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 16:32:57 +0100 Subject: [PATCH 05/11] Dropped keywords passthrough in Json adapter Pass a dumps function instead. Allow customizing by either arg passing or subclassing. The basic Json class now raises ImportError on getquoted() if json is not available, thus allowing using a customized Json subclass even when the json module is not available. --- doc/src/extras.rst | 1 + lib/_json.py | 78 +++++++++++++++++++++++++++----------- tests/test_types_extras.py | 51 +++++++++++++++++++++---- 3 files changed, 100 insertions(+), 30 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 8459b199..bfe6440c 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -152,6 +152,7 @@ versions the `simplejson`_ module is be used if available. Note that the last .. _simplejson: http://pypi.python.org/pypi/simplejson/ .. autoclass:: Json + :members: dumps .. autofunction:: register_json diff --git a/lib/_json.py b/lib/_json.py index 3d211482..c21d6c86 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -48,51 +48,83 @@ JSON_OID = 114 JSONARRAY_OID = 199 class Json(object): - """A wrapper to adapt a Python object to :sql:`json` data type. + """ + An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to + :sql:`json` data type. `!Json` can be used to wrap any object supported by the underlying - `!json` module. Raise `!ImportError` if no module is available. + `!json` module. `~psycopg2.extensions.ISQLQuote.getquoted()` will raise + `!ImportError` if no module is available. - Any keyword argument will be passed to the underlying - :py:func:`json.dumps()` function, allowing extension and customization. :: + The basic usage is to wrap `!Json` around the object to be adapted:: curs.execute("insert into mytable (jsondata) values (%s)", - (Json({'a': 100}),)) + [Json({'a': 100})]) + + If you want to customize the adaptation from Python to PostgreSQL you can + either provide a custom *dumps* function:: + + curs.execute("insert into mytable (jsondata) values (%s)", + [Json({'a': 100}, dumps=simplejson.dumps)]) + + or you can subclass `!Json` overriding the `dumps()` method:: + + class MyJson(Json): + def dumps(self, obj): + return simplejson.dumps(obj) + + curs.execute("insert into mytable (jsondata) values (%s)", + [MyJson({'a': 100})]) .. note:: - You can use `~psycopg2.extensions.register_adapter()` to adapt Python - dictionaries to JSON:: + You can use `~psycopg2.extensions.register_adapter()` to adapt any + Python dictionary to JSON, either using `!Json` or any subclass or + factory creating a compatible adapter:: - psycopg2.extensions.register_adapter(dict, - psycopg2.extras.Json) + psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) - This setting is global though, so it is not compatible with the use of - `register_hstore()`. Any other object supported by the `!json` library - used by Psycopg can be registered the same way, but this will clobber - the default adaptation rule, so be careful to unwanted side effects. + This setting is global though, so it is not compatible with similar + adapters such as the one registered by `register_hstore()`. Any other + object supported by JSON can be registered the same way, but this will + clobber the default adaptation rule, so be careful to unwanted side + effects. """ - def __init__(self, adapted, **kwargs): + def __init__(self, adapted, dumps=None): self.adapted = adapted - self.kwargs = kwargs + + if dumps is not None: + self._dumps = dumps + elif json is not None: + self._dumps = json.dumps + else: + self._dumps = None def __conform__(self, proto): if proto is ISQLQuote: return self + def dumps(self, obj): + """Serialize *obj* in JSON format. + + The default is to call `!json.dumps()` or the *dumps* function + provided in the constructor. You can override this method to create a + customized JSON wrapper. + """ + dumps = self._dumps + if dumps is not None: + return dumps(obj) + else: + raise ImportError( + "json module not available: " + "you should provide a dumps function") + def getquoted(self): - s = json.dumps(self.adapted, **self.kwargs) + s = self.dumps(self.adapted) return QuotedString(s).getquoted() -# clobber the above class if json is not available -if json is None: - class Json(Json): - def __init__(self, adapted): - raise ImportError("no json module available") - - def register_json(conn_or_curs=None, globally=False, loads=None, oid=None, array_oid=None): """Create and register typecasters converting :sql:`json` type to Python objects. diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 93c54d49..4588718e 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -781,6 +781,16 @@ class AdaptTypeTestCase(unittest.TestCase): return oid +def skip_if_json_module(f): + """Skip a test if no Python json module is available""" + def skip_if_json_module_(self): + if psycopg2.extras.json is not None: + return self.skipTest("json module is available") + + return f(self) + + return skip_if_json_module_ + def skip_if_no_json_module(f): """Skip a test if no Python json module is available""" def skip_if_no_json_module_(self): @@ -810,12 +820,20 @@ class JsonTestCase(unittest.TestCase): def tearDown(self): self.conn.close() + @skip_if_json_module def test_module_not_available(self): - from psycopg2.extras import json, Json - if json is not None: - return self.skipTest("json module is available") + from psycopg2.extras import Json + self.assertRaises(ImportError, Json(None).getquoted) - self.assertRaises(ImportError, Json, None) + @skip_if_json_module + def test_customizable_with_module_not_available(self): + from psycopg2.extras import Json + class MyJson(Json): + def dumps(self, obj): + assert obj is None + return "hi" + + self.assertEqual(MyJson(None).getquoted(), "'hi'") @skip_if_no_json_module def test_adapt(self): @@ -830,8 +848,7 @@ class JsonTestCase(unittest.TestCase): psycopg2.extensions.QuotedString(json.dumps(obj)).getquoted()) @skip_if_no_json_module - def test_adapt_extended(self): - """Json passes through kw arguments to dumps""" + def test_adapt_dumps(self): from psycopg2.extras import json, Json class DecimalEncoder(json.JSONEncoder): @@ -842,7 +859,27 @@ class JsonTestCase(unittest.TestCase): curs = self.conn.cursor() obj = Decimal('123.45') - self.assertEqual(curs.mogrify("%s", (Json(obj, cls=DecimalEncoder),)), + dumps = lambda obj: json.dumps(obj, cls=DecimalEncoder) + self.assertEqual(curs.mogrify("%s", (Json(obj, dumps=dumps),)), + b("'123.45'")) + + @skip_if_no_json_module + def test_adapt_subclass(self): + from psycopg2.extras import json, Json + + class DecimalEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder.default(self, obj) + + class MyJson(Json): + def dumps(self, obj): + return json.dumps(obj, cls=DecimalEncoder) + + curs = self.conn.cursor() + obj = Decimal('123.45') + self.assertEqual(curs.mogrify("%s", (MyJson(obj),)), b("'123.45'")) @skip_if_no_json_module From 9b2ad7abb5a67921cc2cc9e18e37eb6f9aeee4b4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 20 Sep 2012 00:36:53 +0100 Subject: [PATCH 06/11] Fixed json typecaster with NULL input --- lib/_json.py | 2 ++ tests/test_types_extras.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/lib/_json.py b/lib/_json.py index c21d6c86..9142c57f 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -197,6 +197,8 @@ def _create_json_typecasters(oid, array_oid, loads=None): loads = json.loads def typecast_json(s, cur): + if s is None: + return None return loads(s) JSON = new_type((oid, ), 'JSON', typecast_json) diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 4588718e..6af3c4c5 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -1005,6 +1005,16 @@ class JsonTestCase(unittest.TestCase): self.assert_(isinstance(data[0]['a'], Decimal)) self.assertEqual(data[0]['a'], Decimal('100.0')) + @skip_if_no_json_module + @skip_if_no_json_type + def test_null(self): + psycopg2.extras.register_json(self.conn) + curs = self.conn.cursor() + curs.execute("""select NULL::json""") + self.assertEqual(curs.fetchone()[0], None) + curs.execute("""select NULL::json[]""") + self.assertEqual(curs.fetchone()[0], None) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 5645e7adef7644bd1acbe723dd5f0cc36c4815c1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 20 Sep 2012 03:44:50 +0100 Subject: [PATCH 07/11] Pasto fixed --- lib/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_json.py b/lib/_json.py index 9142c57f..e117252e 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -5,7 +5,7 @@ on psycopg2.extension, so I can't create the default JSON typecasters in extensions importing register_json from extras. """ -# psycopg/extras.py - miscellaneous extra goodies for psycopg +# psycopg/_json.py - Implementation of the JSON adaptation objects # # Copyright (C) 2012 Daniele Varrazzo # From a3418052e9fdc1fc05cda4b632957decb0fadc7c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 24 Sep 2012 11:23:09 +0100 Subject: [PATCH 08/11] Don't create/register a json array typecaster if no oid provided --- lib/_json.py | 5 ++++- tests/test_types_extras.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/_json.py b/lib/_json.py index e117252e..f90bac5b 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -202,7 +202,10 @@ def _create_json_typecasters(oid, array_oid, loads=None): return loads(s) JSON = new_type((oid, ), 'JSON', typecast_json) - JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + if array_oid is not None: + JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + else: + JSONARRAY = None return JSON, JSONARRAY diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 6af3c4c5..6ca33e41 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -1015,6 +1015,18 @@ class JsonTestCase(unittest.TestCase): curs.execute("""select NULL::json[]""") self.assertEqual(curs.fetchone()[0], None) + @skip_if_no_json_module + def test_no_array_oid(self): + curs = self.conn.cursor() + t1, t2 = psycopg2.extras.register_json(curs, oid=25) + self.assertEqual(t1.values[0], 25) + self.assertEqual(t2, None) + + curs.execute("""select '{"a": 100.0, "b": null}'::text""") + data = curs.fetchone()[0] + self.assertEqual(data['a'], 100) + self.assertEqual(data['b'], None) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 71e2f190c2c7011dd5fd606d6e81dde0d9a56bf9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 24 Sep 2012 11:51:35 +0100 Subject: [PATCH 09/11] Json documentation improved --- doc/src/extras.rst | 74 ++++++++++++++++++++++++++++++++++++++++------ lib/_json.py | 57 ++++------------------------------- 2 files changed, 70 insertions(+), 61 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index bfe6440c..bf02c606 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -134,25 +134,81 @@ Additional data types pair: JSON; Data types pair: JSON; Adaptation -JSON adaptation -^^^^^^^^^^^^^^^ +JSON_ adaptation +^^^^^^^^^^^^^^^^ .. versionadded:: 2.4.6 -Psycopg can use an underlying JSON_ module implementation to adapt Python -objects to and from the PostgreSQL |pgjson|_ data type. The library used -depends on the Python version: with Python 2.6 and following the -:py:mod:`json` module from the standard library is used; with previous -versions the `simplejson`_ module is be used if available. Note that the last -`!simplejson` version supporting Python 2.4 is the 2.0.9. +Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ type. +With PostgreSQL 9.2 adaptation is available out-of-the-box. To use JSON data +with previous database versions (either with the `9.1 json extension`__, but +even if you want to convert text fields to JSON) you can use +`register_json()`. + +.. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html + +The Python library used to convert Python objects to JSON depends on the +language version: with Python 2.6 and following the :py:mod:`json` module from +the standard library is used; with previous versions the `simplejson`_ module +is used if available. Note that the last `!simplejson` version supporting +Python 2.4 is the 2.0.9. .. _JSON: http://www.json.org/ .. |pgjson| replace:: :sql:`json` .. _pgjson: http://www.postgresql.org/docs/current/static/datatype-json.html .. _simplejson: http://pypi.python.org/pypi/simplejson/ +In order to pass a Python object to the database as query argument you can use +the `Json` adapter:: + + curs.execute("insert into mytable (jsondata) values (%s)", + [Json({'a': 100})]) + +Reading from the database, |pgjson| values will be automatically converted to +Python objects. + +.. note:: + + You can use `~psycopg2.extensions.register_adapter()` to adapt any Python + dictionary to JSON, either registering `Json` or any subclass or factory + creating a compatible adapter:: + + psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) + + This setting is global though, so it is not compatible with similar + adapters such as the one registered by `register_hstore()`. Any other + object supported by JSON can be registered the same way, but this will + clobber the default adaptation rule, so be careful to unwanted side + effects. + +If you want to customize the adaptation from Python to PostgreSQL you can +either provide a custom `!dumps()` function to `!Json`:: + + curs.execute("insert into mytable (jsondata) values (%s)", + [Json({'a': 100}, dumps=simplejson.dumps)]) + +or you can subclass it overriding the `~Json.dumps()` method:: + + class MyJson(Json): + def dumps(self, obj): + return simplejson.dumps(obj) + + curs.execute("insert into mytable (jsondata) values (%s)", + [MyJson({'a': 100})]) + +Customizing the conversion from PostgreSQL to Python can be done passing a +custom `!loads()` function to `register_json()` (or `register_default_json()` +for PostgreSQL 9.2). For example, if you want to convert the float values +from :sql:`json` into :py:class:`~decimal.Decimal` you can use:: + + loads = lambda x: json.loads(x, parse_float=Decimal) + psycopg2.extras.register_json(conn, loads=loads) + + + .. autoclass:: Json - :members: dumps + + .. automethod:: dumps .. autofunction:: register_json diff --git a/lib/_json.py b/lib/_json.py index f90bac5b..7757b6e4 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -52,43 +52,11 @@ class Json(object): An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to :sql:`json` data type. - `!Json` can be used to wrap any object supported by the underlying - `!json` module. `~psycopg2.extensions.ISQLQuote.getquoted()` will raise - `!ImportError` if no module is available. - - The basic usage is to wrap `!Json` around the object to be adapted:: - - curs.execute("insert into mytable (jsondata) values (%s)", - [Json({'a': 100})]) - - If you want to customize the adaptation from Python to PostgreSQL you can - either provide a custom *dumps* function:: - - curs.execute("insert into mytable (jsondata) values (%s)", - [Json({'a': 100}, dumps=simplejson.dumps)]) - - or you can subclass `!Json` overriding the `dumps()` method:: - - class MyJson(Json): - def dumps(self, obj): - return simplejson.dumps(obj) - - curs.execute("insert into mytable (jsondata) values (%s)", - [MyJson({'a': 100})]) - - .. note:: - - You can use `~psycopg2.extensions.register_adapter()` to adapt any - Python dictionary to JSON, either using `!Json` or any subclass or - factory creating a compatible adapter:: - - psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) - - This setting is global though, so it is not compatible with similar - adapters such as the one registered by `register_hstore()`. Any other - object supported by JSON can be registered the same way, but this will - clobber the default adaptation rule, so be careful to unwanted side - effects. + `!Json` can be used to wrap any object supported by the provided *dumps* + function. If none is provided, the standard :py:func:`json.dumps()` is + used (`!simplejson` for Python < 2.6; + `~psycopg2.extensions.ISQLQuote.getquoted()` will raise `!ImportError` if + the module is available). """ def __init__(self, adapted, dumps=None): @@ -143,20 +111,6 @@ def register_json(conn_or_curs=None, globally=False, loads=None, :param array_oid: the OID of the :sql:`json[]` array type if known; if not, it will be queried on *conn_or_curs* - Using the function is required to convert :sql:`json` data in PostgreSQL - versions before 9.2. Since 9.2 the oids are hardcoded so a default - typecaster is already registered. The :sql:`json` type is available as - `extension for PostgreSQL 9.1`__. - - .. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html - - Another use of the function is to adapt :sql:`json` using a customized - load function. For example, if you want to convert the float values in the - :sql:`json` into :py:class:`~decimal.Decimal` you can use:: - - loads = lambda x: json.loads(x, parse_float=Decimal) - psycopg2.extras.register_json(conn, loads=loads) - The connection or cursor passed to the function will be used to query the database and look for the OID of the :sql:`json` type. No query is performed if *oid* and *array_oid* are provided. Raise @@ -187,7 +141,6 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) - def _create_json_typecasters(oid, array_oid, loads=None): """Create typecasters for json data type.""" if loads is None: From b894539007ff0107b4a5295540e0fad112e11e92 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 25 Sep 2012 00:12:57 +0100 Subject: [PATCH 10/11] Fixed json doc error --- lib/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_json.py b/lib/_json.py index 7757b6e4..536dd58b 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -56,7 +56,7 @@ class Json(object): function. If none is provided, the standard :py:func:`json.dumps()` is used (`!simplejson` for Python < 2.6; `~psycopg2.extensions.ISQLQuote.getquoted()` will raise `!ImportError` if - the module is available). + the module is not available). """ def __init__(self, adapted, dumps=None): From 465b5cf280f6fe434d2f2fdd5d75d9c28723bac5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 27 Sep 2012 00:40:35 +0100 Subject: [PATCH 11/11] Mention JSON adaptation in the NEWS file --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 679486bc..13d8c5c3 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,7 @@ What's new in psycopg 2.4.6 --------------------------- + - Added JSON adaptation. - Added support for backward scrollable cursors. Thanks to Jon Nelson for the initial patch (ticket #108). - connection.reset() implemented using DISCARD ALL on server versions