Merge branch 'json' into devel

This commit is contained in:
Daniele Varrazzo 2012-09-27 00:41:04 +01:00
commit dda24f082f
6 changed files with 549 additions and 4 deletions

1
NEWS
View File

@ -1,6 +1,7 @@
What's new in psycopg 2.4.6
---------------------------
- Added JSON adaptation.
- Added support for backward scrollable cursors. Thanks to Jon Nelson
for the initial patch (ticket #108).
- Added a simple way to customize casting of composite types into Python

View File

@ -128,6 +128,94 @@ Additional data types
---------------------
.. _adapt-json:
.. index::
pair: JSON; Data types
pair: JSON; Adaptation
JSON_ adaptation
^^^^^^^^^^^^^^^^
.. versionadded:: 2.4.6
Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ type.
With PostgreSQL 9.2 adaptation is available out-of-the-box. To use JSON data
with previous database versions (either with the `9.1 json extension`__, but
even if you want to convert text fields to JSON) you can use
`register_json()`.
.. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html
The Python library used to convert Python objects to JSON depends on the
language version: with Python 2.6 and following the :py:mod:`json` module from
the standard library is used; with previous versions the `simplejson`_ module
is used if available. Note that the last `!simplejson` version supporting
Python 2.4 is the 2.0.9.
.. _JSON: http://www.json.org/
.. |pgjson| replace:: :sql:`json`
.. _pgjson: http://www.postgresql.org/docs/current/static/datatype-json.html
.. _simplejson: http://pypi.python.org/pypi/simplejson/
In order to pass a Python object to the database as query argument you can use
the `Json` adapter::
curs.execute("insert into mytable (jsondata) values (%s)",
[Json({'a': 100})])
Reading from the database, |pgjson| values will be automatically converted to
Python objects.
.. note::
You can use `~psycopg2.extensions.register_adapter()` to adapt any Python
dictionary to JSON, either registering `Json` or any subclass or factory
creating a compatible adapter::
psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)
This setting is global though, so it is not compatible with similar
adapters such as the one registered by `register_hstore()`. Any other
object supported by JSON can be registered the same way, but this will
clobber the default adaptation rule, so be careful to unwanted side
effects.
If you want to customize the adaptation from Python to PostgreSQL you can
either provide a custom `!dumps()` function to `!Json`::
curs.execute("insert into mytable (jsondata) values (%s)",
[Json({'a': 100}, dumps=simplejson.dumps)])
or you can subclass it overriding the `~Json.dumps()` method::
class MyJson(Json):
def dumps(self, obj):
return simplejson.dumps(obj)
curs.execute("insert into mytable (jsondata) values (%s)",
[MyJson({'a': 100})])
Customizing the conversion from PostgreSQL to Python can be done passing a
custom `!loads()` function to `register_json()` (or `register_default_json()`
for PostgreSQL 9.2). For example, if you want to convert the float values
from :sql:`json` into :py:class:`~decimal.Decimal` you can use::
loads = lambda x: json.loads(x, parse_float=Decimal)
psycopg2.extras.register_json(conn, loads=loads)
.. autoclass:: Json
.. automethod:: dumps
.. autofunction:: register_json
.. autofunction:: register_default_json
.. _adapt-hstore:
.. index::

194
lib/_json.py Normal file
View File

@ -0,0 +1,194 @@
"""Implementation of the JSON adaptation objects
This module exists to avoid a circular import problem: pyscopg2.extras depends
on psycopg2.extension, so I can't create the default JSON typecasters in
extensions importing register_json from extras.
"""
# psycopg/_json.py - Implementation of the JSON adaptation objects
#
# Copyright (C) 2012 Daniele Varrazzo <daniele.varrazzo@gmail.com>
#
# psycopg2 is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# In addition, as a special exception, the copyright holders give
# permission to link this program with the OpenSSL library (or with
# modified versions of OpenSSL that use the same license as OpenSSL),
# and distribute linked combinations including the two.
#
# You must obey the GNU Lesser General Public License in all respects for
# all of the code used other than OpenSSL.
#
# psycopg2 is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
# License for more details.
import sys
from psycopg2._psycopg import ISQLQuote, QuotedString
from psycopg2._psycopg import new_type, new_array_type, register_type
# import the best json implementation available
if sys.version_info[:2] >= (2,6):
import json
else:
try:
import simplejson as json
except ImportError:
json = None
# oids from PostgreSQL 9.2
JSON_OID = 114
JSONARRAY_OID = 199
class Json(object):
"""
An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to
:sql:`json` data type.
`!Json` can be used to wrap any object supported by the provided *dumps*
function. If none is provided, the standard :py:func:`json.dumps()` is
used (`!simplejson` for Python < 2.6;
`~psycopg2.extensions.ISQLQuote.getquoted()` will raise `!ImportError` if
the module is not available).
"""
def __init__(self, adapted, dumps=None):
self.adapted = adapted
if dumps is not None:
self._dumps = dumps
elif json is not None:
self._dumps = json.dumps
else:
self._dumps = None
def __conform__(self, proto):
if proto is ISQLQuote:
return self
def dumps(self, obj):
"""Serialize *obj* in JSON format.
The default is to call `!json.dumps()` or the *dumps* function
provided in the constructor. You can override this method to create a
customized JSON wrapper.
"""
dumps = self._dumps
if dumps is not None:
return dumps(obj)
else:
raise ImportError(
"json module not available: "
"you should provide a dumps function")
def getquoted(self):
s = self.dumps(self.adapted)
return QuotedString(s).getquoted()
def register_json(conn_or_curs=None, globally=False, loads=None,
oid=None, array_oid=None):
"""Create and register typecasters converting :sql:`json` type to Python objects.
:param conn_or_curs: a connection or cursor used to find the :sql:`json`
and :sql:`json[]` oids; the typecasters are registered in a scope
limited to this object, unless *globally* is set to `!True`. It can be
`!None` if the oids are provided
:param globally: if `!False` register the typecasters only on
*conn_or_curs*, otherwise register them globally
:param loads: the function used to parse the data into a Python object. If
`!None` use `!json.loads()`, where `!json` is the module chosen
according to the Python version (see above)
:param oid: the OID of the :sql:`json` type if known; If not, it will be
queried on *conn_or_curs*
:param array_oid: the OID of the :sql:`json[]` array type if known;
if not, it will be queried on *conn_or_curs*
The connection or cursor passed to the function will be used to query the
database and look for the OID of the :sql:`json` type. No query is
performed if *oid* and *array_oid* are provided. Raise
`~psycopg2.ProgrammingError` if the type is not found.
"""
if oid is None:
oid, array_oid = _get_json_oids(conn_or_curs)
JSON, JSONARRAY = _create_json_typecasters(oid, array_oid, loads)
register_type(JSON, not globally and conn_or_curs or None)
if JSONARRAY is not None:
register_type(JSONARRAY, not globally and conn_or_curs or None)
return JSON, JSONARRAY
def register_default_json(conn_or_curs=None, globally=False, loads=None):
"""
Create and register :sql:`json` typecasters for PostgreSQL 9.2 and following.
Since PostgreSQL 9.2 :sql:`json` is a builtin type, hence its oid is known
and fixed. This function allows specifying a customized *loads* function
for the default :sql:`json` type without querying the database.
All the parameters have the same meaning of `register_json()`.
"""
return register_json(conn_or_curs=conn_or_curs, globally=globally,
loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID)
def _create_json_typecasters(oid, array_oid, loads=None):
"""Create typecasters for json data type."""
if loads is None:
if json is None:
raise ImportError("no json module available")
else:
loads = json.loads
def typecast_json(s, cur):
if s is None:
return None
return loads(s)
JSON = new_type((oid, ), 'JSON', typecast_json)
if array_oid is not None:
JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON)
else:
JSONARRAY = None
return JSON, JSONARRAY
def _get_json_oids(conn_or_curs):
# lazy imports
from psycopg2.extensions import STATUS_IN_TRANSACTION
from psycopg2.extras import _solve_conn_curs
conn, curs = _solve_conn_curs(conn_or_curs)
# Store the transaction status of the connection to revert it after use
conn_status = conn.status
# column typarray not available before PG 8.3
typarray = conn.server_version >= 80300 and "typarray" or "NULL"
# get the oid for the hstore
curs.execute(
"SELECT t.oid, %s FROM pg_type t WHERE t.typname = 'json';"
% typarray)
r = curs.fetchone()
# revert the status of the connection as before the command
if (conn_status != STATUS_IN_TRANSACTION and not conn.autocommit):
conn.rollback()
if not r:
raise conn.ProgrammingError("json data type not found")
return r

View File

@ -151,6 +151,17 @@ class NoneAdapter(object):
return _null
# Create default json typecasters for PostgreSQL 9.2 oids
from psycopg2._json import register_default_json
try:
JSON, JSONARRAY = register_default_json()
except ImportError:
pass
del register_default_json
# Add the "cleaned" version of the encodings to the key.
# When the encoding is set its name is cleaned up from - and _ and turned
# uppercase, so an encoding not respecting these rules wouldn't be found in the

View File

@ -573,6 +573,9 @@ def wait_select(conn):
def _solve_conn_curs(conn_or_curs):
"""Return the connection and a DBAPI cursor from a connection or cursor."""
if conn_or_curs is None:
raise psycopg2.ProgrammingError("no connection or cursor provided")
if hasattr(conn_or_curs, 'execute'):
conn = conn_or_curs.connection
curs = conn.cursor(cursor_factory=_cursor)
@ -946,4 +949,8 @@ def register_composite(name, conn_or_curs, globally=False, factory=None):
return caster
# expose the json adaptation stuff into the module
from psycopg2._json import json, Json, register_json, register_default_json
__all__ = filter(lambda k: not k.startswith('_'), locals().keys())

View File

@ -14,12 +14,9 @@
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
# License for more details.
try:
import decimal
except:
pass
import re
import sys
from decimal import Decimal
from datetime import date
from testutils import unittest, skip_if_no_uuid, skip_before_postgres
@ -811,6 +808,253 @@ class AdaptTypeTestCase(unittest.TestCase):
return oid
def skip_if_json_module(f):
"""Skip a test if no Python json module is available"""
def skip_if_json_module_(self):
if psycopg2.extras.json is not None:
return self.skipTest("json module is available")
return f(self)
return skip_if_json_module_
def skip_if_no_json_module(f):
"""Skip a test if no Python json module is available"""
def skip_if_no_json_module_(self):
if psycopg2.extras.json is None:
return self.skipTest("json module not available")
return f(self)
return skip_if_no_json_module_
def skip_if_no_json_type(f):
"""Skip a test if PostgreSQL json type is not available"""
def skip_if_no_json_type_(self):
curs = self.conn.cursor()
curs.execute("select oid from pg_type where typname = 'json'")
if not curs.fetchone():
return self.skipTest("json not available in test database")
return f(self)
return skip_if_no_json_type_
class JsonTestCase(unittest.TestCase):
def setUp(self):
self.conn = psycopg2.connect(dsn)
def tearDown(self):
self.conn.close()
@skip_if_json_module
def test_module_not_available(self):
from psycopg2.extras import Json
self.assertRaises(ImportError, Json(None).getquoted)
@skip_if_json_module
def test_customizable_with_module_not_available(self):
from psycopg2.extras import Json
class MyJson(Json):
def dumps(self, obj):
assert obj is None
return "hi"
self.assertEqual(MyJson(None).getquoted(), "'hi'")
@skip_if_no_json_module
def test_adapt(self):
from psycopg2.extras import json, Json
objs = [None, "te'xt", 123, 123.45,
u'\xe0\u20ac', ['a', 100], {'a': 100} ]
curs = self.conn.cursor()
for obj in enumerate(objs):
self.assertEqual(curs.mogrify("%s", (Json(obj),)),
psycopg2.extensions.QuotedString(json.dumps(obj)).getquoted())
@skip_if_no_json_module
def test_adapt_dumps(self):
from psycopg2.extras import json, Json
class DecimalEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Decimal):
return float(obj)
return json.JSONEncoder.default(self, obj)
curs = self.conn.cursor()
obj = Decimal('123.45')
dumps = lambda obj: json.dumps(obj, cls=DecimalEncoder)
self.assertEqual(curs.mogrify("%s", (Json(obj, dumps=dumps),)),
b("'123.45'"))
@skip_if_no_json_module
def test_adapt_subclass(self):
from psycopg2.extras import json, Json
class DecimalEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Decimal):
return float(obj)
return json.JSONEncoder.default(self, obj)
class MyJson(Json):
def dumps(self, obj):
return json.dumps(obj, cls=DecimalEncoder)
curs = self.conn.cursor()
obj = Decimal('123.45')
self.assertEqual(curs.mogrify("%s", (MyJson(obj),)),
b("'123.45'"))
@skip_if_no_json_module
def test_register_on_dict(self):
from psycopg2.extras import Json
psycopg2.extensions.register_adapter(dict, Json)
try:
curs = self.conn.cursor()
obj = {'a': 123}
self.assertEqual(curs.mogrify("%s", (obj,)),
b("""'{"a": 123}'"""))
finally:
del psycopg2.extensions.adapters[dict, psycopg2.extensions.ISQLQuote]
def test_type_not_available(self):
curs = self.conn.cursor()
curs.execute("select oid from pg_type where typname = 'json'")
if curs.fetchone():
return self.skipTest("json available in test database")
self.assertRaises(psycopg2.ProgrammingError,
psycopg2.extras.register_json, self.conn)
@skip_if_no_json_module
@skip_before_postgres(9, 2)
def test_default_cast(self):
curs = self.conn.cursor()
curs.execute("""select '{"a": 100.0, "b": null}'::json""")
self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None})
curs.execute("""select array['{"a": 100.0, "b": null}']::json[]""")
self.assertEqual(curs.fetchone()[0], [{'a': 100.0, 'b': None}])
@skip_if_no_json_module
@skip_if_no_json_type
def test_register_on_connection(self):
psycopg2.extras.register_json(self.conn)
curs = self.conn.cursor()
curs.execute("""select '{"a": 100.0, "b": null}'::json""")
self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None})
@skip_if_no_json_module
@skip_if_no_json_type
def test_register_on_cursor(self):
curs = self.conn.cursor()
psycopg2.extras.register_json(curs)
curs.execute("""select '{"a": 100.0, "b": null}'::json""")
self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None})
@skip_if_no_json_module
@skip_if_no_json_type
def test_register_globally(self):
old = psycopg2.extensions.string_types.get(114)
olda = psycopg2.extensions.string_types.get(199)
try:
new, newa = psycopg2.extras.register_json(self.conn, globally=True)
curs = self.conn.cursor()
curs.execute("""select '{"a": 100.0, "b": null}'::json""")
self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None})
finally:
psycopg2.extensions.string_types.pop(new.values[0])
psycopg2.extensions.string_types.pop(newa.values[0])
if old:
psycopg2.extensions.register_type(old)
if olda:
psycopg2.extensions.register_type(olda)
@skip_if_no_json_module
@skip_if_no_json_type
def test_loads(self):
json = psycopg2.extras.json
loads = lambda x: json.loads(x, parse_float=Decimal)
psycopg2.extras.register_json(self.conn, loads=loads)
curs = self.conn.cursor()
curs.execute("""select '{"a": 100.0, "b": null}'::json""")
data = curs.fetchone()[0]
self.assert_(isinstance(data['a'], Decimal))
self.assertEqual(data['a'], Decimal('100.0'))
@skip_if_no_json_module
@skip_if_no_json_type
def test_no_conn_curs(self):
from psycopg2._json import _get_json_oids
oid, array_oid = _get_json_oids(self.conn)
old = psycopg2.extensions.string_types.get(114)
olda = psycopg2.extensions.string_types.get(199)
loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal)
try:
new, newa = psycopg2.extras.register_json(
loads=loads, oid=oid, array_oid=array_oid)
curs = self.conn.cursor()
curs.execute("""select '{"a": 100.0, "b": null}'::json""")
data = curs.fetchone()[0]
self.assert_(isinstance(data['a'], Decimal))
self.assertEqual(data['a'], Decimal('100.0'))
finally:
psycopg2.extensions.string_types.pop(new.values[0])
psycopg2.extensions.string_types.pop(newa.values[0])
if old:
psycopg2.extensions.register_type(old)
if olda:
psycopg2.extensions.register_type(olda)
@skip_if_no_json_module
@skip_before_postgres(9, 2)
def test_register_default(self):
curs = self.conn.cursor()
loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal)
psycopg2.extras.register_default_json(curs, loads=loads)
curs.execute("""select '{"a": 100.0, "b": null}'::json""")
data = curs.fetchone()[0]
self.assert_(isinstance(data['a'], Decimal))
self.assertEqual(data['a'], Decimal('100.0'))
curs.execute("""select array['{"a": 100.0, "b": null}']::json[]""")
data = curs.fetchone()[0]
self.assert_(isinstance(data[0]['a'], Decimal))
self.assertEqual(data[0]['a'], Decimal('100.0'))
@skip_if_no_json_module
@skip_if_no_json_type
def test_null(self):
psycopg2.extras.register_json(self.conn)
curs = self.conn.cursor()
curs.execute("""select NULL::json""")
self.assertEqual(curs.fetchone()[0], None)
curs.execute("""select NULL::json[]""")
self.assertEqual(curs.fetchone()[0], None)
@skip_if_no_json_module
def test_no_array_oid(self):
curs = self.conn.cursor()
t1, t2 = psycopg2.extras.register_json(curs, oid=25)
self.assertEqual(t1.values[0], 25)
self.assertEqual(t2, None)
curs.execute("""select '{"a": 100.0, "b": null}'::text""")
data = curs.fetchone()[0]
self.assertEqual(data['a'], 100)
self.assertEqual(data['b'], None)
def test_suite():
return unittest.TestLoader().loadTestsFromName(__name__)