Merge branch 'jsonb' into maint_2_5

NEWS

@@ -4,6 +4,7 @@ Current release
 What's new in psycopg 2.5.4
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+- Added :sql:`jsonb` support for PostgreSQL 9.4.
 - Fixed segfault if COPY statements are executed instead of using the
   proper methods (:ticket:`#219`).
 - Don't ignore silently the `cursor.callproc` argument without a length.

doc/src/extras.rst

@@ -160,23 +160,27 @@ JSON_ adaptation
 ^^^^^^^^^^^^^^^^
 
 .. versionadded:: 2.5
+.. versionchanged:: 2.5.4
+    added |jsonb| support. In previous versions |jsonb| values are returned
+    as strings. See :ref:`the FAQ <faq-jsonb-adapt>` for a workaround.
 
-Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ type.
+Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ and
-With PostgreSQL 9.2 adaptation is available out-of-the-box. To use JSON data
+|jsonb| types.  With PostgreSQL 9.2 and following versions adaptation is
-with previous database versions (either with the `9.1 json extension`__, but
+available out-of-the-box. To use JSON data with previous database versions
-even if you want to convert text fields to JSON) you can use
+(either with the `9.1 json extension`__, but even if you want to convert text
-`register_json()`.
+fields to JSON) you can use the `register_json()` function.
 
 .. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html
 
-The Python library used to convert Python objects to JSON depends on the
+The Python library used by default to convert Python objects to JSON and to
-language version: with Python 2.6 and following the :py:mod:`json` module from
+parse data from the database depends on the language version: with Python 2.6
-the standard library is used; with previous versions the `simplejson`_ module
+and following the :py:mod:`json` module from the standard library is used;
-is used if available. Note that the last `!simplejson` version supporting
+with previous versions the `simplejson`_ module is used if available. Note
-Python 2.4 is the 2.0.9.
+that the last `!simplejson` version supporting Python 2.4 is the 2.0.9.
 
 .. _JSON: http://www.json.org/
 .. |pgjson| replace:: :sql:`json`
+.. |jsonb| replace:: :sql:`jsonb`
 .. _pgjson: http://www.postgresql.org/docs/current/static/datatype-json.html
 .. _simplejson: http://pypi.python.org/pypi/simplejson/
 
@@ -186,8 +190,8 @@ the `Json` adapter::
     curs.execute("insert into mytable (jsondata) values (%s)",
         [Json({'a': 100})])
 
-Reading from the database, |pgjson| values will be automatically converted to
+Reading from the database, |pgjson| and |jsonb| values will be automatically
-Python objects.
+converted to Python objects.
 
 .. note::
 
@@ -233,9 +237,11 @@ or you can subclass it overriding the `~Json.dumps()` method::
         [MyJson({'a': 100})])
 
 Customizing the conversion from PostgreSQL to Python can be done passing a
-custom `!loads()` function to `register_json()` (or `register_default_json()`
+custom `!loads()` function to `register_json()`.  For the builtin data types
-for PostgreSQL 9.2).  For example, if you want to convert the float values
+(|pgjson| from PostgreSQL 9.2, |jsonb| from PostgreSQL 9.4) use
-from :sql:`json` into :py:class:`~decimal.Decimal` you can use::
+`register_default_json()` and `register_default_jsonb()`.  For example, if you
+want to convert the float values from :sql:`json` into
+:py:class:`~decimal.Decimal` you can use::
 
     loads = lambda x: json.loads(x, parse_float=Decimal)
     psycopg2.extras.register_json(conn, loads=loads)
@@ -248,8 +254,15 @@ from :sql:`json` into :py:class:`~decimal.Decimal` you can use::
 
 .. autofunction:: register_json
 
+    .. versionchanged:: 2.5.4
+        added the *name* parameter to enable :sql:`jsonb` support.
+
 .. autofunction:: register_default_json
 
+.. autofunction:: register_default_jsonb
+
+    .. versionadded:: 2.5.4
+
 
 
 .. index::

doc/src/faq.rst

@@ -137,6 +137,20 @@ Psycopg automatically converts PostgreSQL :sql:`json` data into Python objects.
     See :ref:`adapt-json` for further details.
 
 
+.. _faq-jsonb-adapt:
+.. cssclass:: faq
+
+Psycopg converts :sql:`json` values into Python objects but :sql:`jsonb` values are returned as strings. Can :sql:`jsonb` be converted automatically?
+    Automatic conversion of :sql:`jsonb` values is supported from Psycopg
+    release 2.5.4. For previous versions you can register the :sql:`json`
+    typecaster on the :sql:`jsonb` oids (which are known and not suppsed to
+    change in future PostgreSQL versions)::
+
+        psycopg2.extras.register_json(oid=3802, array_oid=3807, globally=True)
+
+    See :ref:`adapt-json` for further details.
+
+
 .. _faq-bytea-9.0:
 .. cssclass:: faq
 

lib/_json.py

@@ -47,6 +47,10 @@ else:
 JSON_OID = 114
 JSONARRAY_OID = 199
 
+# oids from PostgreSQL 9.4
+JSONB_OID = 3802
+JSONBARRAY_OID = 3807
+
 class Json(object):
     """
     An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to
@@ -94,7 +98,7 @@ class Json(object):
 
 
 def register_json(conn_or_curs=None, globally=False, loads=None,
-        oid=None, array_oid=None):
+        oid=None, array_oid=None, name='json'):
     """Create and register typecasters converting :sql:`json` type to Python objects.
 
     :param conn_or_curs: a connection or cursor used to find the :sql:`json`
@@ -110,17 +114,19 @@ def register_json(conn_or_curs=None, globally=False, loads=None,
         queried on *conn_or_curs*
     :param array_oid: the OID of the :sql:`json[]` array type if known;
         if not, it will be queried on *conn_or_curs*
+    :param name: the name of the data type to look for in *conn_or_curs*
 
     The connection or cursor passed to the function will be used to query the
-    database and look for the OID of the :sql:`json` type. No query is
+    database and look for the OID of the :sql:`json` type (or an alternative
-    performed if *oid* and *array_oid* are provided.  Raise
+    type if *name* if provided). No query is performed if *oid* and *array_oid*
-    `~psycopg2.ProgrammingError` if the type is not found.
+    are provided.  Raise `~psycopg2.ProgrammingError` if the type is not found.
 
     """
     if oid is None:
-        oid, array_oid = _get_json_oids(conn_or_curs)
+        oid, array_oid = _get_json_oids(conn_or_curs, name)
 
-    JSON, JSONARRAY = _create_json_typecasters(oid, array_oid, loads)
+    JSON, JSONARRAY = _create_json_typecasters(
+        oid, array_oid, loads=loads, name=name.upper())
 
     register_type(JSON, not globally and conn_or_curs or None)
 
@@ -141,7 +147,19 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None):
     return register_json(conn_or_curs=conn_or_curs, globally=globally,
         loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID)
 
-def _create_json_typecasters(oid, array_oid, loads=None):
+def register_default_jsonb(conn_or_curs=None, globally=False, loads=None):
+    """
+    Create and register :sql:`jsonb` typecasters for PostgreSQL 9.4 and following.
+
+    As in `register_default_json()`, the function allows to register a
+    customized *loads* function for the :sql:`jsonb` type at its known oid for
+    PostgreSQL 9.4 and following versions.  All the parameters have the same
+    meaning of `register_json()`.
+    """
+    return register_json(conn_or_curs=conn_or_curs, globally=globally,
+        loads=loads, oid=JSONB_OID, array_oid=JSONBARRAY_OID, name='jsonb')
+
+def _create_json_typecasters(oid, array_oid, loads=None, name='JSON'):
     """Create typecasters for json data type."""
     if loads is None:
         if json is None:
@@ -154,15 +172,15 @@ def _create_json_typecasters(oid, array_oid, loads=None):
             return None
         return loads(s)
 
-    JSON = new_type((oid, ), 'JSON', typecast_json)
+    JSON = new_type((oid, ), name, typecast_json)
     if array_oid is not None:
-        JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON)
+        JSONARRAY = new_array_type((array_oid, ), "%sARRAY" % name, JSON)
     else:
         JSONARRAY = None
 
     return JSON, JSONARRAY
 
-def _get_json_oids(conn_or_curs):
+def _get_json_oids(conn_or_curs, name='json'):
     # lazy imports
     from psycopg2.extensions import STATUS_IN_TRANSACTION
     from psycopg2.extras import _solve_conn_curs
@@ -177,8 +195,8 @@ def _get_json_oids(conn_or_curs):
 
     # get the oid for the hstore
     curs.execute(
-        "SELECT t.oid, %s FROM pg_type t WHERE t.typname = 'json';"
+        "SELECT t.oid, %s FROM pg_type t WHERE t.typname = %%s;"
-            % typarray)
+            % typarray, (name,))
     r = curs.fetchone()
 
     # revert the status of the connection as before the command
@@ -186,7 +204,7 @@ def _get_json_oids(conn_or_curs):
         conn.rollback()
 
     if not r:
-        raise conn.ProgrammingError("json data type not found")
+        raise conn.ProgrammingError("%s data type not found" % name)
 
     return r
 

lib/extensions.py

@@ -152,20 +152,22 @@ class NoneAdapter(object):
 
 
 # Create default json typecasters for PostgreSQL 9.2 oids
-from psycopg2._json import register_default_json
+from psycopg2._json import register_default_json, register_default_jsonb
 
 try:
     JSON, JSONARRAY = register_default_json()
+    JSONB, JSONBARRAY = register_default_jsonb()
 except ImportError:
     pass
 
-del register_default_json
+del register_default_json, register_default_jsonb
 
 
 # Create default Range typecasters
 from psycopg2. _range import Range
 del Range
 
+
 # Add the "cleaned" version of the encodings to the key.
 # When the encoding is set its name is cleaned up from - and _ and turned
 # uppercase, so an encoding not respecting these rules wouldn't be found in the

lib/extras.py

@@ -965,7 +965,8 @@ def register_composite(name, conn_or_curs, globally=False, factory=None):
 
 
 # expose the json adaptation stuff into the module
-from psycopg2._json import json, Json, register_json, register_default_json
+from psycopg2._json import json, Json, register_json
+from psycopg2._json import register_default_json, register_default_jsonb
 
 
 # Expose range-related objects

tests/test_types_extras.py

@@ -1058,6 +1058,97 @@ class JsonTestCase(ConnectingTestCase):
         self.assertEqual(data['b'], None)
 
 
+def skip_if_no_jsonb_type(f):
+    return skip_before_postgres(9, 4)(f)
+
+class JsonbTestCase(ConnectingTestCase):
+    @staticmethod
+    def myloads(s):
+        import json
+        rv = json.loads(s)
+        rv['test'] = 1
+        return rv
+
+    def test_default_cast(self):
+        curs = self.conn.cursor()
+
+        curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""")
+        self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None})
+
+        curs.execute("""select array['{"a": 100.0, "b": null}']::jsonb[]""")
+        self.assertEqual(curs.fetchone()[0], [{'a': 100.0, 'b': None}])
+
+    def test_register_on_connection(self):
+        psycopg2.extras.register_json(self.conn, loads=self.myloads, name='jsonb')
+        curs = self.conn.cursor()
+        curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""")
+        self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1})
+
+    def test_register_on_cursor(self):
+        curs = self.conn.cursor()
+        psycopg2.extras.register_json(curs, loads=self.myloads, name='jsonb')
+        curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""")
+        self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1})
+
+    def test_register_globally(self):
+        old = psycopg2.extensions.string_types.get(3802)
+        olda = psycopg2.extensions.string_types.get(3807)
+        try:
+            new, newa = psycopg2.extras.register_json(self.conn,
+                loads=self.myloads, globally=True, name='jsonb')
+            curs = self.conn.cursor()
+            curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""")
+            self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1})
+        finally:
+            psycopg2.extensions.string_types.pop(new.values[0])
+            psycopg2.extensions.string_types.pop(newa.values[0])
+            if old:
+                psycopg2.extensions.register_type(old)
+            if olda:
+                psycopg2.extensions.register_type(olda)
+
+    def test_loads(self):
+        json = psycopg2.extras.json
+        loads = lambda x: json.loads(x, parse_float=Decimal)
+        psycopg2.extras.register_json(self.conn, loads=loads, name='jsonb')
+        curs = self.conn.cursor()
+        curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""")
+        data = curs.fetchone()[0]
+        self.assert_(isinstance(data['a'], Decimal))
+        self.assertEqual(data['a'], Decimal('100.0'))
+        # sure we are not manling json too?
+        curs.execute("""select '{"a": 100.0, "b": null}'::json""")
+        data = curs.fetchone()[0]
+        self.assert_(isinstance(data['a'], float))
+        self.assertEqual(data['a'], 100.0)
+
+    def test_register_default(self):
+        curs = self.conn.cursor()
+
+        loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal)
+        psycopg2.extras.register_default_jsonb(curs, loads=loads)
+
+        curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""")
+        data = curs.fetchone()[0]
+        self.assert_(isinstance(data['a'], Decimal))
+        self.assertEqual(data['a'], Decimal('100.0'))
+
+        curs.execute("""select array['{"a": 100.0, "b": null}']::jsonb[]""")
+        data = curs.fetchone()[0]
+        self.assert_(isinstance(data[0]['a'], Decimal))
+        self.assertEqual(data[0]['a'], Decimal('100.0'))
+
+    def test_null(self):
+        curs = self.conn.cursor()
+        curs.execute("""select NULL::jsonb""")
+        self.assertEqual(curs.fetchone()[0], None)
+        curs.execute("""select NULL::jsonb[]""")
+        self.assertEqual(curs.fetchone()[0], None)
+
+decorate_all_tests(JsonbTestCase, skip_if_no_json_module)
+decorate_all_tests(JsonbTestCase, skip_if_no_jsonb_type)
+
+
 class RangeTestCase(unittest.TestCase):
     def test_noparam(self):
         from psycopg2.extras import Range