Merge branch 'lo64'

NEWS

@@ -4,6 +4,10 @@ Current release
 What's new in psycopg 2.6
 -------------------------
 
+New features:
+
+- Added support for large objects larger than 2GB.
+
 Bug fixes:
 
 - Json apapter's `!str()` returns the adapted content instead of the `!repr()`

doc/src/extensions.rst

@@ -47,11 +47,13 @@ functionalities defined by the |DBAPI|_.
 
         Database OID of the object.
 
+
     .. attribute:: mode
 
         The mode the database was open. See `connection.lobject()` for a
         description of the available modes.
 
+
     .. method:: read(bytes=-1)
 
         Read a chunk of data from the current file position. If -1 (default)
@@ -64,6 +66,7 @@ functionalities defined by the |DBAPI|_.
         .. versionchanged:: 2.4
             added Unicode support.
 
+
     .. method:: write(str)
 
         Write a string to the large object. Return the number of bytes
@@ -73,6 +76,7 @@ functionalities defined by the |DBAPI|_.
         .. versionchanged:: 2.4
             added Unicode support.
 
+
     .. method:: export(file_name)
 
         Export the large object content to the file system.
@@ -82,33 +86,50 @@ functionalities defined by the |DBAPI|_.
         .. |lo_export| replace:: `!lo_export()`
         .. _lo_export: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-EXPORT
 
+
     .. method:: seek(offset, whence=0)
 
         Set the lobject current position.
 
+        .. versionchanged:: 2.6.0
+            added support for *offset* > 2GB.
+
+
     .. method:: tell()
 
         Return the lobject current position.
 
-    .. method:: truncate(len=0)
-
         .. versionadded:: 2.2.0
 
+        .. versionchanged:: 2.6.0
+            added support for return value > 2GB.
+
+
+    .. method:: truncate(len=0)
+
         Truncate the lobject to the given size.
 
-        The method will only be available if Psycopg has been built against libpq
+        The method will only be available if Psycopg has been built against
-        from PostgreSQL 8.3 or later and can only be used with PostgreSQL servers
+        libpq from PostgreSQL 8.3 or later and can only be used with
-        running these versions. It uses the |lo_truncate|_ libpq function.
+        PostgreSQL servers running these versions. It uses the |lo_truncate|_
+        libpq function.
 
         .. |lo_truncate| replace:: `!lo_truncate()`
         .. _lo_truncate: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-TRUNCATE
 
+        .. versionadded:: 2.2.0
+
+        .. versionchanged:: 2.6.0
+            added support for *len* > 2GB.
+
     .. warning::
 
-            If Psycopg is built with |lo_truncate| support (i.e. if the
+        If Psycopg is built with |lo_truncate| support or with the 64 bits API
-            :program:`pg_config` used during setup is version >= 8.3), but at
+        support (resp. from PostgreSQL versions 8.3 and 9.3) but at runtime an
-            runtime an older libpq is found, Psycopg will fail to import.  See
+        older version of the dynamic library is found, the ``psycopg2`` module
-            :ref:`the lo_truncate FAQ <faq-lo_truncate>` about the problem.
+        will fail to import.  See :ref:`the lo_truncate FAQ <faq-lo_truncate>`
+        about the problem.
+
 
     .. method:: close()
 

doc/src/faq.rst

@@ -248,13 +248,20 @@ I can't compile `!psycopg2`: the compiler says *error: libpq-fe.h: No such file
 .. cssclass:: faq
 
 `!psycopg2` raises `!ImportError` with message *_psycopg.so: undefined symbol: lo_truncate* when imported.
-    This means that Psycopg has been compiled with |lo_truncate|_ support,
+    This means that Psycopg was compiled with |lo_truncate|_ support (*i.e.*
-    which means that the libpq used at compile time was version >= 8.3, but at
+    the libpq used at compile time was version >= 8.3) but at runtime an older
-    runtime an older libpq library is found. You can use::
+    libpq dynamic library is found.
+
+    Fast-forward several years, if the message reports *undefined symbol:
+    lo_truncate64* it means that Psycopg was built with large objects 64 bits
+    API support (*i.e.* the libpq used at compile time was at least 9.3) but
+    at runtime an older libpq dynamic library is found.
+
+    You can use::
 
         $ ldd /path/to/packages/psycopg2/_psycopg.so | grep libpq
 
-    to find what is the version used at runtime.
+    to find what is the libpq dynamic library used at runtime.
 
     You can avoid the problem by using the same version of the
     :program:`pg_config` at install time and the libpq at runtime.

doc/src/usage.rst

@@ -899,6 +899,18 @@ using the |lo_import|_ and |lo_export|_ libpq functions.
 .. |lo_export| replace:: `!lo_export()`
 .. _lo_export: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-EXPORT
 
+.. versionchanged:: 2.6
+    added support for large objects greated than 2GB. Note that the support is
+    enabled only if both these conditions are verified:
+
+    - the extension was built against libpq at least 9.3 (you can check if
+      `psycopg2.__version__` contains the ``lo64`` flag);
+    - the server version is at least PostgreSQL 9.3
+      (`~connection.server_version` must be >= ``90300``).
+
+    If the contitions are not met several `!lobject` methods will fail if the
+    arguments exceed 2GB.
+
 
 
 .. index::

psycopg/lobject.h

@@ -60,8 +60,8 @@ RAISES_NEG HIDDEN int lobject_export(lobjectObject *self, const char *filename);
 RAISES_NEG HIDDEN Py_ssize_t lobject_read(lobjectObject *self, char *buf, size_t len);
 RAISES_NEG HIDDEN Py_ssize_t lobject_write(lobjectObject *self, const char *buf,
                                 size_t len);
-RAISES_NEG HIDDEN int lobject_seek(lobjectObject *self, int pos, int whence);
+RAISES_NEG HIDDEN long lobject_seek(lobjectObject *self, long pos, int whence);
-RAISES_NEG HIDDEN int lobject_tell(lobjectObject *self);
+RAISES_NEG HIDDEN long lobject_tell(lobjectObject *self);
 RAISES_NEG HIDDEN int lobject_truncate(lobjectObject *self, size_t len);
 RAISES_NEG HIDDEN int lobject_close(lobjectObject *self);
 

psycopg/lobject_int.c

@@ -376,21 +376,29 @@ lobject_read(lobjectObject *self, char *buf, size_t len)
 
 /* lobject_seek - move the current position in the lo */
 
-RAISES_NEG int
+RAISES_NEG long
-lobject_seek(lobjectObject *self, int pos, int whence)
+lobject_seek(lobjectObject *self, long pos, int whence)
 {
     PGresult *pgres = NULL;
     char *error = NULL;
-    int where;
+    long where;
 
-    Dprintf("lobject_seek: fd = %d, pos = %d, whence = %d",
+    Dprintf("lobject_seek: fd = %d, pos = %ld, whence = %d",
             self->fd, pos, whence);
 
     Py_BEGIN_ALLOW_THREADS;
     pthread_mutex_lock(&(self->conn->lock));
 
-    where = lo_lseek(self->conn->pgconn, self->fd, pos, whence);
+#ifdef HAVE_LO64
-    Dprintf("lobject_seek: where = %d", where);
+    if (self->conn->server_version < 90300) {
+        where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence);
+    } else {
+        where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence);
+    }
+#else
+    where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence);
+#endif
+    Dprintf("lobject_seek: where = %ld", where);
     if (where < 0)
         collect_error(self->conn, &error);
 
@@ -404,20 +412,28 @@ lobject_seek(lobjectObject *self, int pos, int whence)
 
 /* lobject_tell - tell the current position in the lo */
 
-RAISES_NEG int
+RAISES_NEG long
 lobject_tell(lobjectObject *self)
 {
     PGresult *pgres = NULL;
     char *error = NULL;
-    int where;
+    long where;
 
     Dprintf("lobject_tell: fd = %d", self->fd);
 
     Py_BEGIN_ALLOW_THREADS;
     pthread_mutex_lock(&(self->conn->lock));
 
-    where = lo_tell(self->conn->pgconn, self->fd);
+#ifdef HAVE_LO64
-    Dprintf("lobject_tell: where = %d", where);
+    if (self->conn->server_version < 90300) {
+        where = (long)lo_tell(self->conn->pgconn, self->fd);
+    } else {
+        where = lo_tell64(self->conn->pgconn, self->fd);
+    }
+#else
+    where = (long)lo_tell(self->conn->pgconn, self->fd);
+#endif
+    Dprintf("lobject_tell: where = %ld", where);
     if (where < 0)
         collect_error(self->conn, &error);
 
@@ -473,7 +489,15 @@ lobject_truncate(lobjectObject *self, size_t len)
     Py_BEGIN_ALLOW_THREADS;
     pthread_mutex_lock(&(self->conn->lock));
 
+#ifdef HAVE_LO64
+    if (self->conn->server_version < 90300) {
         retvalue = lo_truncate(self->conn->pgconn, self->fd, len);
+    } else {
+        retvalue = lo_truncate64(self->conn->pgconn, self->fd, len);
+    }
+#else
+    retvalue = lo_truncate(self->conn->pgconn, self->fd, len);
+#endif
     Dprintf("lobject_truncate: result = %d", retvalue);
     if (retvalue < 0)
         collect_error(self->conn, &error);

psycopg/lobject_type.c

@@ -121,7 +121,7 @@ static PyObject *
 psyco_lobj_read(lobjectObject *self, PyObject *args)
 {
     PyObject *res;
-    int where, end;
+    long where, end;
     Py_ssize_t size = -1;
     char *buffer;
 
@@ -165,20 +165,39 @@ psyco_lobj_read(lobjectObject *self, PyObject *args)
 static PyObject *
 psyco_lobj_seek(lobjectObject *self, PyObject *args)
 {
-    int offset, whence=0;
+    long offset, pos=0;
-    int pos=0;
+    int whence=0;
 
-    if (!PyArg_ParseTuple(args, "i|i", &offset, &whence))
+    if (!PyArg_ParseTuple(args, "l|i", &offset, &whence))
         return NULL;
 
     EXC_IF_LOBJ_CLOSED(self);
     EXC_IF_LOBJ_LEVEL0(self);
     EXC_IF_LOBJ_UNMARKED(self);
 
+#ifdef HAVE_LO64
+    if ((offset < INT_MIN || offset > INT_MAX)
+            && self->conn->server_version < 90300) {
+        PyErr_Format(NotSupportedError,
+            "offset out of range (%ld): server version %d "
+            "does not support the lobject 64 API",
+            offset, self->conn->server_version);
+        return NULL;
+    }
+#else
+    if (offset < INT_MIN || offset > INT_MAX) {
+        PyErr_Format(InterfaceError,
+            "offset out of range (%ld): this psycopg version was not built "
+            "with lobject 64 API support",
+            offset);
+        return NULL;
+    }
+#endif
+
     if ((pos = lobject_seek(self, offset, whence)) < 0)
         return NULL;
 
-    return PyInt_FromLong((long)pos);
+    return PyLong_FromLong(pos);
 }
 
 /* tell method - tell current position in the lobject */
@@ -189,7 +208,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args)
 static PyObject *
 psyco_lobj_tell(lobjectObject *self, PyObject *args)
 {
-    int pos;
+    long pos;
 
     EXC_IF_LOBJ_CLOSED(self);
     EXC_IF_LOBJ_LEVEL0(self);
@@ -198,7 +217,7 @@ psyco_lobj_tell(lobjectObject *self, PyObject *args)
     if ((pos = lobject_tell(self)) < 0)
         return NULL;
 
-    return PyInt_FromLong((long)pos);
+    return PyLong_FromLong(pos);
 }
 
 /* unlink method - unlink (destroy) the lobject */
@@ -255,15 +274,33 @@ psyco_lobj_get_closed(lobjectObject *self, void *closure)
 static PyObject *
 psyco_lobj_truncate(lobjectObject *self, PyObject *args)
 {
-    int len = 0;
+    long len = 0;
 
-    if (!PyArg_ParseTuple(args, "|i", &len))
+    if (!PyArg_ParseTuple(args, "|l", &len))
         return NULL;
 
     EXC_IF_LOBJ_CLOSED(self);
     EXC_IF_LOBJ_LEVEL0(self);
     EXC_IF_LOBJ_UNMARKED(self);
 
+#ifdef HAVE_LO64
+    if (len > INT_MAX && self->conn->server_version < 90300) {
+        PyErr_Format(NotSupportedError,
+            "len out of range (%ld): server version %d "
+            "does not support the lobject 64 API",
+            len, self->conn->server_version);
+        return NULL;
+    }
+#else
+    if (len > INT_MAX) {
+        PyErr_Format(InterfaceError,
+            "len out of range (%ld): this psycopg version was not built "
+            "with lobject 64 API support",
+            len);
+        return NULL;
+    }
+#endif
+
     if (lobject_truncate(self, len) < 0)
         return NULL;
 

setup.py

@@ -407,6 +407,9 @@ class psycopg_build_ext(build_ext):
                 pgmajor, pgminor, pgpatch = m.group(1, 2, 3)
                 if pgpatch is None or not pgpatch.isdigit():
                     pgpatch = 0
+                pgmajor = int(pgmajor)
+                pgminor = int(pgminor)
+                pgpatch = int(pgpatch)
             else:
                 sys.stderr.write(
                     "Error: could not determine PostgreSQL version from '%s'"
@@ -414,7 +417,22 @@ class psycopg_build_ext(build_ext):
                 sys.exit(1)
 
             define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" %
-                                  (int(pgmajor), int(pgminor), int(pgpatch))))
+                                  (pgmajor, pgminor, pgpatch)))
+
+            # enable lo64 if libpq >= 9.3 and Python 64 bits
+            if (pgmajor, pgminor) >= (9, 3) and sys.maxint > (1 << 32):
+                define_macros.append(("HAVE_LO64", "1"))
+
+                # Inject the flag in the version string already packed up
+                # because we didn't know the version before.
+                # With distutils everything is complicated.
+                for i, t in enumerate(define_macros):
+                    if t[0] == 'PSYCOPG_VERSION':
+                        n = t[1].find(')')
+                        if n > 0:
+                            define_macros[i] = (
+                                t[0], t[1][:n] + ' lo64' + t[1][n:])
+
         except Warning:
             w = sys.exc_info()[1]  # work around py 2/3 different syntax
             sys.stderr.write("Error: %s\n" % w)

tests/test_lobject.py

@@ -440,6 +440,68 @@ decorate_all_tests(LargeObjectTruncateTests,
     skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate)
 
 
+def _has_lo64(conn):
+    """Return (bool, msg) about the lo64 support"""
+    if conn.server_version < 90300:
+        return (False, "server version %s doesn't support the lo64 API"
+                % conn.server_version)
+
+    if 'lo64' not in psycopg2.__version__:
+        return (False, "this psycopg build doesn't support the lo64 API")
+
+    return (True, "this server and build support the lo64 API")
+
+def skip_if_no_lo64(f):
+    @wraps(f)
+    def skip_if_no_lo64_(self):
+        lo64, msg = _has_lo64(self.conn)
+        if not lo64: return self.skipTest(msg)
+        else: return f(self)
+
+    return skip_if_no_lo64_
+
+class LargeObject64Tests(LargeObjectTestCase):
+    def test_seek_tell_truncate_greater_than_2gb(self):
+        lo = self.conn.lobject()
+
+        length = (1 << 31) + (1 << 30)  # 2gb + 1gb = 3gb
+        lo.truncate(length)
+
+        self.assertEqual(lo.seek(length, 0), length)
+        self.assertEqual(lo.tell(), length)
+
+decorate_all_tests(LargeObject64Tests,
+    skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate, skip_if_no_lo64)
+
+
+def skip_if_lo64(f):
+    @wraps(f)
+    def skip_if_lo64_(self):
+        lo64, msg = _has_lo64(self.conn)
+        if lo64: return self.skipTest(msg)
+        else: return f(self)
+
+    return skip_if_lo64_
+
+class LargeObjectNot64Tests(LargeObjectTestCase):
+    def test_seek_larger_than_2gb(self):
+        lo = self.conn.lobject()
+        offset = 1 << 32  # 4gb
+        self.assertRaises(
+            (psycopg2.InterfaceError, psycopg2.NotSupportedError),
+            lo.seek, offset, 0)
+
+    def test_truncate_larger_than_2gb(self):
+        lo = self.conn.lobject()
+        length = 1 << 32  # 4gb
+        self.assertRaises(
+            (psycopg2.InterfaceError, psycopg2.NotSupportedError),
+            lo.truncate, length)
+
+decorate_all_tests(LargeObjectNot64Tests,
+    skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate, skip_if_lo64)
+
+
 def test_suite():
     return unittest.TestLoader().loadTestsFromName(__name__)