psycopg/psycopg2
1
1
Fork 0
mirror of https://github.com/psycopg/psycopg2.git synced 2025-01-31 09:24:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'lo64'

Browse Source
This commit is contained in:
Daniele Varrazzo 2014-12-25 15:08:02 +01:00
commit b5ac992944
9 changed files with 224 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
NEWS
View File

@ -4,6 +4,10 @@ Current release
What's new in psycopg 2.6
-------------------------
New features:
- Added support for large objects larger than 2GB.
Bug fixes:
- Json apapter's `!str()` returns the adapted content instead of the `!repr()`

47
doc/src/extensions.rst
View File

@ -40,18 +40,20 @@ functionalities defined by the |DBAPI|_.
The class can be subclassed: see the `connection.lobject()` to know
how to specify a `!lobject` subclass.
.. versionadded:: 2.0.8
.. attribute:: oid
Database OID of the object.
.. attribute:: mode
The mode the database was open. See `connection.lobject()` for a
description of the available modes.
.. method:: read(bytes=-1)
Read a chunk of data from the current file position. If -1 (default)
@ -64,6 +66,7 @@ functionalities defined by the |DBAPI|_.
.. versionchanged:: 2.4
added Unicode support.
.. method:: write(str)
Write a string to the large object. Return the number of bytes
@ -73,42 +76,60 @@ functionalities defined by the |DBAPI|_.
.. versionchanged:: 2.4
added Unicode support.
.. method:: export(file_name)
Export the large object content to the file system.
The method uses the efficient |lo_export|_ libpq function.
.. |lo_export| replace:: `!lo_export()`
.. _lo_export: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-EXPORT
.. method:: seek(offset, whence=0)
Set the lobject current position.
.. versionchanged:: 2.6.0
added support for *offset* > 2GB.
.. method:: tell()
Return the lobject current position.
.. method:: truncate(len=0)
.. versionadded:: 2.2.0
.. versionchanged:: 2.6.0
added support for return value > 2GB.
.. method:: truncate(len=0)
Truncate the lobject to the given size.
The method will only be available if Psycopg has been built against libpq
from PostgreSQL 8.3 or later and can only be used with PostgreSQL servers
running these versions. It uses the |lo_truncate|_ libpq function.
The method will only be available if Psycopg has been built against
libpq from PostgreSQL 8.3 or later and can only be used with
PostgreSQL servers running these versions. It uses the |lo_truncate|_
libpq function.
.. |lo_truncate| replace:: `!lo_truncate()`
.. _lo_truncate: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-TRUNCATE
.. warning::
.. versionadded:: 2.2.0
.. versionchanged:: 2.6.0
added support for *len* > 2GB.
.. warning::
If Psycopg is built with |lo_truncate| support or with the 64 bits API
support (resp. from PostgreSQL versions 8.3 and 9.3) but at runtime an
older version of the dynamic library is found, the ``psycopg2`` module
will fail to import. See :ref:`the lo_truncate FAQ <faq-lo_truncate>`
about the problem.
If Psycopg is built with |lo_truncate| support (i.e. if the
:program:`pg_config` used during setup is version >= 8.3), but at
runtime an older libpq is found, Psycopg will fail to import. See
:ref:`the lo_truncate FAQ <faq-lo_truncate>` about the problem.
.. method:: close()

15
doc/src/faq.rst
View File

@ -248,13 +248,20 @@ I can't compile `!psycopg2`: the compiler says *error: libpq-fe.h: No such file
.. cssclass:: faq
`!psycopg2` raises `!ImportError` with message *_psycopg.so: undefined symbol: lo_truncate* when imported.
This means that Psycopg has been compiled with |lo_truncate|_ support,
which means that the libpq used at compile time was version >= 8.3, but at
runtime an older libpq library is found. You can use::
This means that Psycopg was compiled with |lo_truncate|_ support (*i.e.*
the libpq used at compile time was version >= 8.3) but at runtime an older
libpq dynamic library is found.
Fast-forward several years, if the message reports *undefined symbol:
lo_truncate64* it means that Psycopg was built with large objects 64 bits
API support (*i.e.* the libpq used at compile time was at least 9.3) but
at runtime an older libpq dynamic library is found.
You can use::
$ ldd /path/to/packages/psycopg2/_psycopg.so | grep libpq
to find what is the version used at runtime.
to find what is the libpq dynamic library used at runtime.
You can avoid the problem by using the same version of the
:program:`pg_config` at install time and the libpq at runtime.

12
doc/src/usage.rst
View File

@ -899,6 +899,18 @@ using the |lo_import|_ and |lo_export|_ libpq functions.
.. |lo_export| replace:: `!lo_export()`
.. _lo_export: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-EXPORT
.. versionchanged:: 2.6
added support for large objects greated than 2GB. Note that the support is
enabled only if both these conditions are verified:
- the extension was built against libpq at least 9.3 (you can check if
`psycopg2.__version__` contains the ``lo64`` flag);
- the server version is at least PostgreSQL 9.3
(`~connection.server_version` must be >= ``90300``).
If the contitions are not met several `!lobject` methods will fail if the
arguments exceed 2GB.
.. index::

4
psycopg/lobject.h
View File

@ -60,8 +60,8 @@ RAISES_NEG HIDDEN int lobject_export(lobjectObject *self, const char *filename);
RAISES_NEG HIDDEN Py_ssize_t lobject_read(lobjectObject *self, char *buf, size_t len);
RAISES_NEG HIDDEN Py_ssize_t lobject_write(lobjectObject *self, const char *buf,
size_t len);
RAISES_NEG HIDDEN int lobject_seek(lobjectObject *self, int pos, int whence);
RAISES_NEG HIDDEN int lobject_tell(lobjectObject *self);
RAISES_NEG HIDDEN long lobject_seek(lobjectObject *self, long pos, int whence);
RAISES_NEG HIDDEN long lobject_tell(lobjectObject *self);
RAISES_NEG HIDDEN int lobject_truncate(lobjectObject *self, size_t len);
RAISES_NEG HIDDEN int lobject_close(lobjectObject *self);

44
psycopg/lobject_int.c
View File

@ -376,21 +376,29 @@ lobject_read(lobjectObject *self, char *buf, size_t len)
/* lobject_seek - move the current position in the lo */
RAISES_NEG int
lobject_seek(lobjectObject *self, int pos, int whence)
RAISES_NEG long
lobject_seek(lobjectObject *self, long pos, int whence)
{
PGresult *pgres = NULL;
char *error = NULL;
int where;
long where;
Dprintf("lobject_seek: fd = %d, pos = %d, whence = %d",
Dprintf("lobject_seek: fd = %d, pos = %ld, whence = %d",
self->fd, pos, whence);
Py_BEGIN_ALLOW_THREADS;
pthread_mutex_lock(&(self->conn->lock));
where = lo_lseek(self->conn->pgconn, self->fd, pos, whence);
Dprintf("lobject_seek: where = %d", where);
#ifdef HAVE_LO64
if (self->conn->server_version < 90300) {
where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence);
} else {
where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence);
}
#else
where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence);
#endif
Dprintf("lobject_seek: where = %ld", where);
if (where < 0)
collect_error(self->conn, &error);
@ -404,20 +412,28 @@ lobject_seek(lobjectObject *self, int pos, int whence)
/* lobject_tell - tell the current position in the lo */
RAISES_NEG int
RAISES_NEG long
lobject_tell(lobjectObject *self)
{
PGresult *pgres = NULL;
char *error = NULL;
int where;
long where;
Dprintf("lobject_tell: fd = %d", self->fd);
Py_BEGIN_ALLOW_THREADS;
pthread_mutex_lock(&(self->conn->lock));
where = lo_tell(self->conn->pgconn, self->fd);
Dprintf("lobject_tell: where = %d", where);
#ifdef HAVE_LO64
if (self->conn->server_version < 90300) {
where = (long)lo_tell(self->conn->pgconn, self->fd);
} else {
where = lo_tell64(self->conn->pgconn, self->fd);
}
#else
where = (long)lo_tell(self->conn->pgconn, self->fd);
#endif
Dprintf("lobject_tell: where = %ld", where);
if (where < 0)
collect_error(self->conn, &error);
@ -473,7 +489,15 @@ lobject_truncate(lobjectObject *self, size_t len)
Py_BEGIN_ALLOW_THREADS;
pthread_mutex_lock(&(self->conn->lock));
#ifdef HAVE_LO64
if (self->conn->server_version < 90300) {
retvalue = lo_truncate(self->conn->pgconn, self->fd, len);
} else {
retvalue = lo_truncate64(self->conn->pgconn, self->fd, len);
}
#else
retvalue = lo_truncate(self->conn->pgconn, self->fd, len);
#endif
Dprintf("lobject_truncate: result = %d", retvalue);
if (retvalue < 0)
collect_error(self->conn, &error);

55
psycopg/lobject_type.c
View File

@ -121,7 +121,7 @@ static PyObject *
psyco_lobj_read(lobjectObject *self, PyObject *args)
{
PyObject *res;
int where, end;
long where, end;
Py_ssize_t size = -1;
char *buffer;
@ -165,20 +165,39 @@ psyco_lobj_read(lobjectObject *self, PyObject *args)
static PyObject *
psyco_lobj_seek(lobjectObject *self, PyObject *args)
{
int offset, whence=0;
int pos=0;
long offset, pos=0;
int whence=0;
if (!PyArg_ParseTuple(args, "i|i", &offset, &whence))
if (!PyArg_ParseTuple(args, "l|i", &offset, &whence))
return NULL;
EXC_IF_LOBJ_CLOSED(self);
EXC_IF_LOBJ_LEVEL0(self);
EXC_IF_LOBJ_UNMARKED(self);
#ifdef HAVE_LO64
if ((offset < INT_MIN || offset > INT_MAX)
&& self->conn->server_version < 90300) {
PyErr_Format(NotSupportedError,
"offset out of range (%ld): server version %d "
"does not support the lobject 64 API",
offset, self->conn->server_version);
return NULL;
}
#else
if (offset < INT_MIN || offset > INT_MAX) {
PyErr_Format(InterfaceError,
"offset out of range (%ld): this psycopg version was not built "
"with lobject 64 API support",
offset);
return NULL;
}
#endif
if ((pos = lobject_seek(self, offset, whence)) < 0)
return NULL;
return PyInt_FromLong((long)pos);
return PyLong_FromLong(pos);
}
/* tell method - tell current position in the lobject */
@ -189,7 +208,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args)
static PyObject *
psyco_lobj_tell(lobjectObject *self, PyObject *args)
{
int pos;
long pos;
EXC_IF_LOBJ_CLOSED(self);
EXC_IF_LOBJ_LEVEL0(self);
@ -198,7 +217,7 @@ psyco_lobj_tell(lobjectObject *self, PyObject *args)
if ((pos = lobject_tell(self)) < 0)
return NULL;
return PyInt_FromLong((long)pos);
return PyLong_FromLong(pos);
}
/* unlink method - unlink (destroy) the lobject */
@ -255,15 +274,33 @@ psyco_lobj_get_closed(lobjectObject *self, void *closure)
static PyObject *
psyco_lobj_truncate(lobjectObject *self, PyObject *args)
{
int len = 0;
long len = 0;
if (!PyArg_ParseTuple(args, "|i", &len))
if (!PyArg_ParseTuple(args, "|l", &len))
return NULL;
EXC_IF_LOBJ_CLOSED(self);
EXC_IF_LOBJ_LEVEL0(self);
EXC_IF_LOBJ_UNMARKED(self);
#ifdef HAVE_LO64
if (len > INT_MAX && self->conn->server_version < 90300) {
PyErr_Format(NotSupportedError,
"len out of range (%ld): server version %d "
"does not support the lobject 64 API",
len, self->conn->server_version);
return NULL;
}
#else
if (len > INT_MAX) {
PyErr_Format(InterfaceError,
"len out of range (%ld): this psycopg version was not built "
"with lobject 64 API support",
len);
return NULL;
}
#endif
if (lobject_truncate(self, len) < 0)
return NULL;

20
setup.py
View File

@ -407,6 +407,9 @@ class psycopg_build_ext(build_ext):
pgmajor, pgminor, pgpatch = m.group(1, 2, 3)
if pgpatch is None or not pgpatch.isdigit():
pgpatch = 0
pgmajor = int(pgmajor)
pgminor = int(pgminor)
pgpatch = int(pgpatch)
else:
sys.stderr.write(
"Error: could not determine PostgreSQL version from '%s'"
@ -414,7 +417,22 @@ class psycopg_build_ext(build_ext):
sys.exit(1)
define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" %
(int(pgmajor), int(pgminor), int(pgpatch))))
(pgmajor, pgminor, pgpatch)))
# enable lo64 if libpq >= 9.3 and Python 64 bits
if (pgmajor, pgminor) >= (9, 3) and sys.maxint > (1 << 32):
define_macros.append(("HAVE_LO64", "1"))
# Inject the flag in the version string already packed up
# because we didn't know the version before.
# With distutils everything is complicated.
for i, t in enumerate(define_macros):
if t[0] == 'PSYCOPG_VERSION':
n = t[1].find(')')
if n > 0:
define_macros[i] = (
t[0], t[1][:n] + ' lo64' + t[1][n:])
except Warning:
w = sys.exc_info()[1] # work around py 2/3 different syntax
sys.stderr.write("Error: %s\n" % w)

62
tests/test_lobject.py
View File

@ -440,6 +440,68 @@ decorate_all_tests(LargeObjectTruncateTests,
skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate)
def _has_lo64(conn):
"""Return (bool, msg) about the lo64 support"""
if conn.server_version < 90300:
return (False, "server version %s doesn't support the lo64 API"
% conn.server_version)
if 'lo64' not in psycopg2.__version__:
return (False, "this psycopg build doesn't support the lo64 API")
return (True, "this server and build support the lo64 API")
def skip_if_no_lo64(f):
@wraps(f)
def skip_if_no_lo64_(self):
lo64, msg = _has_lo64(self.conn)
if not lo64: return self.skipTest(msg)
else: return f(self)
return skip_if_no_lo64_
class LargeObject64Tests(LargeObjectTestCase):
def test_seek_tell_truncate_greater_than_2gb(self):
lo = self.conn.lobject()
length = (1 << 31) + (1 << 30) # 2gb + 1gb = 3gb
lo.truncate(length)
self.assertEqual(lo.seek(length, 0), length)
self.assertEqual(lo.tell(), length)
decorate_all_tests(LargeObject64Tests,
skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate, skip_if_no_lo64)
def skip_if_lo64(f):
@wraps(f)
def skip_if_lo64_(self):
lo64, msg = _has_lo64(self.conn)
if lo64: return self.skipTest(msg)
else: return f(self)
return skip_if_lo64_
class LargeObjectNot64Tests(LargeObjectTestCase):
def test_seek_larger_than_2gb(self):
lo = self.conn.lobject()
offset = 1 << 32 # 4gb
self.assertRaises(
(psycopg2.InterfaceError, psycopg2.NotSupportedError),
lo.seek, offset, 0)
def test_truncate_larger_than_2gb(self):
lo = self.conn.lobject()
length = 1 << 32 # 4gb
self.assertRaises(
(psycopg2.InterfaceError, psycopg2.NotSupportedError),
lo.truncate, length)
decorate_all_tests(LargeObjectNot64Tests,
skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate, skip_if_lo64)
def test_suite():
return unittest.TestLoader().loadTestsFromName(__name__)