From 352b017fe0ec30bed63d9b1e11c20459c9a02f06 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Fri, 5 Sep 2014 11:55:14 -0400 Subject: [PATCH] Use lseek64 and ltell64 to support large object greater than 2gb in size. --- psycopg/lobject.h | 4 ++-- psycopg/lobject_int.c | 20 ++++++++++---------- psycopg/lobject_type.c | 14 +++++++------- tests/test_lobject.py | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/psycopg/lobject.h b/psycopg/lobject.h index 56f9ead4..b9c8c3d8 100644 --- a/psycopg/lobject.h +++ b/psycopg/lobject.h @@ -60,8 +60,8 @@ RAISES_NEG HIDDEN int lobject_export(lobjectObject *self, const char *filename); RAISES_NEG HIDDEN Py_ssize_t lobject_read(lobjectObject *self, char *buf, size_t len); RAISES_NEG HIDDEN Py_ssize_t lobject_write(lobjectObject *self, const char *buf, size_t len); -RAISES_NEG HIDDEN int lobject_seek(lobjectObject *self, int pos, int whence); -RAISES_NEG HIDDEN int lobject_tell(lobjectObject *self); +RAISES_NEG HIDDEN long lobject_seek(lobjectObject *self, long pos, int whence); +RAISES_NEG HIDDEN long lobject_tell(lobjectObject *self); RAISES_NEG HIDDEN int lobject_truncate(lobjectObject *self, size_t len); RAISES_NEG HIDDEN int lobject_close(lobjectObject *self); diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 43f2b179..29aa94bd 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -378,21 +378,21 @@ lobject_read(lobjectObject *self, char *buf, size_t len) /* lobject_seek - move the current position in the lo */ -RAISES_NEG int -lobject_seek(lobjectObject *self, int pos, int whence) +RAISES_NEG long +lobject_seek(lobjectObject *self, long pos, int whence) { PGresult *pgres = NULL; char *error = NULL; - int where; + long where; - Dprintf("lobject_seek: fd = %d, pos = %d, whence = %d", + Dprintf("lobject_seek: fd = %d, pos = %Ld, whence = %d", self->fd, pos, whence); Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); - where = lo_lseek(self->conn->pgconn, self->fd, pos, whence); - Dprintf("lobject_seek: where = %d", where); + where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence); + Dprintf("lobject_seek: where = %Ld", where); if (where < 0) collect_error(self->conn, &error); @@ -406,20 +406,20 @@ lobject_seek(lobjectObject *self, int pos, int whence) /* lobject_tell - tell the current position in the lo */ -RAISES_NEG int +RAISES_NEG long lobject_tell(lobjectObject *self) { PGresult *pgres = NULL; char *error = NULL; - int where; + long where; Dprintf("lobject_tell: fd = %d", self->fd); Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); - where = lo_tell(self->conn->pgconn, self->fd); - Dprintf("lobject_tell: where = %d", where); + where = lo_tell64(self->conn->pgconn, self->fd); + Dprintf("lobject_tell: where = %Ld", where); if (where < 0) collect_error(self->conn, &error); diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 47f5063a..36d14af1 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -123,7 +123,7 @@ static PyObject * psyco_lobj_read(lobjectObject *self, PyObject *args) { PyObject *res; - int where, end; + long where, end; Py_ssize_t size = -1; char *buffer; @@ -167,10 +167,10 @@ psyco_lobj_read(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_seek(lobjectObject *self, PyObject *args) { - int offset, whence=0; - int pos=0; + long offset, pos=0; + int whence=0; - if (!PyArg_ParseTuple(args, "i|i", &offset, &whence)) + if (!PyArg_ParseTuple(args, "l|i", &offset, &whence)) return NULL; EXC_IF_LOBJ_CLOSED(self); @@ -180,7 +180,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) if ((pos = lobject_seek(self, offset, whence)) < 0) return NULL; - return PyInt_FromLong((long)pos); + return PyLong_FromLong(pos); } /* tell method - tell current position in the lobject */ @@ -191,7 +191,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_tell(lobjectObject *self, PyObject *args) { - int pos; + long pos; EXC_IF_LOBJ_CLOSED(self); EXC_IF_LOBJ_LEVEL0(self); @@ -200,7 +200,7 @@ psyco_lobj_tell(lobjectObject *self, PyObject *args) if ((pos = lobject_tell(self)) < 0) return NULL; - return PyInt_FromLong((long)pos); + return PyLong_FromLong(pos); } /* unlink method - unlink (destroy) the lobject */ diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 66a3d8a1..36b24273 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -225,6 +225,24 @@ class LargeObjectTests(LargeObjectTestCase): self.assertEqual(lo.seek(-2, 2), length - 2) self.assertEqual(lo.read(), "ta") + def test_seek_tell_greater_than_2gb(self): + lo = self.conn.lobject() + + # write chunks until its 3gb + length = 0 + for _ in range(24): + # each chunk is written with 128mb + length += lo.write("data" * (1 << 25)) + self.assertEqual(lo.tell(), length) + lo.close() + lo = self.conn.lobject(lo.oid) + + # seek to 3gb - 4, last written text should be data + offset = (1 << 31) + (1 << 30) - 4 # 2gb + 1gb - 4 + self.assertEqual(lo.seek(offset, 0), offset) + self.assertEqual(lo.tell(), offset) + self.assertEqual(lo.read(), "data") + def test_unlink(self): lo = self.conn.lobject() lo.unlink()