From fab31e94418a010f03205c6377c864f47dab58c4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 4 Feb 2011 17:29:29 +0100 Subject: [PATCH] Fetch 'arraysize' records per roundtrip in named cursors iteration Closes ticket #33. --- NEWS-2.3 | 1 + doc/src/cursor.rst | 17 +++++++++++++++++ psycopg/cursor_type.c | 9 +++++++-- tests/test_cursor.py | 15 +++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/NEWS-2.3 b/NEWS-2.3 index df9e6afc..5e168f19 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -5,6 +5,7 @@ What's new in psycopg 2.3.3 - Added `register_composite()` function to cast PostgreSQL composite types into Python tuples/namedtuples. + - More efficient iteration on named cursors. - The build script refuses to guess values if pg_config is not found. - Connections and cursors are weakly referenceable. diff --git a/doc/src/cursor.rst b/doc/src/cursor.rst index bed2f7a5..57fe73bc 100644 --- a/doc/src/cursor.rst +++ b/doc/src/cursor.rst @@ -208,6 +208,11 @@ The ``cursor`` class (2, None, 'dada') (3, 42, 'bar') + .. versionchanged:: 2.3.3 + iterating over a :ref:`named cursor ` + fetches `~cursor.arraysize` records at time from the backend. + Previously only one record was fetched per roundtrip, resulting + in unefficient iteration. .. method:: fetchone() @@ -300,6 +305,18 @@ The ``cursor`` class This read/write attribute specifies the number of rows to fetch at a time with `~cursor.fetchmany()`. It defaults to 1 meaning to fetch a single row at a time. + + The attribute is also used when iterating a :ref:`named cursor + `: when syntax such as ``for i in cursor:`` is + used, in order to avoid an excessive number of network roundtrips, the + cursor will actually fetch `!arraysize` records at time from the + backend. For this task the default value of 1 is a poor value: if + `!arraysize` is 1, a default value of 2000 will be used instead. If + you really want to retrieve one record at time from the backend use + `fetchone()` in a loop. + + .. versionchanged:: 2.3.3 + `!arraysize` used in named cursor iteration. .. attribute:: rowcount diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index e4790b61..5416269e 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -821,8 +821,13 @@ psyco_curs_next_named(cursorObject *self) Dprintf("psyco_curs_next_named: rowcount = %ld", self->rowcount); if (self->row >= self->rowcount) { char buffer[128]; - /* FIXME: use a cursor member for the size */ - PyOS_snprintf(buffer, 127, "FETCH FORWARD 10 FROM %s", self->name); + + /* fetch 'arraysize' records, but shun the default value of 1 */ + long int size = self->arraysize; + if (size == 1) { size = 2000L; } + + PyOS_snprintf(buffer, 127, "FETCH FORWARD %ld FROM %s", + size, self->name); if (pq_execute(self, buffer, 0) == -1) return NULL; if (_psyco_curs_prefetch(self) < 0) return NULL; } diff --git a/tests/test_cursor.py b/tests/test_cursor.py index da61c592..a859f42b 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -142,6 +142,21 @@ class CursorTests(unittest.TestCase): "named cursor records fetched in 2 roundtrips (delta: %s)" % (t2 - t1)) + def test_iter_named_cursor_default_arraysize(self): + curs = self.conn.cursor('tmp') + curs.execute('select generate_series(1,50)') + rv = [ (r[0], curs.rownumber) for r in curs ] + # everything swallowed in one gulp + self.assertEqual(rv, [(i,i) for i in range(1,51)]) + + def test_iter_named_cursor_arraysize(self): + curs = self.conn.cursor('tmp') + curs.arraysize = 30 + curs.execute('select generate_series(1,50)') + rv = [ (r[0], curs.rownumber) for r in curs ] + # everything swallowed in two gulps + self.assertEqual(rv, [(i,((i - 1) % 30) + 1) for i in range(1,51)]) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__)