From 71d1690870954ec9a3c83d275fe439e5b591c299 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Feb 2012 21:27:45 +0000 Subject: [PATCH 1/5] Test methods reordered to improve readability --- tests/test_extras_dictcursor.py | 56 +++++++++++++++++---------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index 2bbf8288..e808ed15 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -35,6 +35,7 @@ class ExtrasDictCursorTests(unittest.TestCase): def tearDown(self): self.conn.close() + def testDictCursorWithPlainCursorFetchOne(self): self._testWithPlainCursor(lambda curs: curs.fetchone()) @@ -53,6 +54,21 @@ class ExtrasDictCursorTests(unittest.TestCase): return row self._testWithPlainCursor(getter) + def testUpdateRow(self): + row = self._testWithPlainCursor(lambda curs: curs.fetchone()) + row['foo'] = 'qux' + self.failUnless(row['foo'] == 'qux') + self.failUnless(row[0] == 'qux') + + def _testWithPlainCursor(self, getter): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs.execute("SELECT * FROM ExtrasDictCursorTests") + row = getter(curs) + self.failUnless(row['foo'] == 'bar') + self.failUnless(row[0] == 'bar') + return row + + def testDictCursorWithPlainCursorRealFetchOne(self): self._testWithPlainCursorReal(lambda curs: curs.fetchone()) @@ -71,6 +87,12 @@ class ExtrasDictCursorTests(unittest.TestCase): return row self._testWithPlainCursorReal(getter) + def _testWithPlainCursorReal(self, getter): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute("SELECT * FROM ExtrasDictCursorTests") + row = getter(curs) + self.failUnless(row['foo'] == 'bar') + def testDictCursorWithNamedCursorFetchOne(self): self._testWithNamedCursor(lambda curs: curs.fetchone()) @@ -95,6 +117,13 @@ class ExtrasDictCursorTests(unittest.TestCase): curs = self.conn.cursor('tmp', cursor_factory=psycopg2.extras.DictCursor) self._testNamedCursorNotGreedy(curs) + def _testWithNamedCursor(self, getter): + curs = self.conn.cursor('aname', cursor_factory=psycopg2.extras.DictCursor) + curs.execute("SELECT * FROM ExtrasDictCursorTests") + row = getter(curs) + self.failUnless(row['foo'] == 'bar') + self.failUnless(row[0] == 'bar') + def testDictCursorRealWithNamedCursorFetchOne(self): self._testWithNamedCursorReal(lambda curs: curs.fetchone()) @@ -119,39 +148,12 @@ class ExtrasDictCursorTests(unittest.TestCase): curs = self.conn.cursor('tmp', cursor_factory=psycopg2.extras.RealDictCursor) self._testNamedCursorNotGreedy(curs) - - def _testWithPlainCursor(self, getter): - curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - curs.execute("SELECT * FROM ExtrasDictCursorTests") - row = getter(curs) - self.failUnless(row['foo'] == 'bar') - self.failUnless(row[0] == 'bar') - return row - - def _testWithNamedCursor(self, getter): - curs = self.conn.cursor('aname', cursor_factory=psycopg2.extras.DictCursor) - curs.execute("SELECT * FROM ExtrasDictCursorTests") - row = getter(curs) - self.failUnless(row['foo'] == 'bar') - self.failUnless(row[0] == 'bar') - - def _testWithPlainCursorReal(self, getter): - curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) - curs.execute("SELECT * FROM ExtrasDictCursorTests") - row = getter(curs) - self.failUnless(row['foo'] == 'bar') - def _testWithNamedCursorReal(self, getter): curs = self.conn.cursor('aname', cursor_factory=psycopg2.extras.RealDictCursor) curs.execute("SELECT * FROM ExtrasDictCursorTests") row = getter(curs) self.failUnless(row['foo'] == 'bar') - def testUpdateRow(self): - row = self._testWithPlainCursor(lambda curs: curs.fetchone()) - row['foo'] = 'qux' - self.failUnless(row['foo'] == 'qux') - self.failUnless(row[0] == 'qux') def _testNamedCursorNotGreedy(self, curs): curs.itersize = 2 From 7221ea9ec57728b9afb1c5e42f4a509dad9da29b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Feb 2012 22:04:22 +0000 Subject: [PATCH 2/5] Added test to check rowcount behaves fine during named cursor iteration Actually *it doesn't*: once we iterate the first itersize records, rowcount is reset to zero. If we want to fix it we need an extra member in the cursor. --- tests/test_cursor.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 5cb9b7ef..6286942e 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -234,6 +234,17 @@ class CursorTests(unittest.TestCase): # everything swallowed in two gulps self.assertEqual(rv, [(i,((i - 1) % 30) + 1) for i in range(1,51)]) + @skip_before_postgres(8, 0) + def test_iter_named_cursor_rownumber(self): + curs = self.conn.cursor('tmp') + # note: this fails if itersize < dataset: internally we check + # rownumber == rowcount to detect when to read anoter page, so we + # would need an extra attribute to have a monotonic rownumber. + curs.itersize = 20 + curs.execute('select generate_series(1,10)') + for i, rec in enumerate(curs): + self.assertEqual(i + 1, curs.rownumber) + @skip_if_no_namedtuple def test_namedtuple_description(self): curs = self.conn.cursor() From ebec522a07bc6ad091df2cf8f9d7892760423266 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Feb 2012 22:55:13 +0000 Subject: [PATCH 3/5] Fixed rownumber for cursor subclasses during iterations Regression introduced to fix ticket #80. Don't use fetchmany to get the chunks of values. I did it that way because I was ending up into infinite recursion calling __iter__ from __iter__: the solution has been the "while 1: yield next()" idiom. --- lib/extras.py | 20 ++++++-------------- tests/test_extras_dictcursor.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/lib/extras.py b/lib/extras.py index e1091e2d..aa7bc877 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -88,25 +88,17 @@ class DictCursorBase(_cursor): def __iter__(self): if self._prefetch: - res = _cursor.fetchmany(self, self.itersize) - if not res: - return + res = _cursor.__iter__(self) + first = res.next() if self._query_executed: self._build_index() if not self._prefetch: - res = _cursor.fetchmany(self, self.itersize) + res = _cursor.__iter__(self) + first = res.next() - for r in res: - yield r - - # the above was the first itersize record. the following are - # in a repeated loop. + yield first while 1: - res = _cursor.fetchmany(self, self.itersize) - if not res: - return - for r in res: - yield r + yield res.next() class DictConnection(_connection): diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index e808ed15..3bdb3ba3 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -60,6 +60,11 @@ class ExtrasDictCursorTests(unittest.TestCase): self.failUnless(row['foo'] == 'qux') self.failUnless(row[0] == 'qux') + @skip_before_postgres(8, 0) + def testDictCursorWithPlainCursorIterRowNumber(self): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + self._testIterRowNumber(curs) + def _testWithPlainCursor(self, getter): curs = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) curs.execute("SELECT * FROM ExtrasDictCursorTests") @@ -87,6 +92,11 @@ class ExtrasDictCursorTests(unittest.TestCase): return row self._testWithPlainCursorReal(getter) + @skip_before_postgres(8, 0) + def testDictCursorWithPlainCursorRealIterRowNumber(self): + curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + self._testIterRowNumber(curs) + def _testWithPlainCursorReal(self, getter): curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) curs.execute("SELECT * FROM ExtrasDictCursorTests") @@ -117,6 +127,11 @@ class ExtrasDictCursorTests(unittest.TestCase): curs = self.conn.cursor('tmp', cursor_factory=psycopg2.extras.DictCursor) self._testNamedCursorNotGreedy(curs) + @skip_before_postgres(8, 0) + def testDictCursorWithNamedCursorIterRowNumber(self): + curs = self.conn.cursor('tmp', cursor_factory=psycopg2.extras.DictCursor) + self._testIterRowNumber(curs) + def _testWithNamedCursor(self, getter): curs = self.conn.cursor('aname', cursor_factory=psycopg2.extras.DictCursor) curs.execute("SELECT * FROM ExtrasDictCursorTests") @@ -148,6 +163,11 @@ class ExtrasDictCursorTests(unittest.TestCase): curs = self.conn.cursor('tmp', cursor_factory=psycopg2.extras.RealDictCursor) self._testNamedCursorNotGreedy(curs) + @skip_before_postgres(8, 0) + def testDictCursorRealWithNamedCursorIterRowNumber(self): + curs = self.conn.cursor('tmp', cursor_factory=psycopg2.extras.RealDictCursor) + self._testIterRowNumber(curs) + def _testWithNamedCursorReal(self, getter): curs = self.conn.cursor('aname', cursor_factory=psycopg2.extras.RealDictCursor) curs.execute("SELECT * FROM ExtrasDictCursorTests") @@ -167,6 +187,14 @@ class ExtrasDictCursorTests(unittest.TestCase): self.assert_(recs[1]['ts'] - recs[0]['ts'] < timedelta(seconds=0.005)) self.assert_(recs[2]['ts'] - recs[1]['ts'] > timedelta(seconds=0.0099)) + def _testIterRowNumber(self, curs): + # Only checking for dataset < itersize: + # see CursorTests.test_iter_named_cursor_rownumber + curs.itersize = 20 + curs.execute("""select * from generate_series(1,10)""") + for i, r in enumerate(curs): + self.assertEqual(i + 1, curs.rownumber) + class NamedTupleCursorTest(unittest.TestCase): def setUp(self): From b8597dc1d369d69788a446f65d1f8741d09e064a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Feb 2012 22:58:58 +0000 Subject: [PATCH 4/5] Fixed NamedTupleCursor rownumber during iteration. The correction is similar to the other one for the other subclasses. Also added tests for rowcount and rownumber during different fetch styles. Just in case. --- lib/extras.py | 17 ++++++++++------- tests/test_extras_dictcursor.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/lib/extras.py b/lib/extras.py index aa7bc877..870b5ca7 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -310,14 +310,17 @@ class NamedTupleCursor(_cursor): return [nt(*t) for t in ts] def __iter__(self): - # Invoking _cursor.__iter__(self) goes to infinite recursion, - # so we do pagination by hand + it = _cursor.__iter__(self) + t = it.next() + + nt = self.Record + if nt is None: + nt = self.Record = self._make_nt() + + yield nt(*t) + while 1: - recs = self.fetchmany(self.itersize) - if not recs: - return - for rec in recs: - yield rec + yield nt(*it.next()) try: from collections import namedtuple diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index 3bdb3ba3..dd746379 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -222,12 +222,14 @@ class NamedTupleCursorTest(unittest.TestCase): @skip_if_no_namedtuple def test_fetchone(self): curs = self.conn.cursor() - curs.execute("select * from nttest where i = 1") + curs.execute("select * from nttest order by 1") t = curs.fetchone() self.assertEqual(t[0], 1) self.assertEqual(t.i, 1) self.assertEqual(t[1], 'foo') self.assertEqual(t.s, 'foo') + self.assertEqual(curs.rownumber, 1) + self.assertEqual(curs.rowcount, 3) @skip_if_no_namedtuple def test_fetchmany_noarg(self): @@ -240,6 +242,8 @@ class NamedTupleCursorTest(unittest.TestCase): self.assertEqual(res[0].s, 'foo') self.assertEqual(res[1].i, 2) self.assertEqual(res[1].s, 'bar') + self.assertEqual(curs.rownumber, 2) + self.assertEqual(curs.rowcount, 3) @skip_if_no_namedtuple def test_fetchmany(self): @@ -251,6 +255,8 @@ class NamedTupleCursorTest(unittest.TestCase): self.assertEqual(res[0].s, 'foo') self.assertEqual(res[1].i, 2) self.assertEqual(res[1].s, 'bar') + self.assertEqual(curs.rownumber, 2) + self.assertEqual(curs.rowcount, 3) @skip_if_no_namedtuple def test_fetchall(self): @@ -264,6 +270,8 @@ class NamedTupleCursorTest(unittest.TestCase): self.assertEqual(res[1].s, 'bar') self.assertEqual(res[2].i, 3) self.assertEqual(res[2].s, 'baz') + self.assertEqual(curs.rownumber, 3) + self.assertEqual(curs.rowcount, 3) @skip_if_no_namedtuple def test_executemany(self): @@ -281,16 +289,26 @@ class NamedTupleCursorTest(unittest.TestCase): curs = self.conn.cursor() curs.execute("select * from nttest order by 1") i = iter(curs) + self.assertEqual(curs.rownumber, 0) + t = i.next() self.assertEqual(t.i, 1) self.assertEqual(t.s, 'foo') + self.assertEqual(curs.rownumber, 1) + self.assertEqual(curs.rowcount, 3) + t = i.next() self.assertEqual(t.i, 2) self.assertEqual(t.s, 'bar') + self.assertEqual(curs.rownumber, 2) + self.assertEqual(curs.rowcount, 3) + t = i.next() self.assertEqual(t.i, 3) self.assertEqual(t.s, 'baz') self.assertRaises(StopIteration, i.next) + self.assertEqual(curs.rownumber, 3) + self.assertEqual(curs.rowcount, 3) def test_error_message(self): try: @@ -415,6 +433,17 @@ class NamedTupleCursorTest(unittest.TestCase): self.assert_(recs[1].ts - recs[0].ts < timedelta(seconds=0.005)) self.assert_(recs[2].ts - recs[1].ts > timedelta(seconds=0.0099)) + @skip_if_no_namedtuple + @skip_before_postgres(8, 0) + def test_named_rownumber(self): + curs = self.conn.cursor('tmp') + # Only checking for dataset < itersize: + # see CursorTests.test_iter_named_cursor_rownumber + curs.itersize = 4 + curs.execute("""select * from generate_series(1,3)""") + for i, t in enumerate(curs): + self.assertEqual(i + 1, curs.rownumber) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 1d7e6afcf0fb60c737a91be4ad19f688aa10f548 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Feb 2012 23:04:54 +0000 Subject: [PATCH 5/5] Ticket #100 closed Note that rownumber is still broken for named cursors: it is reset to zero when each itersize block is fetched. --- NEWS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS b/NEWS index 3e687ab8..5149cf74 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,8 @@ What's new in psycopg 2.4.5 interaction (ticket #90). - Better efficiency and formatting of timezone offset objects thanks to Menno Smits (tickets #94, #95). + - Fixed 'rownumber' during iteration on cursor subclasses. + Regression introduced in 2.4.4 (ticket #100). What's new in psycopg 2.4.4