From e076e935b9241d936635b75eb5c0aa3e4690c297 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 02:32:53 +0200 Subject: [PATCH 001/132] Added test with objects without length as callproc param --- tests/test_cursor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index cba5ccaa..8470adec 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -427,6 +427,10 @@ class CursorTests(ConnectingTestCase): self.assertRaises(psycopg2.InterfaceError, cur.executemany, 'select 1', []) + def test_callproc_badparam(self): + cur = self.conn.cursor() + self.assertRaises(TypeError, cur.callproc, 'lower', 42) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 115ceea1eb894b23dfbfe92cf7479ddf47f6de1b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 5 Jun 2014 02:46:06 +0200 Subject: [PATCH 002/132] Don't ignore silently the `cursor.callproc` argument without a length --- NEWS | 6 ++++++ psycopg/cursor_type.c | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 9216010e..51830ff1 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,12 @@ Bug fixes: (:ticket:`#191`). +What's new in psycopg 2.5.4 +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Don't ignore silently the `cursor.callproc` argument without a length. + + What's new in psycopg 2.5.3 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 623ed128..ddbed298 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1032,8 +1032,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args) } if (parameters != Py_None) { - nparameters = PyObject_Length(parameters); - if (nparameters < 0) nparameters = 0; + if (-1 == (nparameters = PyObject_Length(parameters))) { goto exit; } } /* allocate some memory, build the SQL and create a PyString from it */ From 56adc590fffbc76fa5e99aa64c657621a630cbe1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 6 Jun 2014 21:21:39 +0200 Subject: [PATCH 003/132] Fixed segfault if COPY statements are executed Close ticket #219 --- NEWS | 2 ++ psycopg/pqpath.c | 16 +++++++++++++++- tests/test_copy.py | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 51830ff1..63568de1 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,8 @@ Bug fixes: What's new in psycopg 2.5.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Fixed segfault if COPY statements are executed instead of using the + proper methods (:ticket:`#219`). - Don't ignore silently the `cursor.callproc` argument without a length. diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 05d22ee9..42460f5b 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -1288,6 +1288,13 @@ _pq_copy_in_v3(cursorObject *curs) Py_ssize_t length = 0; int res, error = 0; + if (!curs->copyfile) { + PyErr_SetString(ProgrammingError, + "can't execute COPY FROM: use the copy_from() method instead"); + error = 1; + goto exit; + } + if (!(func = PyObject_GetAttrString(curs->copyfile, "read"))) { Dprintf("_pq_copy_in_v3: can't get o.read"); error = 1; @@ -1411,7 +1418,8 @@ exit: static int _pq_copy_out_v3(cursorObject *curs) { - PyObject *tmp = NULL, *func; + PyObject *tmp = NULL; + PyObject *func = NULL; PyObject *obj = NULL; int ret = -1; int is_text; @@ -1419,6 +1427,12 @@ _pq_copy_out_v3(cursorObject *curs) char *buffer; Py_ssize_t len; + if (!curs->copyfile) { + PyErr_SetString(ProgrammingError, + "can't execute COPY TO: use the copy_to() method instead"); + goto exit; + } + if (!(func = PyObject_GetAttrString(curs->copyfile, "write"))) { Dprintf("_pq_copy_out_v3: can't get o.write"); goto exit; diff --git a/tests/test_copy.py b/tests/test_copy.py index 43fa5973..5b28b20d 100755 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -28,10 +28,13 @@ from testutils import unittest, ConnectingTestCase, decorate_all_tests from testutils import skip_if_no_iobase, skip_before_postgres from cStringIO import StringIO from itertools import cycle, izip +from subprocess import Popen, PIPE import psycopg2 import psycopg2.extensions -from testutils import skip_copy_if_green +from testutils import skip_copy_if_green, script_to_py3 +from testconfig import dsn + if sys.version_info[0] < 3: _base = object @@ -301,6 +304,41 @@ class CopyTests(ConnectingTestCase): curs.copy_from, StringIO('aaa\nbbb\nccc\n'), 'tcopy') self.assertEqual(curs.rowcount, -1) + def test_copy_from_segfault(self): + # issue #219 + script = ("""\ +import psycopg2 +conn = psycopg2.connect(%(dsn)r) +curs = conn.cursor() +curs.execute("create table copy_segf (id int)") +try: + curs.execute("copy copy_segf from stdin") +except psycopg2.ProgrammingError: + pass +conn.close() +""" % { 'dsn': dsn,}) + + proc = Popen([sys.executable, '-c', script_to_py3(script)]) + proc.communicate() + self.assertEqual(0, proc.returncode) + + def test_copy_to_segfault(self): + # issue #219 + script = ("""\ +import psycopg2 +conn = psycopg2.connect(%(dsn)r) +curs = conn.cursor() +curs.execute("create table copy_segf (id int)") +try: + curs.execute("copy copy_segf to stdout") +except psycopg2.ProgrammingError: + pass +conn.close() +""" % { 'dsn': dsn,}) + + proc = Popen([sys.executable, '-c', script_to_py3(script)], stdout=PIPE) + proc.communicate() + self.assertEqual(0, proc.returncode) decorate_all_tests(CopyTests, skip_copy_if_green) From 081bf843d2c70ccf63e56f8402e3ef07181e4628 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 31 Jul 2014 13:04:53 +0100 Subject: [PATCH 004/132] Parse the error codes from the text file if available The text file was added in PG 9.1. It contains a few errors not available in the SGML. --- scripts/make_errorcodes.py | 57 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py index d2842932..fa45ea48 100755 --- a/scripts/make_errorcodes.py +++ b/scripts/make_errorcodes.py @@ -16,6 +16,7 @@ The script can be run at a new PostgreSQL release to refresh the module. # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. +import re import sys import urllib2 from collections import defaultdict @@ -48,7 +49,46 @@ def read_base_file(filename): raise ValueError("can't find the separator. Is this the right file?") -def parse_errors(url): +def parse_errors_txt(url): + classes = {} + errors = defaultdict(dict) + + page = urllib2.urlopen(url) + for line in page: + # Strip comments and skip blanks + line = line.split('#')[0].strip() + if not line: + continue + + # Parse a section + m = re.match(r"Section: (Class (..) - .+)", line) + if m: + label, class_ = m.groups() + classes[class_] = label + continue + + # Parse an error + m = re.match(r"(.....)\s+(?:E|W|S)\s+ERRCODE_(\S+)(?:\s+(\S+))?$", line) + if m: + errcode, macro, spec = m.groups() + # error 22008 has 2 macros and 1 def: give priority to the def + # as it's the one we used to parse from sgml + if not spec: + if errcode in errors[class_]: + continue + errlabel = macro.upper() + else: + errlabel = spec.upper() + + errors[class_][errcode] = errlabel + continue + + # We don't expect anything else + raise ValueError("unexpected line:\n%s" % line) + + return classes, errors + +def parse_errors_sgml(url): page = BS(urllib2.urlopen(url)) table = page('table')[1]('tbody')[0] @@ -87,14 +127,25 @@ def parse_errors(url): return classes, errors -errors_url="http://www.postgresql.org/docs/%s/static/errcodes-appendix.html" +errors_sgml_url = \ + "http://www.postgresql.org/docs/%s/static/errcodes-appendix.html" + +errors_txt_url = \ + "http://git.postgresql.org/gitweb/?p=postgresql.git;a=blob_plain;" \ + "f=src/backend/utils/errcodes.txt;hb=REL%s_STABLE" def fetch_errors(versions): classes = {} errors = defaultdict(dict) for version in versions: - c1, e1 = parse_errors(errors_url % version) + print >> sys.stderr, version + tver = tuple(map(int, version.split('.'))) + if tver < (9, 1): + c1, e1 = parse_errors_sgml(errors_sgml_url % version) + else: + c1, e1 = parse_errors_txt( + errors_txt_url % version.replace('.', '_')) classes.update(c1) for c, cerrs in e1.iteritems(): errors[c].update(cerrs) From 40dca9924d859b893d84673b516ef5dfda896047 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 31 Jul 2014 13:13:27 +0100 Subject: [PATCH 005/132] Add a few missing errcodes They are used in the code but not defined in the SGML docs so the script failed to parse them. --- NEWS | 2 ++ doc/src/errorcodes.rst | 2 +- lib/errorcodes.py | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 63568de1..9418d52a 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,8 @@ What's new in psycopg 2.5.4 - Fixed segfault if COPY statements are executed instead of using the proper methods (:ticket:`#219`). - Don't ignore silently the `cursor.callproc` argument without a length. +- Added a few errors missing from `~psycopg2.errorcodes`, defined by + PostgreSQL but not documented. What's new in psycopg 2.5.3 diff --git a/doc/src/errorcodes.rst b/doc/src/errorcodes.rst index bfaaeb45..3fdd5499 100644 --- a/doc/src/errorcodes.rst +++ b/doc/src/errorcodes.rst @@ -49,7 +49,7 @@ An example of the available constants defined in the module: >>> errorcodes.UNDEFINED_TABLE '42P01' -Constants representing all the error values documented by PostgreSQL versions +Constants representing all the error values defined by PostgreSQL versions between 8.1 and 9.2 are included in the module. diff --git a/lib/errorcodes.py b/lib/errorcodes.py index 12c300f6..e004cac5 100644 --- a/lib/errorcodes.py +++ b/lib/errorcodes.py @@ -284,6 +284,10 @@ STATEMENT_COMPLETION_UNKNOWN = '40003' DEADLOCK_DETECTED = '40P01' # Class 42 - Syntax Error or Access Rule Violation +UNDEFINED_PSTATEMENT = '26000' +UNDEFINED_CURSOR = '34000' +UNDEFINED_DATABASE = '3D000' +UNDEFINED_SCHEMA = '3F000' SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION = '42000' INSUFFICIENT_PRIVILEGE = '42501' SYNTAX_ERROR = '42601' From 6d6fd9acf70d5624293a47d5cc602e8831b466a6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 31 Jul 2014 13:06:39 +0100 Subject: [PATCH 006/132] Parse errocodes for PG 9.3 too There is no new errcode defined so no need to release this as a change, only change the docs. --- doc/src/errorcodes.rst | 2 +- scripts/make_errorcodes.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/errorcodes.rst b/doc/src/errorcodes.rst index 3fdd5499..f659023d 100644 --- a/doc/src/errorcodes.rst +++ b/doc/src/errorcodes.rst @@ -50,7 +50,7 @@ An example of the available constants defined in the module: '42P01' Constants representing all the error values defined by PostgreSQL versions -between 8.1 and 9.2 are included in the module. +between 8.1 and 9.3 are included in the module. .. autofunction:: lookup(code) diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py index fa45ea48..4da063ef 100755 --- a/scripts/make_errorcodes.py +++ b/scripts/make_errorcodes.py @@ -32,7 +32,7 @@ def main(): file_start = read_base_file(filename) classes, errors = fetch_errors( - ['8.1', '8.2', '8.3', '8.4', '9.0', '9.1', '9.2']) + ['8.1', '8.2', '8.3', '8.4', '9.0', '9.1', '9.2', '9.3']) f = open(filename, "w") for line in file_start: From 44281d669207a98c1fe060619b93ad608867c169 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 4 Aug 2014 22:39:41 +0100 Subject: [PATCH 007/132] Fix supported Py/PG versions in docs --- doc/src/install.rst | 7 +++++-- setup.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/src/install.rst b/doc/src/install.rst index 017be955..3cf24958 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -14,9 +14,12 @@ mature as the C implementation yet. The current `!psycopg2` implementation supports: +.. + NOTE: keep consistent with setup.py and the /features/ page. + - Python 2 versions from 2.5 to 2.7 -- Python 3 versions from 3.1 to 3.3 -- PostgreSQL versions from 7.4 to 9.2 +- Python 3 versions from 3.1 to 3.4 +- PostgreSQL versions from 7.4 to 9.3 .. _PostgreSQL: http://www.postgresql.org/ .. _Python: http://www.python.org/ diff --git a/setup.py b/setup.py index 9fe35f23..92ac1038 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,8 @@ UPDATEs. psycopg2 also provide full asynchronous operations and support for coroutine libraries. """ +# note: if you are changing the list of supported Python version please fix +# the docs in install.rst and the /features/ page on the website. classifiers = """\ Development Status :: 5 - Production/Stable Intended Audience :: Developers From 634fc004fbc70d87148c112ddae6a6722bc93bdc Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 00:29:58 +0100 Subject: [PATCH 008/132] Added wishful test suite for jsonb type --- tests/test_types_extras.py | 91 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 4625995d..46a34217 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -1069,6 +1069,97 @@ class JsonTestCase(ConnectingTestCase): self.assert_(s.endswith("'")) +def skip_if_no_jsonb_type(f): + return skip_before_postgres(9, 4)(f) + +class JsonbTestCase(ConnectingTestCase): + @staticmethod + def myloads(s): + import json + rv = json.loads(s) + rv['test'] = 1 + return rv + + def test_default_cast(self): + curs = self.conn.cursor() + + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None}) + + curs.execute("""select array['{"a": 100.0, "b": null}']::jsonb[]""") + self.assertEqual(curs.fetchone()[0], [{'a': 100.0, 'b': None}]) + + def test_register_on_connection(self): + psycopg2.extras.register_json(self.conn, loads=self.myloads, name='jsonb') + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1}) + + def test_register_on_cursor(self): + curs = self.conn.cursor() + psycopg2.extras.register_json(curs, loads=self.myloads, name='jsonb') + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1}) + + def test_register_globally(self): + old = psycopg2.extensions.string_types.get(3802) + olda = psycopg2.extensions.string_types.get(3807) + try: + new, newa = psycopg2.extras.register_json(self.conn, + loads=self.myloads, globally=True, name='jsonb') + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + self.assertEqual(curs.fetchone()[0], {'a': 100.0, 'b': None, 'test': 1}) + finally: + psycopg2.extensions.string_types.pop(new.values[0]) + psycopg2.extensions.string_types.pop(newa.values[0]) + if old: + psycopg2.extensions.register_type(old) + if olda: + psycopg2.extensions.register_type(olda) + + def test_loads(self): + json = psycopg2.extras.json + loads = lambda x: json.loads(x, parse_float=Decimal) + psycopg2.extras.register_json(self.conn, loads=loads, name='jsonb') + curs = self.conn.cursor() + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], Decimal)) + self.assertEqual(data['a'], Decimal('100.0')) + # sure we are not manling json too? + curs.execute("""select '{"a": 100.0, "b": null}'::json""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], float)) + self.assertEqual(data['a'], 100.0) + + def test_register_default(self): + curs = self.conn.cursor() + + loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) + psycopg2.extras.register_default_jsonb(curs, loads=loads) + + curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") + data = curs.fetchone()[0] + self.assert_(isinstance(data['a'], Decimal)) + self.assertEqual(data['a'], Decimal('100.0')) + + curs.execute("""select array['{"a": 100.0, "b": null}']::jsonb[]""") + data = curs.fetchone()[0] + self.assert_(isinstance(data[0]['a'], Decimal)) + self.assertEqual(data[0]['a'], Decimal('100.0')) + + def test_null(self): + curs = self.conn.cursor() + curs.execute("""select NULL::jsonb""") + self.assertEqual(curs.fetchone()[0], None) + curs.execute("""select NULL::jsonb[]""") + self.assertEqual(curs.fetchone()[0], None) + +decorate_all_tests(JsonbTestCase, skip_if_no_json_module) +decorate_all_tests(JsonbTestCase, skip_if_no_jsonb_type) + + class RangeTestCase(unittest.TestCase): def test_noparam(self): from psycopg2.extras import Range From 6bca443e37cd0cb5e37352eaf2e8aafc832324ee Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 00:43:33 +0100 Subject: [PATCH 009/132] Added name param to register_json() --- doc/src/extras.rst | 3 +++ lib/_json.py | 28 +++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 7fab3384..0abd3354 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -248,6 +248,9 @@ from :sql:`json` into :py:class:`~decimal.Decimal` you can use:: .. autofunction:: register_json + .. versionchanged:: 2.5.4 + added the *name* parameter to enable :sql:`jsonb` support. + .. autofunction:: register_default_json diff --git a/lib/_json.py b/lib/_json.py index 3a4361e8..047551e5 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -102,7 +102,7 @@ class Json(object): def register_json(conn_or_curs=None, globally=False, loads=None, - oid=None, array_oid=None): + oid=None, array_oid=None, name='json'): """Create and register typecasters converting :sql:`json` type to Python objects. :param conn_or_curs: a connection or cursor used to find the :sql:`json` @@ -118,17 +118,19 @@ def register_json(conn_or_curs=None, globally=False, loads=None, queried on *conn_or_curs* :param array_oid: the OID of the :sql:`json[]` array type if known; if not, it will be queried on *conn_or_curs* + :param name: the name of the data type to look for in *conn_or_curs* The connection or cursor passed to the function will be used to query the - database and look for the OID of the :sql:`json` type. No query is - performed if *oid* and *array_oid* are provided. Raise - `~psycopg2.ProgrammingError` if the type is not found. + database and look for the OID of the :sql:`json` type (or an alternative + type if *name* if provided). No query is performed if *oid* and *array_oid* + are provided. Raise `~psycopg2.ProgrammingError` if the type is not found. """ if oid is None: - oid, array_oid = _get_json_oids(conn_or_curs) + oid, array_oid = _get_json_oids(conn_or_curs, name) - JSON, JSONARRAY = _create_json_typecasters(oid, array_oid, loads) + JSON, JSONARRAY = _create_json_typecasters( + oid, array_oid, loads=loads, name=name.upper()) register_type(JSON, not globally and conn_or_curs or None) @@ -149,7 +151,7 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) -def _create_json_typecasters(oid, array_oid, loads=None): +def _create_json_typecasters(oid, array_oid, loads=None, name='JSON'): """Create typecasters for json data type.""" if loads is None: if json is None: @@ -162,15 +164,15 @@ def _create_json_typecasters(oid, array_oid, loads=None): return None return loads(s) - JSON = new_type((oid, ), 'JSON', typecast_json) + JSON = new_type((oid, ), name, typecast_json) if array_oid is not None: - JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + JSONARRAY = new_array_type((array_oid, ), "%sARRAY" % name, JSON) else: JSONARRAY = None return JSON, JSONARRAY -def _get_json_oids(conn_or_curs): +def _get_json_oids(conn_or_curs, name='json'): # lazy imports from psycopg2.extensions import STATUS_IN_TRANSACTION from psycopg2.extras import _solve_conn_curs @@ -185,8 +187,8 @@ def _get_json_oids(conn_or_curs): # get the oid for the hstore curs.execute( - "SELECT t.oid, %s FROM pg_type t WHERE t.typname = 'json';" - % typarray) + "SELECT t.oid, %s FROM pg_type t WHERE t.typname = %%s;" + % typarray, (name,)) r = curs.fetchone() # revert the status of the connection as before the command @@ -194,7 +196,7 @@ def _get_json_oids(conn_or_curs): conn.rollback() if not r: - raise conn.ProgrammingError("json data type not found") + raise conn.ProgrammingError("%s data type not found" % name) return r From 9d547469b8a3f2a9a554edc5218797349d1a4472 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 00:54:49 +0100 Subject: [PATCH 010/132] Add register_default_jsonb() and register the type --- lib/_json.py | 16 ++++++++++++++++ lib/extensions.py | 6 ++++-- lib/extras.py | 3 ++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/_json.py b/lib/_json.py index 047551e5..eef1436e 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -47,6 +47,10 @@ else: JSON_OID = 114 JSONARRAY_OID = 199 +# oids from PostgreSQL 9.4 +JSONB_OID = 3802 +JSONBARRAY_OID = 3807 + class Json(object): """ An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to @@ -151,6 +155,18 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) +def register_default_jsonb(conn_or_curs=None, globally=False, loads=None): + """ + Create and register :sql:`jsonb` typecasters for PostgreSQL 9.4 and following. + + As in `register_default_json()`, the function allows to register a + customized *loads* function for the :sql:`jsonb` type at its known oid for + PostgreSQL 9.4 and following versions. All the parameters have the same + meaning of `register_json()`. + """ + return register_json(conn_or_curs=conn_or_curs, globally=globally, + loads=loads, oid=JSONB_OID, array_oid=JSONBARRAY_OID, name='jsonb') + def _create_json_typecasters(oid, array_oid, loads=None, name='JSON'): """Create typecasters for json data type.""" if loads is None: diff --git a/lib/extensions.py b/lib/extensions.py index f210da4f..71a92b93 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -152,20 +152,22 @@ class NoneAdapter(object): # Create default json typecasters for PostgreSQL 9.2 oids -from psycopg2._json import register_default_json +from psycopg2._json import register_default_json, register_default_jsonb try: JSON, JSONARRAY = register_default_json() + JSONB, JSONBARRAY = register_default_jsonb() except ImportError: pass -del register_default_json +del register_default_json, register_default_jsonb # Create default Range typecasters from psycopg2. _range import Range del Range + # Add the "cleaned" version of the encodings to the key. # When the encoding is set its name is cleaned up from - and _ and turned # uppercase, so an encoding not respecting these rules wouldn't be found in the diff --git a/lib/extras.py b/lib/extras.py index b21e223d..a873c4ef 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -965,7 +965,8 @@ def register_composite(name, conn_or_curs, globally=False, factory=None): # expose the json adaptation stuff into the module -from psycopg2._json import json, Json, register_json, register_default_json +from psycopg2._json import json, Json, register_json +from psycopg2._json import register_default_json, register_default_jsonb # Expose range-related objects From f40ad93a3784c49ecfaa1eb6450311c0117b74da Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 01:32:19 +0100 Subject: [PATCH 011/132] Added jsonb docs --- doc/src/extras.rst | 40 +++++++++++++++++++++++++--------------- doc/src/faq.rst | 14 ++++++++++++++ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 0abd3354..36ef0132 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -160,23 +160,27 @@ JSON_ adaptation ^^^^^^^^^^^^^^^^ .. versionadded:: 2.5 +.. versionchanged:: 2.5.4 + added |jsonb| support. In previous versions |jsonb| values are returned + as strings. See :ref:`the FAQ ` for a workaround. -Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ type. -With PostgreSQL 9.2 adaptation is available out-of-the-box. To use JSON data -with previous database versions (either with the `9.1 json extension`__, but -even if you want to convert text fields to JSON) you can use -`register_json()`. +Psycopg can adapt Python objects to and from the PostgreSQL |pgjson|_ and +|jsonb| types. With PostgreSQL 9.2 and following versions adaptation is +available out-of-the-box. To use JSON data with previous database versions +(either with the `9.1 json extension`__, but even if you want to convert text +fields to JSON) you can use the `register_json()` function. .. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html -The Python library used to convert Python objects to JSON depends on the -language version: with Python 2.6 and following the :py:mod:`json` module from -the standard library is used; with previous versions the `simplejson`_ module -is used if available. Note that the last `!simplejson` version supporting -Python 2.4 is the 2.0.9. +The Python library used by default to convert Python objects to JSON and to +parse data from the database depends on the language version: with Python 2.6 +and following the :py:mod:`json` module from the standard library is used; +with previous versions the `simplejson`_ module is used if available. Note +that the last `!simplejson` version supporting Python 2.4 is the 2.0.9. .. _JSON: http://www.json.org/ .. |pgjson| replace:: :sql:`json` +.. |jsonb| replace:: :sql:`jsonb` .. _pgjson: http://www.postgresql.org/docs/current/static/datatype-json.html .. _simplejson: http://pypi.python.org/pypi/simplejson/ @@ -186,8 +190,8 @@ the `Json` adapter:: curs.execute("insert into mytable (jsondata) values (%s)", [Json({'a': 100})]) -Reading from the database, |pgjson| values will be automatically converted to -Python objects. +Reading from the database, |pgjson| and |jsonb| values will be automatically +converted to Python objects. .. note:: @@ -233,9 +237,11 @@ or you can subclass it overriding the `~Json.dumps()` method:: [MyJson({'a': 100})]) Customizing the conversion from PostgreSQL to Python can be done passing a -custom `!loads()` function to `register_json()` (or `register_default_json()` -for PostgreSQL 9.2). For example, if you want to convert the float values -from :sql:`json` into :py:class:`~decimal.Decimal` you can use:: +custom `!loads()` function to `register_json()`. For the builtin data types +(|pgjson| from PostgreSQL 9.2, |jsonb| from PostgreSQL 9.4) use +`register_default_json()` and `register_default_jsonb()`. For example, if you +want to convert the float values from :sql:`json` into +:py:class:`~decimal.Decimal` you can use:: loads = lambda x: json.loads(x, parse_float=Decimal) psycopg2.extras.register_json(conn, loads=loads) @@ -253,6 +259,10 @@ from :sql:`json` into :py:class:`~decimal.Decimal` you can use:: .. autofunction:: register_default_json +.. autofunction:: register_default_jsonb + + .. versionadded:: 2.5.4 + .. index:: diff --git a/doc/src/faq.rst b/doc/src/faq.rst index fe675231..0646cdff 100644 --- a/doc/src/faq.rst +++ b/doc/src/faq.rst @@ -137,6 +137,20 @@ Psycopg automatically converts PostgreSQL :sql:`json` data into Python objects. See :ref:`adapt-json` for further details. +.. _faq-jsonb-adapt: +.. cssclass:: faq + +Psycopg converts :sql:`json` values into Python objects but :sql:`jsonb` values are returned as strings. Can :sql:`jsonb` be converted automatically? + Automatic conversion of :sql:`jsonb` values is supported from Psycopg + release 2.5.4. For previous versions you can register the :sql:`json` + typecaster on the :sql:`jsonb` oids (which are known and not suppsed to + change in future PostgreSQL versions):: + + psycopg2.extras.register_json(oid=3802, array_oid=3807, globally=True) + + See :ref:`adapt-json` for further details. + + .. _faq-bytea-9.0: .. cssclass:: faq From e225aad042917651cc32a0e198bf0e575ed50d26 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 01:58:28 +0100 Subject: [PATCH 012/132] Habemus jsonb --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 9418d52a..54c1217f 100644 --- a/NEWS +++ b/NEWS @@ -13,6 +13,7 @@ Bug fixes: What's new in psycopg 2.5.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Added :sql:`jsonb` support for PostgreSQL 9.4. - Fixed segfault if COPY statements are executed instead of using the proper methods (:ticket:`#219`). - Don't ignore silently the `cursor.callproc` argument without a length. From c475a0db95c8568972b9b724b6724f62699f691e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 02:07:33 +0100 Subject: [PATCH 013/132] Document PostgreSQL 9.4 as supported Actually there is a test failing in the test suite in PG 9.4beta2, but it's probably because the default logging level is to DEBUG. Will wait for the final release to check if the test is to be fixed. All other tests pass no problem. --- doc/src/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/install.rst b/doc/src/install.rst index 3cf24958..d2a28e49 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -19,7 +19,7 @@ The current `!psycopg2` implementation supports: - Python 2 versions from 2.5 to 2.7 - Python 3 versions from 3.1 to 3.4 -- PostgreSQL versions from 7.4 to 9.3 +- PostgreSQL versions from 7.4 to 9.4 .. _PostgreSQL: http://www.postgresql.org/ .. _Python: http://www.python.org/ From 41a083cec36d3688950b83603887d03213b588ea Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 02:37:00 +0100 Subject: [PATCH 014/132] Convert pool arguments to int Failing to do so may cause dangerous misbehaviours such as an unbounded pool (because of lame comparison operators in Python 2). Fix ticket #220. --- NEWS | 2 ++ lib/pool.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 54c1217f..7afc8628 100644 --- a/NEWS +++ b/NEWS @@ -16,6 +16,8 @@ What's new in psycopg 2.5.4 - Added :sql:`jsonb` support for PostgreSQL 9.4. - Fixed segfault if COPY statements are executed instead of using the proper methods (:ticket:`#219`). +- Force conversion of pool arguments to integer to avoid potentially unbounded + pools (:ticket:`#220`). - Don't ignore silently the `cursor.callproc` argument without a length. - Added a few errors missing from `~psycopg2.errorcodes`, defined by PostgreSQL but not documented. diff --git a/lib/pool.py b/lib/pool.py index 3b41c803..7cdd6af2 100644 --- a/lib/pool.py +++ b/lib/pool.py @@ -42,8 +42,8 @@ class AbstractConnectionPool(object): with given parameters. The connection pool will support a maximum of about 'maxconn' connections. """ - self.minconn = minconn - self.maxconn = maxconn + self.minconn = int(minconn) + self.maxconn = int(maxconn) self.closed = False self._args = args From 31d07e50298881455459fa151a2f26677462e6e5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 13 Aug 2014 02:44:59 +0100 Subject: [PATCH 015/132] Scrape PostgreSQL 9.4 error codes table too No new error code found as of 9.4 beta2. --- scripts/make_errorcodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py index 4da063ef..8a8d54df 100755 --- a/scripts/make_errorcodes.py +++ b/scripts/make_errorcodes.py @@ -32,7 +32,7 @@ def main(): file_start = read_base_file(filename) classes, errors = fetch_errors( - ['8.1', '8.2', '8.3', '8.4', '9.0', '9.1', '9.2', '9.3']) + ['8.1', '8.2', '8.3', '8.4', '9.0', '9.1', '9.2', '9.3', '9.4']) f = open(filename, "w") for line in file_start: From 669e7879190c80d484bba5e969c834b0b3422331 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 15 Aug 2014 01:45:50 +0100 Subject: [PATCH 016/132] Name the types after the module they are exposed from --- psycopg/adapter_asis.c | 4 ++-- psycopg/adapter_binary.c | 4 ++-- psycopg/adapter_pboolean.c | 4 ++-- psycopg/adapter_pfloat.c | 4 ++-- psycopg/adapter_pint.c | 4 ++-- psycopg/adapter_qstring.c | 6 +++--- psycopg/connection_type.c | 6 +++--- psycopg/cursor_type.c | 4 ++-- psycopg/diagnostics_type.c | 2 +- psycopg/lobject_type.c | 2 +- psycopg/microprotocols_proto.c | 2 +- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index a2a8f4dc..c5d89587 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -152,7 +152,7 @@ asis_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * asis_repr(asisObject *self) { - return PyString_FromFormat("", self); + return PyString_FromFormat("", self); } @@ -163,7 +163,7 @@ asis_repr(asisObject *self) PyTypeObject asisType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.AsIs", + "psycopg2.extensions.AsIs", sizeof(asisObject), 0, asis_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 3edcea8d..e6ada5e7 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -257,7 +257,7 @@ binary_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * binary_repr(binaryObject *self) { - return PyString_FromFormat("", self); + return PyString_FromFormat("", self); } /* object type */ @@ -267,7 +267,7 @@ binary_repr(binaryObject *self) PyTypeObject binaryType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.Binary", + "psycopg2.extensions.Binary", sizeof(binaryObject), 0, binary_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index f8cd9959..13262663 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -149,7 +149,7 @@ pboolean_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * pboolean_repr(pbooleanObject *self) { - return PyString_FromFormat("", + return PyString_FromFormat("", self); } @@ -161,7 +161,7 @@ pboolean_repr(pbooleanObject *self) PyTypeObject pbooleanType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.Boolean", + "psycopg2.extensions.Boolean", sizeof(pbooleanObject), 0, pboolean_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 7bb7a467..789fb492 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -178,7 +178,7 @@ pfloat_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * pfloat_repr(pfloatObject *self) { - return PyString_FromFormat("", + return PyString_FromFormat("", self); } @@ -190,7 +190,7 @@ pfloat_repr(pfloatObject *self) PyTypeObject pfloatType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.Float", + "psycopg2.extensions.Float", sizeof(pfloatObject), 0, pfloat_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/adapter_pint.c b/psycopg/adapter_pint.c index 6465acec..25d435ab 100644 --- a/psycopg/adapter_pint.c +++ b/psycopg/adapter_pint.c @@ -164,7 +164,7 @@ pint_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * pint_repr(pintObject *self) { - return PyString_FromFormat("", + return PyString_FromFormat("", self); } @@ -176,7 +176,7 @@ pint_repr(pintObject *self) PyTypeObject pintType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.Int", + "psycopg2.extensions.Int", sizeof(pintObject), 0, pint_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 91c14673..832f04b0 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -245,8 +245,8 @@ qstring_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static PyObject * qstring_repr(qstringObject *self) { - return PyString_FromFormat("", - self); + return PyString_FromFormat( + "", self); } /* object type */ @@ -256,7 +256,7 @@ qstring_repr(qstringObject *self) PyTypeObject qstringType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.QuotedString", + "psycopg2.extensions.QuotedString", sizeof(qstringObject), 0, qstring_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 99235250..d86040b0 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -103,7 +103,7 @@ psyco_conn_cursor(connectionObject *self, PyObject *args, PyObject *kwargs) if (PyObject_IsInstance(obj, (PyObject *)&cursorType) == 0) { PyErr_SetString(PyExc_TypeError, - "cursor factory must be subclass of psycopg2._psycopg.cursor"); + "cursor factory must be subclass of psycopg2.extensions.cursor"); goto exit; } @@ -787,7 +787,7 @@ psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) if (obj == NULL) return NULL; if (PyObject_IsInstance(obj, (PyObject *)&lobjectType) == 0) { PyErr_SetString(PyExc_TypeError, - "lobject factory must be subclass of psycopg2._psycopg.lobject"); + "lobject factory must be subclass of psycopg2.extensions.lobject"); Py_DECREF(obj); return NULL; } @@ -1214,7 +1214,7 @@ connection_traverse(connectionObject *self, visitproc visit, void *arg) PyTypeObject connectionType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.connection", + "psycopg2.extensions.connection", sizeof(connectionObject), 0, connection_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index ddbed298..c194ab33 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1825,7 +1825,7 @@ cursor_setup(cursorObject *self, connectionObject *conn, const char *name) if (PyObject_IsInstance((PyObject*)conn, (PyObject *)&connectionType) == 0) { PyErr_SetString(PyExc_TypeError, - "argument 1 must be subclass of psycopg2._psycopg.connection"); + "argument 1 must be subclass of psycopg2.extensions.connection"); return -1; } */ Py_INCREF(conn); @@ -1966,7 +1966,7 @@ cursor_traverse(cursorObject *self, visitproc visit, void *arg) PyTypeObject cursorType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.cursor", + "psycopg2.extensions.cursor", sizeof(cursorObject), 0, cursor_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/diagnostics_type.c b/psycopg/diagnostics_type.c index dbcbf38e..ec5f9287 100644 --- a/psycopg/diagnostics_type.c +++ b/psycopg/diagnostics_type.c @@ -158,7 +158,7 @@ static const char diagnosticsType_doc[] = PyTypeObject diagnosticsType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.Diagnostics", + "psycopg2.extensions.Diagnostics", sizeof(diagnosticsObject), 0, (destructor)diagnostics_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 068923c9..63511149 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -406,7 +406,7 @@ lobject_repr(lobjectObject *self) PyTypeObject lobjectType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.lobject", + "psycopg2.extensions.lobject", sizeof(lobjectObject), 0, lobject_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ diff --git a/psycopg/microprotocols_proto.c b/psycopg/microprotocols_proto.c index f30da3f4..dfbf8e3c 100644 --- a/psycopg/microprotocols_proto.c +++ b/psycopg/microprotocols_proto.c @@ -142,7 +142,7 @@ isqlquote_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyTypeObject isqlquoteType = { PyVarObject_HEAD_INIT(NULL, 0) - "psycopg2._psycopg.ISQLQuote", + "psycopg2.extensions.ISQLQuote", sizeof(isqlquoteObject), 0, isqlquote_dealloc, /*tp_dealloc*/ 0, /*tp_print*/ From 95165cef7d6157945802ffeb98d51c43fd4033e9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 15 Aug 2014 02:06:27 +0100 Subject: [PATCH 017/132] Dropped almost-no-op customized objects repr() The default repr is enough: it prints instead of . The only people being hurt by this change are the ones using doctests: they deserve it. --- psycopg/adapter_asis.c | 8 +------- psycopg/adapter_binary.c | 7 +------ psycopg/adapter_datetime.c | 8 +------- psycopg/adapter_list.c | 7 +------ psycopg/adapter_mxdatetime.c | 8 +------- psycopg/adapter_pboolean.c | 9 +-------- psycopg/adapter_pdecimal.c | 9 +-------- psycopg/adapter_pfloat.c | 9 +-------- psycopg/adapter_pint.c | 9 +-------- psycopg/adapter_qstring.c | 8 +------- 10 files changed, 10 insertions(+), 72 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index c5d89587..72768ac7 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -149,12 +149,6 @@ asis_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -asis_repr(asisObject *self) -{ - return PyString_FromFormat("", self); -} - /* object type */ @@ -170,7 +164,7 @@ PyTypeObject asisType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)asis_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index e6ada5e7..1bf04aa4 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -254,11 +254,6 @@ binary_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -binary_repr(binaryObject *self) -{ - return PyString_FromFormat("", self); -} /* object type */ @@ -274,7 +269,7 @@ PyTypeObject binaryType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)binary_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 67697fba..aa26d383 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -214,12 +214,6 @@ pydatetime_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -pydatetime_repr(pydatetimeObject *self) -{ - return PyString_FromFormat("", - self); -} /* object type */ @@ -235,7 +229,7 @@ PyTypeObject pydatetimeType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)pydatetime_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 1198a81b..ed26b680 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -215,11 +215,6 @@ list_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -list_repr(listObject *self) -{ - return PyString_FromFormat("", self); -} /* object type */ @@ -235,7 +230,7 @@ PyTypeObject listType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)list_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index 4696a9d0..94642df5 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -205,12 +205,6 @@ mxdatetime_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -mxdatetime_repr(mxdatetimeObject *self) -{ - return PyString_FromFormat("", - self); -} /* object type */ @@ -226,7 +220,7 @@ PyTypeObject mxdatetimeType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)mxdatetime_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index 13262663..e48f5eb6 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -146,13 +146,6 @@ pboolean_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -pboolean_repr(pbooleanObject *self) -{ - return PyString_FromFormat("", - self); -} - /* object type */ @@ -168,7 +161,7 @@ PyTypeObject pbooleanType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)pboolean_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index 35721417..0ec4826c 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -202,13 +202,6 @@ pdecimal_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -pdecimal_repr(pdecimalObject *self) -{ - return PyString_FromFormat("", - self); -} - /* object type */ @@ -224,7 +217,7 @@ PyTypeObject pdecimalType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)pdecimal_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 789fb492..5343f746 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -175,13 +175,6 @@ pfloat_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -pfloat_repr(pfloatObject *self) -{ - return PyString_FromFormat("", - self); -} - /* object type */ @@ -197,7 +190,7 @@ PyTypeObject pfloatType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)pfloat_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_pint.c b/psycopg/adapter_pint.c index 25d435ab..977932bc 100644 --- a/psycopg/adapter_pint.c +++ b/psycopg/adapter_pint.c @@ -161,13 +161,6 @@ pint_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -pint_repr(pintObject *self) -{ - return PyString_FromFormat("", - self); -} - /* object type */ @@ -183,7 +176,7 @@ PyTypeObject pintType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)pint_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 832f04b0..36b0a959 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -242,12 +242,6 @@ qstring_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return type->tp_alloc(type, 0); } -static PyObject * -qstring_repr(qstringObject *self) -{ - return PyString_FromFormat( - "", self); -} /* object type */ @@ -263,7 +257,7 @@ PyTypeObject qstringType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ - (reprfunc)qstring_repr, /*tp_repr*/ + 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ From 1b322a9b19e3edd4274d5802272d8eed62b4caa7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 15 Aug 2014 02:10:17 +0100 Subject: [PATCH 018/132] Fixed segfault in List function This function is never called: it segfaults 100%. To be removed. --- psycopg/adapter_list.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index ed26b680..6335f64b 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -267,10 +267,10 @@ PyTypeObject listType = { PyObject * psyco_List(PyObject *module, PyObject *args) { - PyObject *str; + PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &str)) + if (!PyArg_ParseTuple(args, "O", &obj)) return NULL; - return PyObject_CallFunctionObjArgs((PyObject *)&listType, "O", str, NULL); + return PyObject_CallFunctionObjArgs((PyObject *)&listType, obj, NULL); } From a2b01cdf4221f4fad94143606c06141ce91ef7bd Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 15 Aug 2014 02:37:37 +0100 Subject: [PATCH 019/132] Dropped simple type wrapper functions These functions don't need to exist: exposing the type in the module is enough. It is actually better as one may use isinstance and such. --- psycopg/adapter_asis.c | 14 -------------- psycopg/adapter_asis.h | 6 ------ psycopg/adapter_binary.c | 14 -------------- psycopg/adapter_binary.h | 7 ------- psycopg/adapter_list.c | 14 -------------- psycopg/adapter_list.h | 4 ---- psycopg/adapter_pboolean.c | 14 -------------- psycopg/adapter_pboolean.h | 6 ------ psycopg/adapter_pdecimal.c | 14 -------------- psycopg/adapter_pdecimal.h | 6 ------ psycopg/adapter_pfloat.c | 14 -------------- psycopg/adapter_pfloat.h | 6 ------ psycopg/adapter_pint.c | 14 -------------- psycopg/adapter_pint.h | 6 ------ psycopg/adapter_qstring.c | 14 -------------- psycopg/adapter_qstring.h | 6 ------ psycopg/psycopgmodule.c | 24 ++++++++---------------- 17 files changed, 8 insertions(+), 175 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index 72768ac7..d64b1410 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -194,17 +194,3 @@ PyTypeObject asisType = { 0, /*tp_alloc*/ asis_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_AsIs(PyObject *module, PyObject *args) -{ - PyObject *obj; - - if (!PyArg_ParseTuple(args, "O", &obj)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&asisType, obj, NULL); -} diff --git a/psycopg/adapter_asis.h b/psycopg/adapter_asis.h index 04ba08bc..0e1caaee 100644 --- a/psycopg/adapter_asis.h +++ b/psycopg/adapter_asis.h @@ -40,12 +40,6 @@ typedef struct { } asisObject; -/* functions exported to psycopgmodule.c */ - -HIDDEN PyObject *psyco_AsIs(PyObject *module, PyObject *args); -#define psyco_AsIs_doc \ - "AsIs(obj) -> new AsIs wrapper object" - #ifdef __cplusplus } #endif diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 1bf04aa4..485dc5a4 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -299,17 +299,3 @@ PyTypeObject binaryType = { 0, /*tp_alloc*/ binary_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_Binary(PyObject *module, PyObject *args) -{ - PyObject *str; - - if (!PyArg_ParseTuple(args, "O", &str)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&binaryType, str, NULL); -} diff --git a/psycopg/adapter_binary.h b/psycopg/adapter_binary.h index 408efc77..1c12b070 100644 --- a/psycopg/adapter_binary.h +++ b/psycopg/adapter_binary.h @@ -40,13 +40,6 @@ typedef struct { PyObject *conn; } binaryObject; -/* functions exported to psycopgmodule.c */ - -HIDDEN PyObject *psyco_Binary(PyObject *module, PyObject *args); -#define psyco_Binary_doc \ - "Binary(buffer) -> new binary object\n\n" \ - "Build an object capable to hold a binary string value." - #ifdef __cplusplus } #endif diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 6335f64b..e68b1978 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -260,17 +260,3 @@ PyTypeObject listType = { 0, /*tp_alloc*/ list_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_List(PyObject *module, PyObject *args) -{ - PyObject *obj; - - if (!PyArg_ParseTuple(args, "O", &obj)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&listType, obj, NULL); -} diff --git a/psycopg/adapter_list.h b/psycopg/adapter_list.h index 8ec20f68..e5bf77ee 100644 --- a/psycopg/adapter_list.h +++ b/psycopg/adapter_list.h @@ -39,10 +39,6 @@ typedef struct { PyObject *connection; } listObject; -HIDDEN PyObject *psyco_List(PyObject *module, PyObject *args); -#define psyco_List_doc \ - "List(list, enc) -> new quoted list" - #ifdef __cplusplus } #endif diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index e48f5eb6..8a5950d8 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -191,17 +191,3 @@ PyTypeObject pbooleanType = { 0, /*tp_alloc*/ pboolean_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_Boolean(PyObject *module, PyObject *args) -{ - PyObject *obj; - - if (!PyArg_ParseTuple(args, "O", &obj)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&pbooleanType, obj, NULL); -} diff --git a/psycopg/adapter_pboolean.h b/psycopg/adapter_pboolean.h index b1bb18ae..bd642bf5 100644 --- a/psycopg/adapter_pboolean.h +++ b/psycopg/adapter_pboolean.h @@ -40,12 +40,6 @@ typedef struct { } pbooleanObject; -/* functions exported to psycopgmodule.c */ - -HIDDEN PyObject *psyco_Boolean(PyObject *module, PyObject *args); -#define psyco_Boolean_doc \ - "Boolean(obj) -> new boolean value" - #ifdef __cplusplus } #endif diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index 0ec4826c..fcff9d20 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -247,17 +247,3 @@ PyTypeObject pdecimalType = { 0, /*tp_alloc*/ pdecimal_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_Decimal(PyObject *module, PyObject *args) -{ - PyObject *obj; - - if (!PyArg_ParseTuple(args, "O", &obj)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&pdecimalType, obj, NULL); -} diff --git a/psycopg/adapter_pdecimal.h b/psycopg/adapter_pdecimal.h index 4f89fad5..4a38e299 100644 --- a/psycopg/adapter_pdecimal.h +++ b/psycopg/adapter_pdecimal.h @@ -40,12 +40,6 @@ typedef struct { } pdecimalObject; -/* functions exported to psycopgmodule.c */ - -HIDDEN PyObject *psyco_Decimal(PyObject *module, PyObject *args); -#define psyco_Decimal_doc \ - "Decimal(obj) -> new decimal.Decimal value" - #ifdef __cplusplus } #endif diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 5343f746..c33ccae7 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -220,17 +220,3 @@ PyTypeObject pfloatType = { 0, /*tp_alloc*/ pfloat_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_Float(PyObject *module, PyObject *args) -{ - PyObject *obj; - - if (!PyArg_ParseTuple(args, "O", &obj)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&pfloatType, obj, NULL); -} diff --git a/psycopg/adapter_pfloat.h b/psycopg/adapter_pfloat.h index 7439c04f..2c0af473 100644 --- a/psycopg/adapter_pfloat.h +++ b/psycopg/adapter_pfloat.h @@ -40,12 +40,6 @@ typedef struct { } pfloatObject; -/* functions exported to psycopgmodule.c */ - -HIDDEN PyObject *psyco_Float(PyObject *module, PyObject *args); -#define psyco_Float_doc \ - "Float(obj) -> new float value" - #ifdef __cplusplus } #endif diff --git a/psycopg/adapter_pint.c b/psycopg/adapter_pint.c index 977932bc..18c74b28 100644 --- a/psycopg/adapter_pint.c +++ b/psycopg/adapter_pint.c @@ -206,17 +206,3 @@ PyTypeObject pintType = { 0, /*tp_alloc*/ pint_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_Int(PyObject *module, PyObject *args) -{ - PyObject *obj; - - if (!PyArg_ParseTuple(args, "O", &obj)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&pintType, obj, NULL); -} diff --git a/psycopg/adapter_pint.h b/psycopg/adapter_pint.h index fd553e8b..5488d24f 100644 --- a/psycopg/adapter_pint.h +++ b/psycopg/adapter_pint.h @@ -40,12 +40,6 @@ typedef struct { } pintObject; -/* functions exported to psycopgmodule.c */ - -HIDDEN PyObject *psyco_Int(PyObject *module, PyObject *args); -#define psyco_Int_doc \ - "Int(obj) -> new int value" - #ifdef __cplusplus } #endif diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 36b0a959..2e3ab0ae 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -287,17 +287,3 @@ PyTypeObject qstringType = { 0, /*tp_alloc*/ qstring_new, /*tp_new*/ }; - - -/** module-level functions **/ - -PyObject * -psyco_QuotedString(PyObject *module, PyObject *args) -{ - PyObject *str; - - if (!PyArg_ParseTuple(args, "O", &str)) - return NULL; - - return PyObject_CallFunctionObjArgs((PyObject *)&qstringType, str, NULL); -} diff --git a/psycopg/adapter_qstring.h b/psycopg/adapter_qstring.h index 0446f276..b7b086f3 100644 --- a/psycopg/adapter_qstring.h +++ b/psycopg/adapter_qstring.h @@ -41,12 +41,6 @@ typedef struct { connectionObject *conn; } qstringObject; -/* functions exported to psycopgmodule.c */ - -HIDDEN PyObject *psyco_QuotedString(PyObject *module, PyObject *args); -#define psyco_QuotedString_doc \ - "QuotedString(str, enc) -> new quoted string" - #ifdef __cplusplus } #endif diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 98cc995b..04c0ed7c 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -678,20 +678,6 @@ static PyMethodDef psycopgMethods[] = { {"new_array_type", (PyCFunction)typecast_array_from_python, METH_VARARGS|METH_KEYWORDS, typecast_array_from_python_doc}, - {"AsIs", (PyCFunction)psyco_AsIs, - METH_VARARGS, psyco_AsIs_doc}, - {"QuotedString", (PyCFunction)psyco_QuotedString, - METH_VARARGS, psyco_QuotedString_doc}, - {"Boolean", (PyCFunction)psyco_Boolean, - METH_VARARGS, psyco_Boolean_doc}, - {"Int", (PyCFunction)psyco_Int, - METH_VARARGS, psyco_Int_doc}, - {"Float", (PyCFunction)psyco_Float, - METH_VARARGS, psyco_Float_doc}, - {"Decimal", (PyCFunction)psyco_Decimal, - METH_VARARGS, psyco_Decimal_doc}, - {"Binary", (PyCFunction)psyco_Binary, - METH_VARARGS, psyco_Binary_doc}, {"Date", (PyCFunction)psyco_Date, METH_VARARGS, psyco_Date_doc}, {"Time", (PyCFunction)psyco_Time, @@ -704,8 +690,6 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS, psyco_TimeFromTicks_doc}, {"TimestampFromTicks", (PyCFunction)psyco_TimestampFromTicks, METH_VARARGS, psyco_TimestampFromTicks_doc}, - {"List", (PyCFunction)psyco_List, - METH_VARARGS, psyco_List_doc}, {"DateFromPy", (PyCFunction)psyco_DateFromPy, METH_VARARGS, psyco_DateFromPy_doc}, @@ -912,6 +896,14 @@ INIT_MODULE(_psycopg)(void) PyModule_AddObject(module, "Notify", (PyObject*)¬ifyType); PyModule_AddObject(module, "Xid", (PyObject*)&xidType); PyModule_AddObject(module, "Diagnostics", (PyObject*)&diagnosticsType); + PyModule_AddObject(module, "AsIs", (PyObject*)&asisType); + PyModule_AddObject(module, "Binary", (PyObject*)&binaryType); + PyModule_AddObject(module, "Boolean", (PyObject*)&pbooleanType); + PyModule_AddObject(module, "Decimal", (PyObject*)&pdecimalType); + PyModule_AddObject(module, "Int", (PyObject*)&pintType); + PyModule_AddObject(module, "Float", (PyObject*)&pfloatType); + PyModule_AddObject(module, "List", (PyObject*)&listType); + PyModule_AddObject(module, "QuotedString", (PyObject*)&qstringType); #ifdef PSYCOPG_EXTENSIONS PyModule_AddObject(module, "lobject", (PyObject*)&lobjectType); #endif From 283a422b4dda106667b7cf5a6eee91b38b4b718d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 21 Aug 2014 05:01:34 +0100 Subject: [PATCH 020/132] Added test to verify withhold transaction behaviour A withhold cursor can read its data when the transaction is closed, so it shouldn't start a new one upon movement/close. --- tests/test_cursor.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 8470adec..2f92b6e4 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -176,10 +176,7 @@ class CursorTests(ConnectingTestCase): curs.execute("select data from invname order by data") self.assertEqual(curs.fetchall(), [(10,), (20,), (30,)]) - def test_withhold(self): - self.assertRaises(psycopg2.ProgrammingError, self.conn.cursor, - withhold=True) - + def _create_withhold_table(self): curs = self.conn.cursor() try: curs.execute("drop table withhold") @@ -190,6 +187,11 @@ class CursorTests(ConnectingTestCase): curs.execute("insert into withhold values (%s)", (i,)) curs.close() + def test_withhold(self): + self.assertRaises(psycopg2.ProgrammingError, self.conn.cursor, + withhold=True) + + self._create_withhold_table() curs = self.conn.cursor("W") self.assertEqual(curs.withhold, False); curs.withhold = True @@ -209,6 +211,30 @@ class CursorTests(ConnectingTestCase): curs.execute("drop table withhold") self.conn.commit() + def test_withhold_no_begin(self): + self._create_withhold_table() + curs = self.conn.cursor("w", withhold=True) + curs.execute("select data from withhold order by data") + self.assertEqual(curs.fetchone(), (10,)) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_BEGIN) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_INTRANS) + + self.conn.commit() + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + self.assertEqual(curs.fetchone(), (20,)) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + curs.close() + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + def test_scrollable(self): self.assertRaises(psycopg2.ProgrammingError, self.conn.cursor, scrollable=True) From 478e66f7612072062bbc4673853a3d2bd4476c59 Mon Sep 17 00:00:00 2001 From: Alexey Borzenkov Date: Tue, 9 Jul 2013 23:36:04 +0400 Subject: [PATCH 021/132] No implicit transaction on named cursor close Also, don't start an implicit transaction when fetching with named with hold cursor, since it already returns results from a previously committed transaction. --- psycopg/cursor_type.c | 20 ++++++++++---------- psycopg/pqpath.c | 4 ++-- psycopg/pqpath.h | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index c194ab33..a4751760 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -63,7 +63,7 @@ psyco_curs_close(cursorObject *self) EXC_IF_NO_MARK(self); PyOS_snprintf(buffer, 127, "CLOSE \"%s\"", self->name); - if (pq_execute(self, buffer, 0, 0) == -1) return NULL; + if (pq_execute(self, buffer, 0, 0, 1) == -1) return NULL; } self->closed = 1; @@ -444,7 +444,7 @@ _psyco_curs_execute(cursorObject *self, /* At this point, the SQL statement must be str, not unicode */ - tmp = pq_execute(self, Bytes_AS_STRING(self->query), async, no_result); + tmp = pq_execute(self, Bytes_AS_STRING(self->query), async, no_result, 0); Dprintf("psyco_curs_execute: res = %d, pgres = %p", tmp, self->pgres); if (tmp < 0) { goto exit; } @@ -766,7 +766,7 @@ psyco_curs_fetchone(cursorObject *self) EXC_IF_ASYNC_IN_PROGRESS(self, fetchone); EXC_IF_TPC_PREPARED(self->conn, fetchone); PyOS_snprintf(buffer, 127, "FETCH FORWARD 1 FROM \"%s\"", self->name); - if (pq_execute(self, buffer, 0, 0) == -1) return NULL; + if (pq_execute(self, buffer, 0, 0, self->withhold) == -1) return NULL; if (_psyco_curs_prefetch(self) < 0) return NULL; } @@ -816,7 +816,7 @@ psyco_curs_next_named(cursorObject *self) PyOS_snprintf(buffer, 127, "FETCH FORWARD %ld FROM \"%s\"", self->itersize, self->name); - if (pq_execute(self, buffer, 0, 0) == -1) return NULL; + if (pq_execute(self, buffer, 0, 0, self->withhold) == -1) return NULL; if (_psyco_curs_prefetch(self) < 0) return NULL; } @@ -885,7 +885,7 @@ psyco_curs_fetchmany(cursorObject *self, PyObject *args, PyObject *kwords) EXC_IF_TPC_PREPARED(self->conn, fetchone); PyOS_snprintf(buffer, 127, "FETCH FORWARD %d FROM \"%s\"", (int)size, self->name); - if (pq_execute(self, buffer, 0, 0) == -1) { goto exit; } + if (pq_execute(self, buffer, 0, 0, self->withhold) == -1) { goto exit; } if (_psyco_curs_prefetch(self) < 0) { goto exit; } } @@ -960,7 +960,7 @@ psyco_curs_fetchall(cursorObject *self) EXC_IF_ASYNC_IN_PROGRESS(self, fetchall); EXC_IF_TPC_PREPARED(self->conn, fetchall); PyOS_snprintf(buffer, 127, "FETCH FORWARD ALL FROM \"%s\"", self->name); - if (pq_execute(self, buffer, 0, 0) == -1) { goto exit; } + if (pq_execute(self, buffer, 0, 0, self->withhold) == -1) { goto exit; } if (_psyco_curs_prefetch(self) < 0) { goto exit; } } @@ -1178,7 +1178,7 @@ psyco_curs_scroll(cursorObject *self, PyObject *args, PyObject *kwargs) else { PyOS_snprintf(buffer, 127, "MOVE %d FROM \"%s\"", value, self->name); } - if (pq_execute(self, buffer, 0, 0) == -1) return NULL; + if (pq_execute(self, buffer, 0, 0, self->withhold) == -1) return NULL; if (_psyco_curs_prefetch(self) < 0) return NULL; } @@ -1391,7 +1391,7 @@ psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs) Py_INCREF(file); self->copyfile = file; - if (pq_execute(self, query, 0, 0) >= 0) { + if (pq_execute(self, query, 0, 0, 0) >= 0) { res = Py_None; Py_INCREF(Py_None); } @@ -1485,7 +1485,7 @@ psyco_curs_copy_to(cursorObject *self, PyObject *args, PyObject *kwargs) Py_INCREF(file); self->copyfile = file; - if (pq_execute(self, query, 0, 0) >= 0) { + if (pq_execute(self, query, 0, 0, 0) >= 0) { res = Py_None; Py_INCREF(Py_None); } @@ -1559,7 +1559,7 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs) self->copyfile = file; /* At this point, the SQL statement must be str, not unicode */ - if (pq_execute(self, Bytes_AS_STRING(sql), 0, 0) >= 0) { + if (pq_execute(self, Bytes_AS_STRING(sql), 0, 0, 0) >= 0) { res = Py_None; Py_INCREF(res); } diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 42460f5b..3512f639 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -893,7 +893,7 @@ pq_flush(connectionObject *conn) */ RAISES_NEG int -pq_execute(cursorObject *curs, const char *query, int async, int no_result) +pq_execute(cursorObject *curs, const char *query, int async, int no_result, int no_begin) { PGresult *pgres = NULL; char *error = NULL; @@ -916,7 +916,7 @@ pq_execute(cursorObject *curs, const char *query, int async, int no_result) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(curs->conn->lock)); - if (pq_begin_locked(curs->conn, &pgres, &error, &_save) < 0) { + if (!no_begin && pq_begin_locked(curs->conn, &pgres, &error, &_save) < 0) { pthread_mutex_unlock(&(curs->conn->lock)); Py_BLOCK_THREADS; pq_complete_error(curs->conn, &pgres, &error); diff --git a/psycopg/pqpath.h b/psycopg/pqpath.h index 40beea19..bd3293f8 100644 --- a/psycopg/pqpath.h +++ b/psycopg/pqpath.h @@ -36,7 +36,7 @@ HIDDEN PGresult *pq_get_last_result(connectionObject *conn); RAISES_NEG HIDDEN int pq_fetch(cursorObject *curs, int no_result); RAISES_NEG HIDDEN int pq_execute(cursorObject *curs, const char *query, - int async, int no_result); + int async, int no_result, int no_begin); HIDDEN int pq_send_query(connectionObject *conn, const char *query); HIDDEN int pq_begin_locked(connectionObject *conn, PGresult **pgres, char **error, PyThreadState **tstate); From c81522079ee699c2ed4aba83c7d0123249bfb17e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 21 Aug 2014 05:14:20 +0100 Subject: [PATCH 022/132] Added test to verify withhold cursors work in autocommit --- tests/test_cursor.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 2f92b6e4..07956554 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -235,6 +235,28 @@ class CursorTests(ConnectingTestCase): self.assertEqual(self.conn.get_transaction_status(), psycopg2.extensions.TRANSACTION_STATUS_IDLE) + def test_withhold_autocommit(self): + self._create_withhold_table() + self.conn.commit() + self.conn.autocommit = True + curs = self.conn.cursor("w", withhold=True) + curs.execute("select data from withhold order by data") + + self.assertEqual(curs.fetchone(), (10,)) + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + self.conn.commit() + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + + curs.close() + self.assertEqual(self.conn.status, psycopg2.extensions.STATUS_READY) + self.assertEqual(self.conn.get_transaction_status(), + psycopg2.extensions.TRANSACTION_STATUS_IDLE) + def test_scrollable(self): self.assertRaises(psycopg2.ProgrammingError, self.conn.cursor, scrollable=True) From 13b0852619d005d593abd76d201143ff37ec6aa9 Mon Sep 17 00:00:00 2001 From: Alexey Borzenkov Date: Tue, 9 Jul 2013 23:29:58 +0400 Subject: [PATCH 023/132] Allow using named with hold cursors in autocommit --- psycopg/cursor_type.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index a4751760..40dbaef0 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -478,7 +478,7 @@ psyco_curs_execute(cursorObject *self, PyObject *args, PyObject *kwargs) "can't call .execute() on named cursors more than once"); return NULL; } - if (self->conn->autocommit) { + if (self->conn->autocommit && !self->withhold) { psyco_set_error(ProgrammingError, self, "can't use a named cursor outside of transactions"); return NULL; From d20c03310df7d2e1685fbe2eb97e6eb434bd38fb Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 21 Aug 2014 05:27:57 +0100 Subject: [PATCH 024/132] Document WITH HOLD corrections. --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index 7afc8628..048c646d 100644 --- a/NEWS +++ b/NEWS @@ -18,6 +18,9 @@ What's new in psycopg 2.5.4 proper methods (:ticket:`#219`). - Force conversion of pool arguments to integer to avoid potentially unbounded pools (:ticket:`#220`). +- Cursors :sql:`WITH HOLD` don't begin a new transaction upon move/fetch/close + (:ticket:`#228`). +- Cursors :sql:`WITH HOLD` can be used in autocommit (:ticket:`#229`). - Don't ignore silently the `cursor.callproc` argument without a length. - Added a few errors missing from `~psycopg2.errorcodes`, defined by PostgreSQL but not documented. From 6e841a41e680f8c1570e97086cf923f923851e98 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 23 Aug 2014 19:30:48 +0100 Subject: [PATCH 025/132] Dropped PSYCOPG_EXTENSIONS flag Building without extensions has been long broken and nobody really cares about a pure-DBAPI implementation (which could be created using a wrapper instead). --- psycopg/connection_type.c | 18 ------------------ psycopg/cursor_type.c | 12 ------------ psycopg/green.c | 4 ---- psycopg/lobject_int.c | 5 ----- psycopg/lobject_type.c | 4 ---- psycopg/pqpath.c | 6 ------ psycopg/psycopg.h | 4 ---- psycopg/psycopgmodule.c | 10 ---------- setup.cfg | 3 +-- setup.py | 5 +---- 10 files changed, 2 insertions(+), 69 deletions(-) diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index d86040b0..1f54ccfe 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -442,9 +442,6 @@ exit: } -#ifdef PSYCOPG_EXTENSIONS - - /* parse a python object into one of the possible isolation level values */ extern const IsolationLevel conn_isolevels[]; @@ -860,8 +857,6 @@ psyco_conn_poll(connectionObject *self) } -/* extension: fileno - return the file descriptor of the connection */ - #define psyco_conn_fileno_doc \ "fileno() -> int -- Return file descriptor associated to database connection." @@ -878,8 +873,6 @@ psyco_conn_fileno(connectionObject *self) } -/* extension: isexecuting - check for asynchronous operations */ - #define psyco_conn_isexecuting_doc \ "isexecuting() -> bool -- Return True if the connection is " \ "executing an asynchronous operation." @@ -911,8 +904,6 @@ psyco_conn_isexecuting(connectionObject *self) } -/* extension: cancel - cancel the current operation */ - #define psyco_conn_cancel_doc \ "cancel() -- cancel the current operation" @@ -941,8 +932,6 @@ psyco_conn_cancel(connectionObject *self) Py_RETURN_NONE; } -#endif /* PSYCOPG_EXTENSIONS */ - /** the connection object **/ @@ -974,7 +963,6 @@ static struct PyMethodDef connectionObject_methods[] = { METH_NOARGS, psyco_conn_enter_doc}, {"__exit__", (PyCFunction)psyco_conn_exit, METH_VARARGS, psyco_conn_exit_doc}, -#ifdef PSYCOPG_EXTENSIONS {"set_session", (PyCFunction)psyco_conn_set_session, METH_VARARGS|METH_KEYWORDS, psyco_conn_set_session_doc}, {"set_isolation_level", (PyCFunction)psyco_conn_set_isolation_level, @@ -999,14 +987,12 @@ static struct PyMethodDef connectionObject_methods[] = { METH_NOARGS, psyco_conn_isexecuting_doc}, {"cancel", (PyCFunction)psyco_conn_cancel, METH_NOARGS, psyco_conn_cancel_doc}, -#endif {NULL} }; /* object member list */ static struct PyMemberDef connectionObject_members[] = { -#ifdef PSYCOPG_EXTENSIONS {"closed", T_LONG, offsetof(connectionObject, closed), READONLY, "True if the connection is closed."}, {"encoding", T_STRING, offsetof(connectionObject, encoding), READONLY, @@ -1032,7 +1018,6 @@ static struct PyMemberDef connectionObject_members[] = { {"server_version", T_INT, offsetof(connectionObject, server_version), READONLY, "Server version."}, -#endif {NULL} }; @@ -1040,7 +1025,6 @@ static struct PyMemberDef connectionObject_members[] = { { #exc, psyco_conn_get_exception, NULL, exc ## _doc, &exc } static struct PyGetSetDef connectionObject_getsets[] = { - /* DBAPI-2.0 extensions (exception objects) */ EXCEPTION_GETTER(Error), EXCEPTION_GETTER(Warning), EXCEPTION_GETTER(InterfaceError), @@ -1051,7 +1035,6 @@ static struct PyGetSetDef connectionObject_getsets[] = { EXCEPTION_GETTER(IntegrityError), EXCEPTION_GETTER(DataError), EXCEPTION_GETTER(NotSupportedError), -#ifdef PSYCOPG_EXTENSIONS { "autocommit", (getter)psyco_conn_autocommit_get, (setter)psyco_conn_autocommit_set, @@ -1060,7 +1043,6 @@ static struct PyGetSetDef connectionObject_getsets[] = { (getter)psyco_conn_isolation_level_get, (setter)NULL, "The current isolation level." }, -#endif {NULL} }; #undef EXCEPTION_GETTER diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 40dbaef0..784c0a1e 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -559,7 +559,6 @@ psyco_curs_executemany(cursorObject *self, PyObject *args, PyObject *kwargs) } -#ifdef PSYCOPG_EXTENSIONS #define psyco_curs_mogrify_doc \ "mogrify(query, vars=None) -> str -- Return query after vars binding." @@ -622,7 +621,6 @@ psyco_curs_mogrify(cursorObject *self, PyObject *args, PyObject *kwargs) return _psyco_curs_mogrify(self, operation, vars); } -#endif /* cast method - convert an oid/string into a Python object */ @@ -1222,8 +1220,6 @@ exit: } -#ifdef PSYCOPG_EXTENSIONS - /* Return a newly allocated buffer containing the list of columns to be * copied. On error return NULL and set an exception. */ @@ -1669,8 +1665,6 @@ psyco_curs_scrollable_set(cursorObject *self, PyObject *pyvalue) return 0; } -#endif - /** the cursor object **/ @@ -1738,7 +1732,6 @@ static struct PyMethodDef cursorObject_methods[] = { {"__exit__", (PyCFunction)psyco_curs_exit, METH_VARARGS, psyco_curs_exit_doc}, /* psycopg extensions */ -#ifdef PSYCOPG_EXTENSIONS {"cast", (PyCFunction)psyco_curs_cast, METH_VARARGS, psyco_curs_cast_doc}, {"mogrify", (PyCFunction)psyco_curs_mogrify, @@ -1749,7 +1742,6 @@ static struct PyMethodDef cursorObject_methods[] = { METH_VARARGS|METH_KEYWORDS, psyco_curs_copy_to_doc}, {"copy_expert", (PyCFunction)psyco_curs_copy_expert, METH_VARARGS|METH_KEYWORDS, psyco_curs_copy_expert_doc}, -#endif {NULL} }; @@ -1775,7 +1767,6 @@ static struct PyMemberDef cursorObject_members[] = { "The current row position."}, {"connection", T_OBJECT, OFFSETOF(conn), READONLY, "The connection where the cursor comes from."}, -#ifdef PSYCOPG_EXTENSIONS {"name", T_STRING, OFFSETOF(name), READONLY}, {"statusmessage", T_OBJECT, OFFSETOF(pgstatus), READONLY, "The return message of the last command."}, @@ -1786,13 +1777,11 @@ static struct PyMemberDef cursorObject_members[] = { {"typecaster", T_OBJECT, OFFSETOF(caster), READONLY}, {"string_types", T_OBJECT, OFFSETOF(string_types), 0}, {"binary_types", T_OBJECT, OFFSETOF(binary_types), 0}, -#endif {NULL} }; /* object calculated member list */ static struct PyGetSetDef cursorObject_getsets[] = { -#ifdef PSYCOPG_EXTENSIONS { "closed", (getter)psyco_curs_get_closed, NULL, psyco_curs_closed_doc, NULL }, { "withhold", @@ -1803,7 +1792,6 @@ static struct PyGetSetDef cursorObject_getsets[] = { (getter)psyco_curs_scrollable_get, (setter)psyco_curs_scrollable_set, psyco_curs_scrollable_doc, NULL }, -#endif {NULL} }; diff --git a/psycopg/green.c b/psycopg/green.c index e7604076..ab37b8be 100644 --- a/psycopg/green.c +++ b/psycopg/green.c @@ -80,11 +80,7 @@ psyco_get_wait_callback(PyObject *self, PyObject *obj) int psyco_green() { -#ifdef PSYCOPG_EXTENSIONS return (NULL != wait_callback); -#else - return 0; -#endif } /* Return the wait callback if available. diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 43f2b179..21b1e189 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -32,8 +32,6 @@ #include -#ifdef PSYCOPG_EXTENSIONS - static void collect_error(connectionObject *conn, char **error) { @@ -490,6 +488,3 @@ lobject_truncate(lobjectObject *self, size_t len) } #endif /* PG_VERSION_HEX >= 0x080300 */ - -#endif - diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 63511149..e62ddc27 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -35,8 +35,6 @@ #include -#ifdef PSYCOPG_EXTENSIONS - /** public methods **/ /* close method - close the lobject */ @@ -272,8 +270,6 @@ psyco_lobj_truncate(lobjectObject *self, PyObject *args) Py_RETURN_NONE; } -#endif /* PG_VERSION_HEX >= 0x080300 */ - /** the lobject object **/ diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 3512f639..ae6a18bb 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -113,11 +113,7 @@ exception_from_sqlstate(const char *sqlstate) case '4': switch (sqlstate[1]) { case '0': /* Class 40 - Transaction Rollback */ -#ifdef PSYCOPG_EXTENSIONS return TransactionRollbackError; -#else - return OperationalError; -#endif case '2': /* Class 42 - Syntax Error or Access Rule Violation */ case '4': /* Class 44 - WITH CHECK OPTION Violation */ return ProgrammingError; @@ -129,11 +125,9 @@ exception_from_sqlstate(const char *sqlstate) Class 55 - Object Not In Prerequisite State Class 57 - Operator Intervention Class 58 - System Error (errors external to PostgreSQL itself) */ -#ifdef PSYCOPG_EXTENSIONS if (!strcmp(sqlstate, "57014")) return QueryCanceledError; else -#endif return OperationalError; case 'F': /* Class F0 - Configuration File Error */ return InternalError; diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index 2a90c255..eb406fd2 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -63,9 +63,7 @@ HIDDEN psyco_errors_set_RETURN psyco_errors_set psyco_errors_set_PROTO; extern HIDDEN PyObject *Error, *Warning, *InterfaceError, *DatabaseError, *InternalError, *OperationalError, *ProgrammingError, *IntegrityError, *DataError, *NotSupportedError; -#ifdef PSYCOPG_EXTENSIONS extern HIDDEN PyObject *QueryCanceledError, *TransactionRollbackError; -#endif /* python versions and compatibility stuff */ #ifndef PyMODINIT_FUNC @@ -164,13 +162,11 @@ STEALS(1) HIDDEN PyObject * psycopg_ensure_text(PyObject *obj); #define NotSupportedError_doc \ "A method or database API was used which is not supported by the database." -#ifdef PSYCOPG_EXTENSIONS #define QueryCanceledError_doc \ "Error related to SQL query cancellation." #define TransactionRollbackError_doc \ "Error causing transaction rollback (deadlocks, serialization failures, etc)." -#endif #ifdef __cplusplus } diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 04c0ed7c..3ccd2828 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -398,9 +398,7 @@ exit: PyObject *Error, *Warning, *InterfaceError, *DatabaseError, *InternalError, *OperationalError, *ProgrammingError, *IntegrityError, *DataError, *NotSupportedError; -#ifdef PSYCOPG_EXTENSIONS PyObject *QueryCanceledError, *TransactionRollbackError; -#endif /* mapping between exception names and their PyObject */ static struct { @@ -423,13 +421,11 @@ static struct { { "psycopg2.DataError", &DataError, &DatabaseError, DataError_doc }, { "psycopg2.NotSupportedError", &NotSupportedError, &DatabaseError, NotSupportedError_doc }, -#ifdef PSYCOPG_EXTENSIONS { "psycopg2.extensions.QueryCanceledError", &QueryCanceledError, &OperationalError, QueryCanceledError_doc }, { "psycopg2.extensions.TransactionRollbackError", &TransactionRollbackError, &OperationalError, TransactionRollbackError_doc }, -#endif {NULL} /* Sentinel */ }; @@ -712,12 +708,10 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS, psyco_IntervalFromMx_doc}, #endif -#ifdef PSYCOPG_EXTENSIONS {"set_wait_callback", (PyCFunction)psyco_set_wait_callback, METH_O, psyco_set_wait_callback_doc}, {"get_wait_callback", (PyCFunction)psyco_get_wait_callback, METH_NOARGS, psyco_get_wait_callback_doc}, -#endif {NULL, NULL, 0, NULL} /* Sentinel */ }; @@ -806,10 +800,8 @@ INIT_MODULE(_psycopg)(void) Py_TYPE(&diagnosticsType) = &PyType_Type; if (PyType_Ready(&diagnosticsType) == -1) goto exit; -#ifdef PSYCOPG_EXTENSIONS Py_TYPE(&lobjectType) = &PyType_Type; if (PyType_Ready(&lobjectType) == -1) goto exit; -#endif /* import mx.DateTime module, if necessary */ #ifdef HAVE_MXDATETIME @@ -904,9 +896,7 @@ INIT_MODULE(_psycopg)(void) PyModule_AddObject(module, "Float", (PyObject*)&pfloatType); PyModule_AddObject(module, "List", (PyObject*)&listType); PyModule_AddObject(module, "QuotedString", (PyObject*)&qstringType); -#ifdef PSYCOPG_EXTENSIONS PyModule_AddObject(module, "lobject", (PyObject*)&lobjectType); -#endif /* encodings dictionary in module dictionary */ PyModule_AddObject(module, "encodings", psycoEncodings); diff --git a/setup.cfg b/setup.cfg index 7f6e84e5..83962bfb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,6 @@ [build_ext] -define=PSYCOPG_EXTENSIONS,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM +define=PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM -# PSYCOPG_EXTENSIONS enables extensions to PEP-249 (you really want this) # PSYCOPG_DISPLAY_SIZE enable display size calculation (a little slower) # HAVE_PQFREEMEM should be defined on PostgreSQL >= 7.4 # PSYCOPG_DEBUG can be added to enable verbose debug information diff --git a/setup.py b/setup.py index 92ac1038..f08a4f3f 100644 --- a/setup.py +++ b/setup.py @@ -511,10 +511,7 @@ you probably need to install its companion -dev or -devel package.""" # generate a nice version string to avoid confusion when users report bugs version_flags.append('pq3') # no more a choice - -for have in parser.get('build_ext', 'define').split(','): - if have == 'PSYCOPG_EXTENSIONS': - version_flags.append('ext') +version_flags.append('ext') # no more a choice if version_flags: PSYCOPG_VERSION_EX = PSYCOPG_VERSION + " (%s)" % ' '.join(version_flags) From 68a4308c3dec3fe58ab07fb32ce2bff84a86845e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 01:25:02 +0100 Subject: [PATCH 026/132] Dropped PSYCOPG_NEW_BOOLEAN flag Introduced in 2.0 beta 8, 2006 A.D. Went absolutely untouched in 8 years of refactoring, when Python 2.5 and PostgreSQL 8.1 roamed the earth. I would say it has stood the test of the time. --- psycopg/adapter_pboolean.c | 9 --------- setup.cfg | 3 +-- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index 8a5950d8..c7e3b807 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -37,21 +37,12 @@ static PyObject * pboolean_getquoted(pbooleanObject *self, PyObject *args) { -#ifdef PSYCOPG_NEW_BOOLEAN if (PyObject_IsTrue(self->wrapped)) { return Bytes_FromString("true"); } else { return Bytes_FromString("false"); } -#else - if (PyObject_IsTrue(self->wrapped)) { - return Bytes_FromString("'t'"); - } - else { - return Bytes_FromString("'f'"); - } -#endif } static PyObject * diff --git a/setup.cfg b/setup.cfg index 83962bfb..86fdd42b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,9 @@ [build_ext] -define=PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM +define=HAVE_PQFREEMEM # PSYCOPG_DISPLAY_SIZE enable display size calculation (a little slower) # HAVE_PQFREEMEM should be defined on PostgreSQL >= 7.4 # PSYCOPG_DEBUG can be added to enable verbose debug information -# PSYCOPG_NEW_BOOLEAN to format booleans as true/false vs 't'/'f' # "pg_config" is required to locate PostgreSQL headers and libraries needed to # build psycopg2. If pg_config is not in the path or is installed under a From 1d729ab40e410bf952edfdbe363163f40fe15072 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 01:43:12 +0100 Subject: [PATCH 027/132] Dropped HAVE_PQFREEMEM flag It was necessary before PG 7.4, in versions which have long been unsupported. --- psycopg/config.h | 5 ----- setup.cfg | 2 +- setup.py | 3 ++- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/psycopg/config.h b/psycopg/config.h index 45157709..b6cd4190 100644 --- a/psycopg/config.h +++ b/psycopg/config.h @@ -152,11 +152,6 @@ static double round(double num) } #endif -/* postgresql < 7.4 does not have PQfreemem */ -#ifndef HAVE_PQFREEMEM -#define PQfreemem free -#endif - /* resolve missing isinf() function for Solaris */ #if defined (__SVR4) && defined (__sun) #include diff --git a/setup.cfg b/setup.cfg index 86fdd42b..90a47dd4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [build_ext] -define=HAVE_PQFREEMEM +define= # PSYCOPG_DISPLAY_SIZE enable display size calculation (a little slower) # HAVE_PQFREEMEM should be defined on PostgreSQL >= 7.4 diff --git a/setup.py b/setup.py index f08a4f3f..f22f6b1b 100644 --- a/setup.py +++ b/setup.py @@ -538,7 +538,8 @@ else: # when called e.g. with "pip -e git+url'. This results in declarations # duplicate on the commandline, which I hope is not a problem. for define in parser.get('build_ext', 'define').split(','): - define_macros.append((define, '1')) + if define: + define_macros.append((define, '1')) # build the extension From 353b36c6575d0b7280470938d6d4cbed221bbb6f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 22 Aug 2014 06:08:38 +0100 Subject: [PATCH 028/132] Readme cleaned up and converted to reST Dropped outdated info and references to the INSTALL, which is less up-to-date than the docs. Converted to reST to have it prettier on GitHub. --- MANIFEST.in | 2 +- README | 38 -------------------------------------- README.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ psycopg2.cproj | 2 +- 4 files changed, 46 insertions(+), 40 deletions(-) delete mode 100644 README create mode 100644 README.rst diff --git a/MANIFEST.in b/MANIFEST.in index 1aa0ef43..58aa51ad 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,5 +6,5 @@ recursive-include doc README HACKING SUCCESS COPYING.LESSER pep-0249.txt recursive-include doc/src *.rst *.py *.css Makefile recursive-include scripts *.py *.sh include scripts/maketypes.sh scripts/buildtypes.py -include AUTHORS README INSTALL LICENSE NEWS +include AUTHORS README.rst INSTALL LICENSE NEWS include PKG-INFO MANIFEST.in MANIFEST setup.py setup.cfg Makefile diff --git a/README b/README deleted file mode 100644 index 7466bf91..00000000 --- a/README +++ /dev/null @@ -1,38 +0,0 @@ -psycopg2 - Python-PostgreSQL Database Adapter -******************************************** - -psycopg2 is a PostgreSQL database adapter for the Python programming -language. psycopg2 was written with the aim of being very small and fast, -and stable as a rock. - -psycopg2 is different from the other database adapter because it was -designed for heavily multi-threaded applications that create and destroy -lots of cursors and make a conspicuous number of concurrent INSERTs or -UPDATEs. psycopg2 also provides full asynchronous operations and support -for coroutine libraries. - -psycopg2 can compile and run on Linux, FreeBSD, Solaris, MacOS X and -Windows architecture. It supports Python versions from 2.4 onwards and -PostgreSQL versions from 7.4 onwards. - -psycopg2 is free software ("free as in freedom" but I like beer too.) -It is licensed under the GNU Lesser General Public License, version 3 or -later plus an exception to allow OpenSSL (libpq) linking; see LICENSE for -more details. - -Documentation -------------- - -Start by reading the INSTALL file. More information about psycopg2 extensions -to the DBAPI-2.0 is available in the files located in the doc/ direcory. -Example code can be found in the examples/ directory. If you make any changes -to the code make sure to run the unit tests localed in tests/. - -Online documentation can be found at: http://initd.org/psycopg/ - -If you stumble upon any bugs, please tell us at: http://psycopg.lighthouseapp.com/ - -Contributors ------------- - -For a list of contributors to the project, see the AUTHORS file. diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..d83d2d4c --- /dev/null +++ b/README.rst @@ -0,0 +1,44 @@ +psycopg2 - Python-PostgreSQL Database Adapter +============================================= + +Psycopg is the most popular PostgreSQL database adapter for the Python +programming language. Its main features are the complete implementation of +the Python DB API 2.0 specification and the thread safety (several threads can +share the same connection). It was designed for heavily multi-threaded +applications that create and destroy lots of cursors and make a large number +of concurrent "INSERT"s or "UPDATE"s. + +Psycopg 2 is mostly implemented in C as a libpq wrapper, resulting in being +both efficient and secure. It features client-side and server-side cursors, +asynchronous communication and notifications, "COPY TO/COPY FROM" support. +Many Python types are supported out-of-the-box and adapted to matching +PostgreSQL data types; adaptation can be extended and customized thanks to a +flexible objects adaptation system. + +Psycopg 2 is both Unicode and Python 3 friendly. + + +Documentation +------------- + +Documentation is included in the 'doc' directory and is `available online`__. + +.. __: http://initd.org/psycopg/docs/ + + +Installation +------------ + +If all the dependencies are met (i.e. you have the Python and libpq +development packages installed in your system) the standard:: + + python setup.py build + sudo python setup.py install + +should work no problem. In case you have any problem check the 'install' and +the 'faq' documents in the docs or online. + +For any other resource (source code repository, bug tracker, mailing list) +please check the `project homepage`__. + +.. __: http://initd.org/psycopg/ diff --git a/psycopg2.cproj b/psycopg2.cproj index 1640ab1d..42cfca9d 100644 --- a/psycopg2.cproj +++ b/psycopg2.cproj @@ -44,7 +44,7 @@ - + From d8bbaf04815a507142a6e67be6e6196be762f395 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 23 Aug 2014 18:57:12 +0100 Subject: [PATCH 029/132] Added info about running the test suite in the docs They were in the INSTALL file, which is quite out-of-date now. Also fixed a couple of other things. --- doc/src/install.rst | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/doc/src/install.rst b/doc/src/install.rst index d2a28e49..ec1eeea8 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -205,6 +205,33 @@ supported. +.. index:: + single: tests + +.. _test-suite: + +Running the test suite +^^^^^^^^^^^^^^^^^^^^^^ + +The included ``Makefile`` allows to run all the tests included in the +distribution. Just run:: + + make + make check + +The tests run against a database called ``psycopg2_test`` on UNIX socket and +the standard port. You can configure a different database to run the test by +setting the environment variables: + +- :envvar:`PSYCOPG2_TESTDB` +- :envvar:`PSYCOPG2_TESTDB_HOST` +- :envvar:`PSYCOPG2_TESTDB_PORT` +- :envvar:`PSYCOPG2_TESTDB_USER` + +The database should already exist before running the tests. + + + .. index:: single: debug single: PSYCOPG_DEBUG @@ -225,13 +252,13 @@ order to create a debug package: - :ref:`Compile and install ` the package. -- Set the :envvar:`PSYCOPG_DEBUG` variable:: +- Set the :envvar:`PSYCOPG_DEBUG` environment variable:: $ export PSYCOPG_DEBUG=1 - Run your program (making sure that the `!psycopg2` package imported is the one you just compiled and not e.g. the system one): you will have a copious - stream of informations printed on stdout. + stream of informations printed on stderr. .. __: http://initd.org/psycopg/download/ From 2be26804d30a3301d66ad9b52cc273cd61041362 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 23 Aug 2014 19:02:43 +0100 Subject: [PATCH 030/132] Dropped content from the INSTALL file The docs have it right. --- INSTALL | 105 ++------------------------------------------------------ 1 file changed, 3 insertions(+), 102 deletions(-) diff --git a/INSTALL b/INSTALL index d5eefe51..bb509ac2 100644 --- a/INSTALL +++ b/INSTALL @@ -1,103 +1,4 @@ -Compiling and installing psycopg -******************************** - -** Important note: if you plan to use psycopg2 in a multithreaded application, - make sure that your libpq has been compiled with the --with-thread-safety - option. psycopg2 will work correctly even with a non-thread-safe libpq but - libpq will leak memory. - -psycopg2 uses distutils for its build process, so most of the process is -executed by the setup.py script. Before building psycopg look at -setup.cfg file and change any settings to follow your system (or taste); -then: - - python setup.py build - -to build in the local directory; and: - - python setup.py install - -to install system-wide. - - -Common errors and build problems -================================ - -One of the most common errors is trying to build psycopg without the right -development headers for PostgreSQL, Python or both. If you get errors, look -for the following messages and then take the appropriate action: - -libpq-fe.h: No such file or directory - PostgreSQL headers are not properly installed on your system or are - installed in a non default path. First make sure they are installed, then - check setup.cfg and make sure pg_config points to a valid pg_config - executable. If you don't have a working pg_config try to play with the - include_dirs variable (and note that a working pg_config is better.) - - -Running the test suite -====================== - -The included Makefile allows to run all the tests included in the -distribution. Just use: - - make - make check - -The tests are run against a database called psycopg2_test on unix socket -and standard port. You can configure a different database to run the test -by setting the environment variables: - -- PSYCOPG2_TESTDB -- PSYCOPG2_TESTDB_HOST -- PSYCOPG2_TESTDB_PORT -- PSYCOPG2_TESTDB_USER - -The database should be created before running the tests. - -The standard Python unittest is used to run the tests. But if unittest2 is -found it will be used instead, with the result of having more informations -about skipped tests. - - -Building the documentation -========================== - -In order to build the documentation included in the distribution, use - - make env - make docs - -The first command will install all the dependencies (Sphinx, Docutils) in -an 'env' directory in the project tree. The second command will build both -the html format (in the 'doc/html' directory) and in plain text -(doc/psycopg2.txt) - - -Using setuptools and EasyInstall -================================ - -If setuptools are installed on your system you can easily create an egg for -psycopg and install it. Download the source distribution (if you're reading -this file you probably already have) and then edit setup.cfg to your taste -and build from the source distribution top-level directory using: - - easy_install . - - -Compiling under Windows with mingw32 -==================================== - -You can compile psycopg under Windows platform with mingw32 -(http://www.mingw.org/) compiler. MinGW is also shipped with IDEs such as -Dev-C++ (http://www.bloodshed.net/devcpp.html) and Code::Blocks -(http://www.codeblocks.org). gcc binaries should be in your PATH. - -You need a PostgreSQL with include and library files installed. At least v8.0 -is required. - -First you need to create a libpython2X.a as described in -http://starship.python.net/crew/kernr/mingw32/Notes.html. Then run: - - python setup.py build_ext --compiler=mingw32 install +Installation instructions are included in the docs. +Please check the 'doc/src/install.rst' file or online at +. From c0e94ad01fb0ae2763518b40bc1a5bf4b63c0f46 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 23 Aug 2014 19:10:40 +0100 Subject: [PATCH 031/132] Dropped outdated HACKING file --- MANIFEST.in | 2 +- doc/HACKING | 43 ------------------------------------------- psycopg2.cproj | 1 - 3 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 doc/HACKING diff --git a/MANIFEST.in b/MANIFEST.in index 58aa51ad..1603b1be 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ recursive-include psycopg *.c *.h *.manifest recursive-include lib *.py recursive-include tests *.py recursive-include examples *.py somehackers.jpg whereareyou.jpg -recursive-include doc README HACKING SUCCESS COPYING.LESSER pep-0249.txt +recursive-include doc README SUCCESS COPYING.LESSER pep-0249.txt recursive-include doc/src *.rst *.py *.css Makefile recursive-include scripts *.py *.sh include scripts/maketypes.sh scripts/buildtypes.py diff --git a/doc/HACKING b/doc/HACKING deleted file mode 100644 index f60474ce..00000000 --- a/doc/HACKING +++ /dev/null @@ -1,43 +0,0 @@ -General information -******************* - -Some help to people wanting to hack on psycopg. First of all, note that -*every* function in the psycopg module source code is prefixed by one of the -following words: - - psyco is used for function directly callable from python (i.e., functions - in the psycopg module itself.) the only notable exception is the - source code for the module itself, that uses "psyco" even for C-only - functions. - - conn is used for functions related to connection objects. - - curs is used for functions related to cursor objects. - - typecast is used for typecasters and utility function related to - typecaster creation and registration. - -Pythonic definition of types and functions available from python are defined -in *_type.c files. Internal functions, callable only from C are located in -*_int.c files and extensions to the DBAPI can be found in the *_ext.c files. - - -Patches -******* - -If you submit a patch, please send a diff generated with the "-u" switch. -Also note that I don't like that much cosmetic changes (like renaming -already existing variables) and I will rewrap the patch to 78 columns -anyway, so it is much better if you do that beforehand. - - -The type system -*************** - -Simple types, like integers and strings, are converted to python base types -(the conversion functions are in typecast_base.c). Complex types are -converted to ad-hoc types, defined in the typeobj_*.{c,h} files. The -conversion function are in the other typecast_*.c files. typecast.c defines -the basic utility functions (available through the psycopg module) used when -defining new typecasters from C and python. - diff --git a/psycopg2.cproj b/psycopg2.cproj index 42cfca9d..7755b961 100644 --- a/psycopg2.cproj +++ b/psycopg2.cproj @@ -47,7 +47,6 @@ - From 843de765a1b5fd081b57e102bb4b1c1d70470e91 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 14:57:26 +0100 Subject: [PATCH 032/132] Fixed doc example about Error attributes Catch the Error, not everything! Also, whitespaces. --- doc/src/module.rst | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/doc/src/module.rst b/doc/src/module.rst index feaef516..8de9f87e 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -111,7 +111,7 @@ The module interface respects the standard defined in the |DBAPI|_. -.. index:: +.. index:: single: Exceptions; DB API .. _dbapi-exceptions: @@ -122,12 +122,12 @@ Exceptions In compliance with the |DBAPI|_, the module makes informations about errors available through the following exceptions: -.. exception:: Warning - +.. exception:: Warning + Exception raised for important warnings like data truncations while inserting, etc. It is a subclass of the Python `~exceptions.StandardError`. - -.. exception:: Error + +.. exception:: Error Exception that is the base class of all other error exceptions. You can use this to catch all errors with one single `!except` statement. Warnings @@ -150,7 +150,7 @@ available through the following exceptions: >>> try: ... cur.execute("SELECT * FROM barf") - ... except Exception, e: + ... except psycopg2.Error as e: ... pass >>> e.pgcode @@ -159,6 +159,7 @@ available through the following exceptions: ERROR: relation "barf" does not exist LINE 1: SELECT * FROM barf ^ + .. attribute:: cursor The cursor the exception was raised from; `None` if not applicable. @@ -170,7 +171,7 @@ available through the following exceptions: >>> try: ... cur.execute("SELECT * FROM barf") - ... except Exception, e: + ... except psycopg2.Error, e: ... pass >>> e.diag.severity @@ -195,41 +196,41 @@ available through the following exceptions: Exception raised for errors that are related to the database. It is a subclass of `Error`. - + .. exception:: DataError - + Exception raised for errors that are due to problems with the processed data like division by zero, numeric value out of range, etc. It is a subclass of `DatabaseError`. - + .. exception:: OperationalError - + Exception raised for errors that are related to the database's operation and not necessarily under the control of the programmer, e.g. an unexpected disconnect occurs, the data source name is not found, a transaction could not be processed, a memory allocation error occurred during processing, etc. It is a subclass of `DatabaseError`. - -.. exception:: IntegrityError - + +.. exception:: IntegrityError + Exception raised when the relational integrity of the database is affected, e.g. a foreign key check fails. It is a subclass of `DatabaseError`. - -.. exception:: InternalError - + +.. exception:: InternalError + Exception raised when the database encounters an internal error, e.g. the cursor is not valid anymore, the transaction is out of sync, etc. It is a subclass of `DatabaseError`. - + .. exception:: ProgrammingError - + Exception raised for programming errors, e.g. table not found or already exists, syntax error in the SQL statement, wrong number of parameters specified, etc. It is a subclass of `DatabaseError`. - + .. exception:: NotSupportedError - + Exception raised in case a method or database API was used which is not supported by the database, e.g. requesting a `!rollback()` on a connection that does not support transaction or has transactions turned From 2ad67ee56e63f6bc23c86672864f7c3dcd964672 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 16:18:13 +0100 Subject: [PATCH 033/132] Move the `extras` module up in the docs It is more interesting than `tz` and `pool`. --- doc/src/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/index.rst b/doc/src/index.rst index 04fb929a..5cf0f24e 100644 --- a/doc/src/index.rst +++ b/doc/src/index.rst @@ -43,9 +43,9 @@ Psycopg 2 is both Unicode and Python 3 friendly. cursor advanced extensions + extras tz pool - extras errorcodes faq news From b8b15637aae8ec17dd3f8157b0c4651d6b334658 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 17:02:57 +0100 Subject: [PATCH 034/132] Take the PYTHONPATH into account when building the docs --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index db0c2574..ce9063d0 100644 --- a/Makefile +++ b/Makefile @@ -122,10 +122,10 @@ MANIFEST: MANIFEST.in $(SOURCE) # docs depend on the build as it partly use introspection. doc/html/genindex.html: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) - PYTHONPATH=$(ENV_LIB):$(BUILD_DIR) $(MAKE) SPHINXBUILD=$(ENV_BIN)/sphinx-build -C doc html + PYTHONPATH=$(ENV_LIB):$(BUILD_DIR):$$PYTHONPATH $(MAKE) SPHINXBUILD=$(ENV_BIN)/sphinx-build -C doc html doc/psycopg2.txt: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) - PYTHONPATH=$(ENV_LIB):$(BUILD_DIR) $(MAKE) SPHINXBUILD=$(ENV_BIN)/sphinx-build -C doc text + PYTHONPATH=$(ENV_LIB):$(BUILD_DIR):$$PYTHONPATH $(MAKE) SPHINXBUILD=$(ENV_BIN)/sphinx-build -C doc text doc/docs.zip: doc/html/genindex.html (cd doc/html && zip -r ../docs.zip *) From c1da93a7b2a1da9fc181f0b9813d9a6c73efcdbe Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 22:04:43 +0100 Subject: [PATCH 035/132] Document we known 9.4 errorcodes --- doc/src/errorcodes.rst | 2 +- scripts/make_errorcodes.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/src/errorcodes.rst b/doc/src/errorcodes.rst index f659023d..d662d0c8 100644 --- a/doc/src/errorcodes.rst +++ b/doc/src/errorcodes.rst @@ -50,7 +50,7 @@ An example of the available constants defined in the module: '42P01' Constants representing all the error values defined by PostgreSQL versions -between 8.1 and 9.3 are included in the module. +between 8.1 and 9.4 are included in the module. .. autofunction:: lookup(code) diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py index 8a8d54df..d6e25337 100755 --- a/scripts/make_errorcodes.py +++ b/scripts/make_errorcodes.py @@ -31,6 +31,7 @@ def main(): filename = sys.argv[1] file_start = read_base_file(filename) + # If you add a version to the list fix the docs (errorcodes.rst, err.rst) classes, errors = fetch_errors( ['8.1', '8.2', '8.3', '8.4', '9.0', '9.1', '9.2', '9.3', '9.4']) From 4a4e0d8f9a5440890b154e06ed8dfae517b1987d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 23:07:22 +0100 Subject: [PATCH 036/132] Self-updating copyright year in docs --- doc/src/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/src/conf.py b/doc/src/conf.py index 5937a7b4..17a61f58 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -42,7 +42,9 @@ master_doc = 'index' # General information about the project. project = u'Psycopg' -copyright = u'2001-2013, Federico Di Gregorio. Documentation by Daniele Varrazzo' +from datetime import date +year = date.today().year +copyright = u'2001-%s, Federico Di Gregorio, Daniele Varrazzo' % year # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From f7ee006bef9eb1f0d515dd810d32e30fca1efc52 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 23:08:01 +0100 Subject: [PATCH 037/132] Use virtualenv as build environment for the docs --- Makefile | 30 +++++------------------------- doc/Makefile | 19 +++++++++++++------ doc/requirements.txt | 3 +++ 3 files changed, 21 insertions(+), 31 deletions(-) create mode 100644 doc/requirements.txt diff --git a/Makefile b/Makefile index ce9063d0..68eaf14a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ # # Build the documentation:: # -# make env +# make env (once) # make docs # # Create a source package:: @@ -20,9 +20,6 @@ PYTHON := python$(PYTHON_VERSION) PYTHON_VERSION ?= $(shell $(PYTHON) -c 'import sys; print ("%d.%d" % sys.version_info[:2])') BUILD_DIR = $(shell pwd)/build/lib.$(PYTHON_VERSION) -ENV_DIR = $(shell pwd)/env/py-$(PYTHON_VERSION) -ENV_BIN = $(ENV_DIR)/bin -ENV_LIB = $(ENV_DIR)/lib SOURCE_C := $(wildcard psycopg/*.c psycopg/*.h) SOURCE_PY := $(wildcard lib/*.py) @@ -46,9 +43,6 @@ endif VERSION := $(shell grep PSYCOPG_VERSION setup.py | head -1 | sed -e "s/.*'\(.*\)'/\1/") SDIST := dist/psycopg2-$(VERSION).tar.gz -EASY_INSTALL = PYTHONPATH=$(ENV_LIB) $(ENV_BIN)/easy_install-$(PYTHON_VERSION) -d $(ENV_LIB) -s $(ENV_BIN) -EZ_SETUP = $(ENV_BIN)/ez_setup.py - .PHONY: env check clean default: package @@ -68,22 +62,8 @@ docs-zip: doc/docs.zip sdist: $(SDIST) -# The environment is currently required to build the documentation. -# It is not clean by 'make clean' - -env: easy_install - mkdir -p $(ENV_BIN) - mkdir -p $(ENV_LIB) - $(EASY_INSTALL) docutils - $(EASY_INSTALL) sphinx - -easy_install: ez_setup - PYTHONPATH=$(ENV_LIB) $(PYTHON) $(EZ_SETUP) -d $(ENV_LIB) -s $(ENV_BIN) setuptools - -ez_setup: - mkdir -p $(ENV_BIN) - mkdir -p $(ENV_LIB) - wget -O $(EZ_SETUP) http://peak.telecommunity.com/dist/ez_setup.py +env: + $(MAKE) -C doc $@ check: PYTHONPATH=$(BUILD_DIR):$(PYTHONPATH) $(PYTHON) -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" --verbose @@ -122,10 +102,10 @@ MANIFEST: MANIFEST.in $(SOURCE) # docs depend on the build as it partly use introspection. doc/html/genindex.html: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) - PYTHONPATH=$(ENV_LIB):$(BUILD_DIR):$$PYTHONPATH $(MAKE) SPHINXBUILD=$(ENV_BIN)/sphinx-build -C doc html + PYTHONPATH=:$(BUILD_DIR):$$PYTHONPATH $(MAKE) -C doc html doc/psycopg2.txt: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) - PYTHONPATH=$(ENV_LIB):$(BUILD_DIR):$$PYTHONPATH $(MAKE) SPHINXBUILD=$(ENV_BIN)/sphinx-build -C doc text + PYTHONPATH=$(BUILD_DIR):$$PYTHONPATH $(MAKE) -C doc text doc/docs.zip: doc/html/genindex.html (cd doc/html && zip -r ../docs.zip *) diff --git a/doc/Makefile b/doc/Makefile index 54b59232..6d639526 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,23 +1,30 @@ -.PHONY: help clean html text doctest +.PHONY: env help clean html text doctest docs: html text check: doctest +# The environment is currently required to build the documentation. +# It is not clean by 'make clean' + help: - cd src && $(MAKE) $@ + $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ html: - cd src && $(MAKE) $@ + $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ cp -r src/_build/html . text: - cd src && $(MAKE) $@ + $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ cd src && tools/stitch_text.py index.rst _build/text > ../psycopg2.txt doctest: - cd src && $(MAKE) $@ + $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ clean: - cd src && $(MAKE) $@ + $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ rm -rf html psycopg2.txt + +env: requirements.txt + virtualenv env + ./env/bin/pip install -r requirements.txt diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 00000000..b5323e79 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,3 @@ +# Packages only needed to build the docs +Pygments>=1.5 +Sphinx>=1.2,<=1.3 From df9fbc515d6f586e84d7853965f2bc544177ac6a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 24 Aug 2014 23:33:14 +0100 Subject: [PATCH 038/132] Direct link to the install docs in the readme --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d83d2d4c..51d2d6b6 100644 --- a/README.rst +++ b/README.rst @@ -36,7 +36,9 @@ development packages installed in your system) the standard:: sudo python setup.py install should work no problem. In case you have any problem check the 'install' and -the 'faq' documents in the docs or online. +the 'faq' documents in the docs or online__. + +.. __: http://initd.org/psycopg/docs/install.html For any other resource (source code repository, bug tracker, mailing list) please check the `project homepage`__. From 54d904138dc7c3f8d39fa928c836474faa18dcd6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 25 Aug 2014 23:08:38 +0100 Subject: [PATCH 039/132] Don't import psycopg2.tz into the C extension This makes possible to import _psycopg directly, after adding the package directory to the pythonpath. This enables hacks such as: sys.path.insert(0, '/path/to/psycopg2') import _psycopg sys.modules['psycopg2._psycopg'] = _psycopg sys.path.pop(0) which can work around e.g. the problem of #201, freeze that cannot freeze psycopg2. Well, freeze cannot freeze it because it's just not designed to deal with C extensions. At least now the frozen application can hack the pythonpath and work around the limitation by importing _psycopg as above and then doing the rest of the imports normally. Keeping long-lived references to python objects is bad anyway: the tz module couldn't be reloaded before. --- psycopg/adapter_datetime.c | 40 ++++++++++++++++++++++---------------- psycopg/cursor_type.c | 16 ++++++++++----- psycopg/psycopgmodule.c | 17 ---------------- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index aa26d383..b6a46e20 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -35,9 +35,6 @@ #include -extern HIDDEN PyObject *pyPsycopgTzModule; -extern HIDDEN PyObject *pyPsycopgTzLOCAL; - int psyco_adapter_datetime_init(void) { @@ -386,9 +383,9 @@ psyco_DateFromTicks(PyObject *self, PyObject *args) Py_DECREF(args); } } - else { - PyErr_SetString(InterfaceError, "failed localtime call"); - } + else { + PyErr_SetString(InterfaceError, "failed localtime call"); + } return res; } @@ -414,9 +411,9 @@ psyco_TimeFromTicks(PyObject *self, PyObject *args) Py_DECREF(args); } } - else { - PyErr_SetString(InterfaceError, "failed localtime call"); - } + else { + PyErr_SetString(InterfaceError, "failed localtime call"); + } return res; } @@ -424,6 +421,8 @@ psyco_TimeFromTicks(PyObject *self, PyObject *args) PyObject * psyco_TimestampFromTicks(PyObject *self, PyObject *args) { + PyObject *m = NULL; + PyObject *tz = NULL; PyObject *res = NULL; struct tm tm; time_t t; @@ -432,18 +431,25 @@ psyco_TimestampFromTicks(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "d", &ticks)) return NULL; + /* get psycopg2.tz.LOCAL from pythonland */ + if (!(m = PyImport_ImportModule("psycopg2.tz"))) { goto exit; } + if (!(tz = PyObject_GetAttrString(m, "LOCAL"))) { goto exit; } + t = (time_t)floor(ticks); ticks -= (double)t; - if (localtime_r(&t, &tm)) { - res = _psyco_Timestamp( - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, - tm.tm_hour, tm.tm_min, (double)tm.tm_sec + ticks, - pyPsycopgTzLOCAL); + if (!localtime_r(&t, &tm)) { + PyErr_SetString(InterfaceError, "failed localtime call"); + goto exit; } - else { - PyErr_SetString(InterfaceError, "failed localtime call"); - } + res = _psyco_Timestamp( + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, (double)tm.tm_sec + ticks, + tz); + +exit: + Py_DECREF(tz); + Py_XDECREF(m); return res; } diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 784c0a1e..6927c398 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -39,9 +39,6 @@ #include -extern PyObject *pyPsycopgTzFixedOffsetTimezone; - - /** DBAPI methods **/ /* close method - close the cursor */ @@ -1830,8 +1827,17 @@ cursor_setup(cursorObject *self, connectionObject *conn, const char *name) self->tuple_factory = Py_None; /* default tzinfo factory */ - Py_INCREF(pyPsycopgTzFixedOffsetTimezone); - self->tzinfo_factory = pyPsycopgTzFixedOffsetTimezone; + { + PyObject *m = NULL; + if ((m = PyImport_ImportModule("psycopg2.tz"))) { + self->tzinfo_factory = PyObject_GetAttrString( + m, "FixedOffsetTimezone"); + Py_DECREF(m); + } + if (!self->tzinfo_factory) { + return -1; + } + } Dprintf("cursor_setup: good cursor object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 3ccd2828..745dd6e3 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -58,11 +58,6 @@ #include "psycopg/adapter_datetime.h" HIDDEN PyObject *pyDateTimeModuleP = NULL; -/* pointers to the psycopg.tz classes */ -HIDDEN PyObject *pyPsycopgTzModule = NULL; -HIDDEN PyObject *pyPsycopgTzLOCAL = NULL; -HIDDEN PyObject *pyPsycopgTzFixedOffsetTimezone = NULL; - HIDDEN PyObject *psycoEncodings = NULL; #ifdef PSYCOPG_DEBUG @@ -835,18 +830,6 @@ INIT_MODULE(_psycopg)(void) Py_TYPE(&pydatetimeType) = &PyType_Type; if (PyType_Ready(&pydatetimeType) == -1) goto exit; - /* import psycopg2.tz anyway (TODO: replace with C-level module?) */ - pyPsycopgTzModule = PyImport_ImportModule("psycopg2.tz"); - if (pyPsycopgTzModule == NULL) { - Dprintf("initpsycopg: can't import psycopg2.tz module"); - PyErr_SetString(PyExc_ImportError, "can't import psycopg2.tz module"); - goto exit; - } - pyPsycopgTzLOCAL = - PyObject_GetAttrString(pyPsycopgTzModule, "LOCAL"); - pyPsycopgTzFixedOffsetTimezone = - PyObject_GetAttrString(pyPsycopgTzModule, "FixedOffsetTimezone"); - /* initialize the module and grab module's dictionary */ #if PY_MAJOR_VERSION < 3 module = Py_InitModule("_psycopg", psycopgMethods); From ea54aa77eda09c948a078c4cd7264b0c354b67e7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 26 Aug 2014 03:42:34 +0100 Subject: [PATCH 040/132] Added test to verify _psycopg can be imported --- NEWS | 2 ++ tests/test_module.py | 27 ++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 048c646d..8e739a0e 100644 --- a/NEWS +++ b/NEWS @@ -24,6 +24,8 @@ What's new in psycopg 2.5.4 - Don't ignore silently the `cursor.callproc` argument without a length. - Added a few errors missing from `~psycopg2.errorcodes`, defined by PostgreSQL but not documented. +- Make sure the internal `_psycopg.so` module can be imported stand-alone + (to allow modules juggling such as the one described in :ticket:`#201`). What's new in psycopg 2.5.3 diff --git a/tests/test_module.py b/tests/test_module.py index b2f5279d..608f703d 100755 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -22,8 +22,12 @@ # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. +import os +import sys +from subprocess import Popen + from testutils import unittest, skip_before_python, skip_before_postgres -from testutils import ConnectingTestCase, skip_copy_if_green +from testutils import ConnectingTestCase, skip_copy_if_green, script_to_py3 import psycopg2 @@ -295,6 +299,27 @@ class ExceptionsTestCase(ConnectingTestCase): self.assert_(e1.cursor is None) +class TestExtensionModule(unittest.TestCase): + def test_import_internal(self): + # check that the internal package can be imported "naked" + # we may break this property if there is a compelling reason to do so, + # however having it allows for some import juggling such as the one + # required in ticket #201. + pkgdir = os.path.dirname(psycopg2.__file__) + pardir = os.path.dirname(pkgdir) + self.assert_(pardir in sys.path) + script = (""" +import sys +sys.path.remove(%r) +sys.path.insert(0, %r) +import _psycopg +""" % (pardir, pkgdir)) + + proc = Popen([sys.executable, '-c', script_to_py3(script)]) + proc.communicate() + self.assertEqual(0, proc.returncode) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 2d601ef157ffcd0d15749fbfb2d5e7fc684144fb Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 26 Aug 2014 02:27:51 +0100 Subject: [PATCH 041/132] Fixed memory leak with large objects Deallocating closed large objects failed to decrement the connection refcount. The fact the lobject is closed doesn't matter for refcount. Issue detected by the always useful scripts/refcounter.py With an extra bit of unrequested whitespace love. --- NEWS | 1 + psycopg/lobject_type.c | 13 ++++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index 8e739a0e..66d9aac8 100644 --- a/NEWS +++ b/NEWS @@ -26,6 +26,7 @@ What's new in psycopg 2.5.4 PostgreSQL but not documented. - Make sure the internal `_psycopg.so` module can be imported stand-alone (to allow modules juggling such as the one described in :ticket:`#201`). +- Fixed memory leak with large objects (regression introduced in 2.5.3). What's new in psycopg 2.5.3 diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index e62ddc27..ef442f90 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -50,7 +50,7 @@ psyco_lobj_close(lobjectObject *self, PyObject *args) opened large objects */ if (!lobject_is_closed(self) && !self->conn->autocommit - && self->conn->mark == self->mark) + && self->conn->mark == self->mark) { Dprintf("psyco_lobj_close: closing lobject at %p", self); if (lobject_close(self) < 0) @@ -169,14 +169,14 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) int pos=0; if (!PyArg_ParseTuple(args, "i|i", &offset, &whence)) - return NULL; + return NULL; EXC_IF_LOBJ_CLOSED(self); EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); if ((pos = lobject_seek(self, offset, whence)) < 0) - return NULL; + return NULL; return PyInt_FromLong((long)pos); } @@ -196,7 +196,7 @@ psyco_lobj_tell(lobjectObject *self, PyObject *args) EXC_IF_LOBJ_UNMARKED(self); if ((pos = lobject_tell(self)) < 0) - return NULL; + return NULL; return PyInt_FromLong((long)pos); } @@ -329,11 +329,10 @@ lobject_setup(lobjectObject *self, connectionObject *conn, return -1; } + Py_INCREF((PyObject*)conn); self->conn = conn; self->mark = conn->mark; - Py_INCREF((PyObject*)self->conn); - self->fd = -1; self->oid = InvalidOid; @@ -354,8 +353,8 @@ lobject_dealloc(PyObject* obj) if (self->conn && self->fd != -1) { if (lobject_close(self) < 0) PyErr_Print(); - Py_XDECREF((PyObject*)self->conn); } + Py_CLEAR(self->conn); PyMem_Free(self->smode); Dprintf("lobject_dealloc: deleted lobject object at %p, refcnt = " From ccc30e1877a76c5affe1cdd9c7384f62e54a1d63 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 26 Aug 2014 02:59:28 +0100 Subject: [PATCH 042/132] Clear adapters in test to maintain reference count --- tests/test_types_extras.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 46a34217..b81cecab 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -27,6 +27,7 @@ from testutils import py3_raises_typeerror import psycopg2 import psycopg2.extras +import psycopg2.extensions as ext from psycopg2.extensions import b @@ -111,9 +112,9 @@ class TypesExtrasTests(ConnectingTestCase): def test_adapt_fail(self): class Foo(object): pass self.assertRaises(psycopg2.ProgrammingError, - psycopg2.extensions.adapt, Foo(), psycopg2.extensions.ISQLQuote, None) + psycopg2.extensions.adapt, Foo(), ext.ISQLQuote, None) try: - psycopg2.extensions.adapt(Foo(), psycopg2.extensions.ISQLQuote, None) + psycopg2.extensions.adapt(Foo(), ext.ISQLQuote, None) except psycopg2.ProgrammingError, err: self.failUnless(str(err) == "can't adapt type 'Foo'") @@ -460,7 +461,6 @@ class AdaptTypeTestCase(ConnectingTestCase): def test_none_fast_path(self): # the None adapter is not actually invoked in regular adaptation - ext = psycopg2.extensions class WonkyAdapter(object): def __init__(self, obj): pass @@ -923,7 +923,7 @@ class JsonTestCase(ConnectingTestCase): self.assertEqual(curs.mogrify("%s", (obj,)), b("""'{"a": 123}'""")) finally: - del psycopg2.extensions.adapters[dict, psycopg2.extensions.ISQLQuote] + del psycopg2.extensions.adapters[dict, ext.ISQLQuote] def test_type_not_available(self): @@ -1632,6 +1632,9 @@ class RangeCasterTestCase(ConnectingTestCase): self.assert_(not r1.lower_inc) self.assert_(r1.upper_inc) + # clear the adapters to allow precise count by scripts/refcounter.py + del ext.adapters[rc.range, ext.ISQLQuote] + def test_range_escaping(self): from psycopg2.extras import register_range cur = self.conn.cursor() @@ -1683,6 +1686,9 @@ class RangeCasterTestCase(ConnectingTestCase): self.assertEqual(ranges[i].lower_inf, r.lower_inf) self.assertEqual(ranges[i].upper_inf, r.upper_inf) + # clear the adapters to allow precise count by scripts/refcounter.py + del ext.adapters[TextRange, ext.ISQLQuote] + def test_range_not_found(self): from psycopg2.extras import register_range cur = self.conn.cursor() @@ -1716,6 +1722,10 @@ class RangeCasterTestCase(ConnectingTestCase): register_range, 'rs.r1', 'FailRange', cur) cur.execute("rollback to savepoint x;") + # clear the adapters to allow precise count by scripts/refcounter.py + for r in [ra1, ra2, rars2, rars3]: + del ext.adapters[r.range, ext.ISQLQuote] + decorate_all_tests(RangeCasterTestCase, skip_if_no_range) From 6705e4051dc97256fb1d2f59773c84d43dcb0589 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 28 Aug 2014 02:05:54 +0100 Subject: [PATCH 043/132] Dropped creation of errcodes with missing spec field On further inspection these names are just aliases for values already defined: we don't need the duplication. --- NEWS | 2 -- lib/errorcodes.py | 4 ---- scripts/make_errorcodes.py | 11 +++-------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/NEWS b/NEWS index 66d9aac8..890da26c 100644 --- a/NEWS +++ b/NEWS @@ -22,8 +22,6 @@ What's new in psycopg 2.5.4 (:ticket:`#228`). - Cursors :sql:`WITH HOLD` can be used in autocommit (:ticket:`#229`). - Don't ignore silently the `cursor.callproc` argument without a length. -- Added a few errors missing from `~psycopg2.errorcodes`, defined by - PostgreSQL but not documented. - Make sure the internal `_psycopg.so` module can be imported stand-alone (to allow modules juggling such as the one described in :ticket:`#201`). - Fixed memory leak with large objects (regression introduced in 2.5.3). diff --git a/lib/errorcodes.py b/lib/errorcodes.py index e004cac5..12c300f6 100644 --- a/lib/errorcodes.py +++ b/lib/errorcodes.py @@ -284,10 +284,6 @@ STATEMENT_COMPLETION_UNKNOWN = '40003' DEADLOCK_DETECTED = '40P01' # Class 42 - Syntax Error or Access Rule Violation -UNDEFINED_PSTATEMENT = '26000' -UNDEFINED_CURSOR = '34000' -UNDEFINED_DATABASE = '3D000' -UNDEFINED_SCHEMA = '3F000' SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION = '42000' INSUFFICIENT_PRIVILEGE = '42501' SYNTAX_ERROR = '42601' diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py index d6e25337..122e0d56 100755 --- a/scripts/make_errorcodes.py +++ b/scripts/make_errorcodes.py @@ -72,15 +72,10 @@ def parse_errors_txt(url): m = re.match(r"(.....)\s+(?:E|W|S)\s+ERRCODE_(\S+)(?:\s+(\S+))?$", line) if m: errcode, macro, spec = m.groups() - # error 22008 has 2 macros and 1 def: give priority to the def - # as it's the one we used to parse from sgml + # skip errcodes without specs as they are not publically visible if not spec: - if errcode in errors[class_]: - continue - errlabel = macro.upper() - else: - errlabel = spec.upper() - + continue + errlabel = spec.upper() errors[class_][errcode] = errlabel continue From 37d80f2c0325951d3ee6b07caf7d343d4a97a23d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 31 Aug 2014 02:59:49 +0100 Subject: [PATCH 044/132] Use the readme as PyPI desctiption --- setup.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f22f6b1b..49069f71 100644 --- a/setup.py +++ b/setup.py @@ -560,6 +560,14 @@ download_url = ( "http://initd.org/psycopg/tarballs/PSYCOPG-%s/psycopg2-%s.tar.gz" % ('-'.join(PSYCOPG_VERSION.split('.')[:2]), PSYCOPG_VERSION)) +try: + f = open("README.rst") + readme = f.read() + f.close() +except: + print("failed to read readme: ignoring...") + readme = __doc__ + setup(name="psycopg2", version=PSYCOPG_VERSION, maintainer="Federico Di Gregorio", @@ -570,8 +578,8 @@ setup(name="psycopg2", download_url=download_url, license="LGPL with exceptions or ZPL", platforms=["any"], - description=__doc__.split("\n")[0], - long_description="\n".join(__doc__.split("\n")[2:]), + description=readme.split("\n")[0], + long_description="\n".join(readme.split("\n")[2:]).lstrip(), classifiers=[x for x in classifiers.split("\n") if x], data_files=data_files, package_dir={'psycopg2': 'lib', 'psycopg2.tests': 'tests'}, From 7d81c488494686f3a9cc776bcce3e067ba037f20 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 22 Aug 2014 06:23:54 +0100 Subject: [PATCH 045/132] Point tickets to GitHub instead of Lighthouse Keep into account the numbers reshuffling: Lighthouse bugs with a number matching a GitHub merge request are shifted ahead. --- doc/src/conf.py | 4 +++- doc/src/tools/lib/ticket_role.py | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/src/conf.py b/doc/src/conf.py index 17a61f58..18b81e07 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -68,7 +68,9 @@ intersphinx_mapping = { } # Pattern to generate links to the bug tracker -ticket_url = 'http://psycopg.lighthouseapp.com/projects/62710/tickets/%s' +ticket_url = 'https://github.com/psycopg/psycopg2/issues/%s' +ticket_remap_until = 25 +ticket_remap_offset = 230 # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/src/tools/lib/ticket_role.py b/doc/src/tools/lib/ticket_role.py index f8ceea17..28900661 100644 --- a/doc/src/tools/lib/ticket_role.py +++ b/doc/src/tools/lib/ticket_role.py @@ -3,7 +3,7 @@ ticket role ~~~~~~~~~~~ - An interpreted text role to link docs to lighthouse issues. + An interpreted text role to link docs to tickets issues. :copyright: Copyright 2013 by Daniele Varrazzo. """ @@ -20,13 +20,24 @@ def ticket_role(name, rawtext, text, lineno, inliner, options={}, content=[]): prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] - url_pattern = inliner.document.settings.env.app.config.ticket_url + cfg = inliner.document.settings.env.app.config + url_pattern = cfg.ticket_url if url_pattern is None: msg = inliner.reporter.warning( "ticket not configured: please configure ticket_url in conf.py") prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] + # Push numbers of the oldel tickets ahead. + # We moved the tickets from a different tracker to GitHub and the + # latter already had a few ticket numbers taken (as merge + # requests). + remap_until = cfg.ticket_remap_until + remap_offset = cfg.ticket_remap_offset + if remap_until and remap_offset: + if num <= remap_until: + num += remap_offset + url = url_pattern % num roles.set_classes(options) node = nodes.reference(rawtext, 'ticket ' + utils.unescape(text), @@ -35,5 +46,7 @@ def ticket_role(name, rawtext, text, lineno, inliner, options={}, content=[]): def setup(app): app.add_config_value('ticket_url', None, 'env') + app.add_config_value('ticket_remap_until', None, 'env') + app.add_config_value('ticket_remap_offset', None, 'env') app.add_role('ticket', ticket_role) From 8e06a5188438b34cf7921f9bcecb6392cce77943 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 22 Aug 2014 06:54:59 +0100 Subject: [PATCH 046/132] Added plural version of the tickets role --- NEWS | 4 +-- doc/src/tools/lib/ticket_role.py | 55 ++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/NEWS b/NEWS index 890da26c..7cb70c82 100644 --- a/NEWS +++ b/NEWS @@ -132,7 +132,7 @@ What's new in psycopg 2.4.6 - 'register_hstore()', 'register_composite()', 'tpc_recover()' work with RealDictConnection and Cursor (:ticket:`#114`). - Fixed broken pool for Zope and connections re-init across ZSQL methods - in the same request (tickets #123, #125, #142). + in the same request (:tickets:`#123, #125, #142`). - connect() raises an exception instead of swallowing keyword arguments when a connection string is specified as well (:ticket:`#131`). - Discard any result produced by 'executemany()' (:ticket:`#133`). @@ -154,7 +154,7 @@ What's new in psycopg 2.4.5 - Error and its subclasses are picklable, useful for multiprocessing interaction (:ticket:`#90`). - Better efficiency and formatting of timezone offset objects thanks - to Menno Smits (tickets #94, #95). + to Menno Smits (:tickets:`#94, #95`). - Fixed 'rownumber' during iteration on cursor subclasses. Regression introduced in 2.4.4 (:ticket:`#100`). - Added support for 'inet' arrays. diff --git a/doc/src/tools/lib/ticket_role.py b/doc/src/tools/lib/ticket_role.py index 28900661..d8ded227 100644 --- a/doc/src/tools/lib/ticket_role.py +++ b/doc/src/tools/lib/ticket_role.py @@ -8,45 +8,52 @@ :copyright: Copyright 2013 by Daniele Varrazzo. """ +import re from docutils import nodes, utils from docutils.parsers.rst import roles def ticket_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - try: - num = int(text.replace('#', '')) - except ValueError: - msg = inliner.reporter.error( - "ticket number must be... a number, got '%s'" % text) - prb = inliner.problematic(rawtext, rawtext, msg) - return [prb], [msg] - cfg = inliner.document.settings.env.app.config - url_pattern = cfg.ticket_url - if url_pattern is None: + if cfg.ticket_url is None: msg = inliner.reporter.warning( "ticket not configured: please configure ticket_url in conf.py") prb = inliner.problematic(rawtext, rawtext, msg) return [prb], [msg] - # Push numbers of the oldel tickets ahead. - # We moved the tickets from a different tracker to GitHub and the - # latter already had a few ticket numbers taken (as merge - # requests). - remap_until = cfg.ticket_remap_until - remap_offset = cfg.ticket_remap_offset - if remap_until and remap_offset: - if num <= remap_until: - num += remap_offset + rv = [nodes.Text(name + ' ')] + tokens = re.findall(r'(#?\d+)|([^\d#]+)', text) + for ticket, noise in tokens: + if ticket: + num = int(ticket.replace('#', '')) + + # Push numbers of the oldel tickets ahead. + # We moved the tickets from a different tracker to GitHub and the + # latter already had a few ticket numbers taken (as merge + # requests). + remap_until = cfg.ticket_remap_until + remap_offset = cfg.ticket_remap_offset + if remap_until and remap_offset: + if num <= remap_until: + num += remap_offset + + url = cfg.ticket_url % num + roles.set_classes(options) + node = nodes.reference(ticket, utils.unescape(ticket), + refuri=url, **options) + + rv.append(node) + + else: + assert noise + rv.append(nodes.Text(noise)) + + return rv, [] - url = url_pattern % num - roles.set_classes(options) - node = nodes.reference(rawtext, 'ticket ' + utils.unescape(text), - refuri=url, **options) - return [node], [] def setup(app): app.add_config_value('ticket_url', None, 'env') app.add_config_value('ticket_remap_until', None, 'env') app.add_config_value('ticket_remap_offset', None, 'env') app.add_role('ticket', ticket_role) + app.add_role('tickets', ticket_role) From 6210af27635fbd6156d4f76aebf757702a9a818c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 30 Aug 2014 14:30:20 +0100 Subject: [PATCH 047/132] Fixed excessively strict notices test Failing with PG 9.4 because it generates other debug messages during the test run (rehashing catalog cache). --- tests/test_connection.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 65074048..340693e2 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -127,9 +127,6 @@ class ConnectionTests(ConnectingTestCase): cur.execute(sql) self.assertEqual(50, len(conn.notices)) - self.assert_('table50' in conn.notices[0], conn.notices[0]) - self.assert_('table51' in conn.notices[1], conn.notices[1]) - self.assert_('table98' in conn.notices[-2], conn.notices[-2]) self.assert_('table99' in conn.notices[-1], conn.notices[-1]) def test_server_version(self): From 9fdfa86200580fe9cd45689108d40981db93b618 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 30 Aug 2014 18:23:24 +0100 Subject: [PATCH 048/132] Release notes cleanup before releasing 2.4.5 --- NEWS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index 7cb70c82..776e2b1d 100644 --- a/NEWS +++ b/NEWS @@ -13,18 +13,18 @@ Bug fixes: What's new in psycopg 2.5.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Added :sql:`jsonb` support for PostgreSQL 9.4. -- Fixed segfault if COPY statements are executed instead of using the - proper methods (:ticket:`#219`). +- Added :sql:`jsonb` support for PostgreSQL 9.4 (:ticket:`#226`). +- Fixed segfault if COPY statements are passed to `~cursor.execute()` instead + of using the proper methods (:ticket:`#219`). - Force conversion of pool arguments to integer to avoid potentially unbounded pools (:ticket:`#220`). - Cursors :sql:`WITH HOLD` don't begin a new transaction upon move/fetch/close (:ticket:`#228`). - Cursors :sql:`WITH HOLD` can be used in autocommit (:ticket:`#229`). -- Don't ignore silently the `cursor.callproc` argument without a length. -- Make sure the internal `_psycopg.so` module can be imported stand-alone - (to allow modules juggling such as the one described in :ticket:`#201`). -- Fixed memory leak with large objects (regression introduced in 2.5.3). +- `~cursor.callproc()` doesn't silently ignore an argument without a length. +- Fixed memory leak with large objects (:ticket:`#256`). +- Make sure the internal ``_psycopg.so`` module can be imported stand-alone (to + allow modules juggling such as the one described in :ticket:`#201`). What's new in psycopg 2.5.3 From 7faa06ce0b60148bdc6e9b6df8c9c88f7d99ebb9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 31 Aug 2014 02:44:21 +0100 Subject: [PATCH 049/132] Added scripts to release and upload docs --- .gitignore | 2 ++ scripts/upload-docs.sh | 15 +++++++++++ scripts/upload-release.sh | 55 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100755 scripts/upload-docs.sh create mode 100755 scripts/upload-release.sh diff --git a/.gitignore b/.gitignore index 1b63d531..a017eb3e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,7 @@ build/* doc/src/_build/* doc/html/* doc/psycopg2.txt +scripts/pypi_docs_upload.py env .tox +/rel diff --git a/scripts/upload-docs.sh b/scripts/upload-docs.sh new file mode 100755 index 00000000..c450270c --- /dev/null +++ b/scripts/upload-docs.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DOCDIR="$DIR/../doc" + +# this command requires ssh configured to the proper target +tar czf - -C "$DOCDIR/html" . | ssh psycoweb tar xzvf - -C docs/current + +# download the script to upload the docs to PyPI +test -e "$DIR/pypi_docs_upload.py" \ + || wget -O "$DIR/pypi_docs_upload.py" \ + https://gist.githubusercontent.com/dvarrazzo/dac46237070d69dbc075/raw + +# this command requires a ~/.pypirc with the right privileges +python "$DIR/pypi_docs_upload.py" psycopg2 "$DOCDIR/html" diff --git a/scripts/upload-release.sh b/scripts/upload-release.sh new file mode 100755 index 00000000..7ce16d83 --- /dev/null +++ b/scripts/upload-release.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Script to create a psycopg release +# +# You must create a release tag before running the script, e.g. 2_5_4. +# The script will check out in a clear environment, build the sdist package, +# unpack and test it, then upload on PyPI and on the psycopg website. + +set -e + +REPO_URL=git@github.com:psycopg/psycopg2.git + +VER=$(grep ^PSYCOPG_VERSION setup.py | cut -d "'" -f 2) + +# avoid releasing a testing version +echo "$VER" | grep -qE '^[0-9]+\.[0-9]+(\.[0-9]+)?$' \ + || (echo "bad release: $VER" >&2 && exit 1) + +# Check out in a clean environment +rm -rf rel +mkdir rel +cd rel +git clone $REPO_URL psycopg +cd psycopg +TAG=${VER//./_} +git checkout -b $TAG $TAG +make env # to build the docs +make sdist + +# Test the sdist just created +cd dist +tar xzvf psycopg2-$VER.tar.gz +cd psycopg2-$VER +make +make check +cd ../../ + +read -p "if you are not fine with the above stop me now..." + +# upload to pypi and to the website + +python setup.py sdist --formats=gztar upload -s + +DASHVER=${VER//./-} +DASHVER=${DASHVER:0:3} + +# Requires ssh configuration for 'psycoweb' +scp dist/psycopg2-${VER}.tar.gz psycoweb:tarballs/PSYCOPG-${DASHVER}/ +ssh psycoweb ln -sfv PSYCOPG-${DASHVER}/psycopg2-${VER}.tar.gz \ + tarballs/psycopg2-latest.tar.gz + +scp dist/psycopg2-${VER}.tar.gz.asc psycoweb:tarballs/PSYCOPG-${DASHVER}/ +ssh psycoweb ln -sfv PSYCOPG-${DASHVER}/psycopg2-${VER}.tar.gz.asc \ + tarballs/psycopg2-latest.tar.gz.asc + +echo "great, now write release notes and an email!" From b4d38c455c428a0bd4a89042fc620c6a20c0de91 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 31 Aug 2014 03:08:39 +0100 Subject: [PATCH 050/132] Don't need to build the env for a sdist After 2.5 we are not releasing the html docs in the sdist anymore. --- scripts/upload-release.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/upload-release.sh b/scripts/upload-release.sh index 7ce16d83..618e8232 100755 --- a/scripts/upload-release.sh +++ b/scripts/upload-release.sh @@ -23,7 +23,6 @@ git clone $REPO_URL psycopg cd psycopg TAG=${VER//./_} git checkout -b $TAG $TAG -make env # to build the docs make sdist # Test the sdist just created From 6de7315210ffd4fc06e0a082f4262d794b7216b4 Mon Sep 17 00:00:00 2001 From: Tomer Chachamu Date: Mon, 1 Sep 2014 15:59:13 +0100 Subject: [PATCH 051/132] Fix typo in docs --- doc/src/extensions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index b7aae389..ceedbc9c 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -189,7 +189,7 @@ deal with Python objects adaptation: .. function:: adapt(obj) - Return the SQL representation of *obj* as a string. Raise a + Return the SQL representation of *obj* as an `ISQLQuote`. Raise a `~psycopg2.ProgrammingError` if how to adapt the object is unknown. In order to allow new objects to be adapted, register a new adapter for it using the `register_adapter()` function. @@ -203,7 +203,7 @@ deal with Python objects adaptation: Register a new adapter for the objects of class *class*. *adapter* should be a function taking a single argument (the object - to adapt) and returning an object conforming the `ISQLQuote` + to adapt) and returning an object conforming to the `ISQLQuote` protocol (e.g. exposing a `!getquoted()` method). The `AsIs` is often useful for this task. From 48a32b766b183e9699470d3a766a4c10e1befb91 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Sep 2014 12:08:11 +0100 Subject: [PATCH 052/132] In 2010 I couldn't speak English --- doc/src/usage.rst | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/doc/src/usage.rst b/doc/src/usage.rst index 684a4719..b87887d1 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -145,13 +145,15 @@ query: The problem with the query parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The SQL representation for many data types is often not the same of the Python -string representation. The classic example is with single quotes in -strings: SQL uses them as string constants bounds and requires them to be -escaped, whereas in Python single quotes can be left unescaped in strings -bounded by double quotes. For this reason a naïve approach to the composition -of query strings, e.g. using string concatenation, is a recipe for terrible -problems:: +The SQL representation of many data types is often different from their Python +string representation. The typical example is with single quotes in strings: +in SQL single quotes are used as string literal delimiters, so the ones +appearing inside the string itself must be escaped, whereas in Python single +quotes can be left unescaped if the string is delimited by double quotes. + +Because of the difference, sometime subtle, between the data types +representations, a naïve approach to query strings composition, such as using +Python strings concatenation, is a recipe for *terrible* problems:: >>> SQL = "INSERT INTO authors (name) VALUES ('%s');" # NEVER DO THIS >>> data = ("O'Reilly", ) @@ -160,13 +162,13 @@ problems:: LINE 1: INSERT INTO authors (name) VALUES ('O'Reilly') ^ -If the variable containing the data to be sent to the database comes from an -untrusted source (e.g. a form published on a web site) an attacker could +If the variables containing the data to send to the database come from an +untrusted source (such as a form published on a web site) an attacker could easily craft a malformed string, either gaining access to unauthorized data or performing destructive operations on the database. This form of attack is called `SQL injection`_ and is known to be one of the most widespread forms of -attack to servers. Before continuing, please print `this page`__ as a memo and -hang it onto your desk. +attack to database servers. Before continuing, please print `this page`__ as a +memo and hang it onto your desk. .. _SQL injection: http://en.wikipedia.org/wiki/SQL_injection .. __: http://xkcd.com/327/ From 1b48033345c912ea60ff5fca822bef13c3400067 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 16 Sep 2014 06:46:39 +0100 Subject: [PATCH 053/132] Don't try to close the server cursor in error state `close()` is implicitly called by `__exit__()`, so an exit on error would run a query on a inerr connection, causing another exception hiding the original one. The fix is on `close()`, not on `__exit__()`, because the semantic of the latter is simply to call the former. Closes #262. --- NEWS | 7 +++++++ psycopg/cursor_type.c | 20 +++++++++++++++++--- tests/test_with.py | 12 ++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 776e2b1d..1db0ad80 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,13 @@ Bug fixes: (:ticket:`#191`). +What's new in psycopg 2.5.5 +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Named cursors used as context manager don't swallow the exception on exit + (:ticket:`#262`). + + What's new in psycopg 2.5.4 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 6927c398..fd019812 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -57,10 +57,24 @@ psyco_curs_close(cursorObject *self) if (self->name != NULL) { char buffer[128]; + PGTransactionStatusType status; - EXC_IF_NO_MARK(self); - PyOS_snprintf(buffer, 127, "CLOSE \"%s\"", self->name); - if (pq_execute(self, buffer, 0, 0, 1) == -1) return NULL; + if (self->conn) { + status = PQtransactionStatus(self->conn->pgconn); + } + else { + status = PQTRANS_UNKNOWN; + } + + if (!(status == PQTRANS_UNKNOWN || status == PQTRANS_INERROR)) { + EXC_IF_NO_MARK(self); + PyOS_snprintf(buffer, 127, "CLOSE \"%s\"", self->name); + if (pq_execute(self, buffer, 0, 0, 1) == -1) return NULL; + } + else { + Dprintf("skipping named curs close because tx status %d", + (int)status); + } } self->closed = 1; diff --git a/tests/test_with.py b/tests/test_with.py index d39016c6..13e4d7ef 100755 --- a/tests/test_with.py +++ b/tests/test_with.py @@ -200,6 +200,18 @@ class WithCursorTestCase(WithTestCase): self.assert_(curs.closed) self.assert_(closes) + def test_exception_swallow(self): + # bug #262: __exit__ calls cur.close() that hides the exception + # with another error. + try: + with self.conn as conn: + with conn.cursor('named') as cur: + cur.execute("select 1/0") + except psycopg2.DataError, e: + self.assertEqual(e.pgcode, '22012') + else: + self.fail("where is my exception?") + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From ab4afd0e2f9eaaff3553bf57f39d283adb4cd5b5 Mon Sep 17 00:00:00 2001 From: Asmund Tokheim Date: Sun, 2 Nov 2014 14:15:51 +0100 Subject: [PATCH 054/132] Correction to type adaption example, making it more transparent --- doc/src/advanced.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index dafb1f58..eecbcfda 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -145,7 +145,9 @@ geometric type: ... self.y = y >>> def adapt_point(point): - ... return AsIs("'(%s, %s)'" % (adapt(point.x), adapt(point.y))) + ... x = adapt(point.x).getquoted() + ... y = adapt(point.y).getquoted() + ... return AsIs("'(%s, %s)'" % (x, y)) >>> register_adapter(Point, adapt_point) From 4dbda021458eaf972865ef356f657d57c4eb5d0a Mon Sep 17 00:00:00 2001 From: Hyunjun Kim Date: Thu, 13 Nov 2014 17:32:06 +0900 Subject: [PATCH 055/132] Cast time into timetz when the tzinfo field is set. --- psycopg/adapter_datetime.c | 5 ++++- tests/test_dates.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index b6a46e20..0571837d 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -62,7 +62,10 @@ _pydatetime_string_date_time(pydatetimeObject *self) char *fmt = NULL; switch (self->type) { case PSYCO_DATETIME_TIME: - fmt = "'%s'::time"; + tz = PyObject_GetAttrString(self->wrapped, "tzinfo"); + if (!tz) { goto error; } + fmt = (tz == Py_None) ? "'%s'::time" : "'%s'::timetz"; + Py_DECREF(tz); break; case PSYCO_DATETIME_DATE: fmt = "'%s'::date"; diff --git a/tests/test_dates.py b/tests/test_dates.py index 24c4a9ad..9c4f840a 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -287,7 +287,17 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_time(self): from datetime import time - self._test_type_roundtrip(time(10,20,30)) + tm = self._test_type_roundtrip(time(10,20,30)) + self.assertEqual(None, tm.tzinfo) + + def test_type_roundtrip_timetz(self): + from datetime import time + import psycopg2.tz + tz = psycopg2.tz.FixedOffsetTimezone(8*60) + tm1 = time(10,20,30, tzinfo=tz) + tm2 = self._test_type_roundtrip(tm1) + self.assertNotEqual(None, tm2.tzinfo) + self.assertEqual(tm1, tm2) def test_type_roundtrip_interval(self): from datetime import timedelta From e9f54a2d6e2523c9a585f75748bde2d896ccb296 Mon Sep 17 00:00:00 2001 From: Erik Lee Date: Mon, 17 Nov 2014 16:40:53 +0000 Subject: [PATCH 056/132] Correct exausted -> exhausted typo --- lib/pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pool.py b/lib/pool.py index 7cdd6af2..4f858ab1 100644 --- a/lib/pool.py +++ b/lib/pool.py @@ -86,7 +86,7 @@ class AbstractConnectionPool(object): return conn else: if len(self._used) == self.maxconn: - raise PoolError("connection pool exausted") + raise PoolError("connection pool exhausted") return self._connect(key) def _putconn(self, conn, key=None, close=False): From 6bccb05c90c964374e9b2b0e55c281f3cf7bc4a9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 25 Dec 2014 14:57:31 +0100 Subject: [PATCH 057/132] timetz adaptation documented --- NEWS | 5 +++++ doc/src/usage.rst | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 1db0ad80..c8ff325c 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,11 @@ Current release What's new in psycopg 2.6 ------------------------- +New features: + +- Python `time` objects with a tzinfo specified and PostgreSQL :sql:`timetz` + data are converted into each other (ticket #272). + Bug fixes: - Json apapter's `!str()` returns the adapted content instead of the `!repr()` diff --git a/doc/src/usage.rst b/doc/src/usage.rst index b87887d1..7ffa3baa 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -245,7 +245,8 @@ types: +--------------------+-------------------------+--------------------------+ | `!date` | :sql:`date` | :ref:`adapt-date` | +--------------------+-------------------------+ | - | `!time` | :sql:`time` | | + | `!time` | | :sql:`time` | | + | | | :sql:`timetz` | | +--------------------+-------------------------+ | | `!datetime` | | :sql:`timestamp` | | | | | :sql:`timestamptz` | | @@ -482,7 +483,7 @@ Date/Time objects adaptation Python builtin `~datetime.datetime`, `~datetime.date`, `~datetime.time`, `~datetime.timedelta` are converted into PostgreSQL's -:sql:`timestamp[tz]`, :sql:`date`, :sql:`time`, :sql:`interval` data types. +:sql:`timestamp[tz]`, :sql:`date`, :sql:`time[tz]`, :sql:`interval` data types. Time zones are supported too. The Egenix `mx.DateTime`_ objects are adapted the same way:: From e13ec67da393480e7cec408f94f21a8e9d266bc3 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Fri, 5 Sep 2014 11:55:14 -0400 Subject: [PATCH 058/132] Use lseek64 and ltell64 to support large object greater than 2gb in size. --- psycopg/lobject.h | 4 ++-- psycopg/lobject_int.c | 20 ++++++++++---------- psycopg/lobject_type.c | 14 +++++++------- tests/test_lobject.py | 18 ++++++++++++++++++ 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/psycopg/lobject.h b/psycopg/lobject.h index 56f9ead4..b9c8c3d8 100644 --- a/psycopg/lobject.h +++ b/psycopg/lobject.h @@ -60,8 +60,8 @@ RAISES_NEG HIDDEN int lobject_export(lobjectObject *self, const char *filename); RAISES_NEG HIDDEN Py_ssize_t lobject_read(lobjectObject *self, char *buf, size_t len); RAISES_NEG HIDDEN Py_ssize_t lobject_write(lobjectObject *self, const char *buf, size_t len); -RAISES_NEG HIDDEN int lobject_seek(lobjectObject *self, int pos, int whence); -RAISES_NEG HIDDEN int lobject_tell(lobjectObject *self); +RAISES_NEG HIDDEN long lobject_seek(lobjectObject *self, long pos, int whence); +RAISES_NEG HIDDEN long lobject_tell(lobjectObject *self); RAISES_NEG HIDDEN int lobject_truncate(lobjectObject *self, size_t len); RAISES_NEG HIDDEN int lobject_close(lobjectObject *self); diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 21b1e189..731d3da7 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -376,21 +376,21 @@ lobject_read(lobjectObject *self, char *buf, size_t len) /* lobject_seek - move the current position in the lo */ -RAISES_NEG int -lobject_seek(lobjectObject *self, int pos, int whence) +RAISES_NEG long +lobject_seek(lobjectObject *self, long pos, int whence) { PGresult *pgres = NULL; char *error = NULL; - int where; + long where; - Dprintf("lobject_seek: fd = %d, pos = %d, whence = %d", + Dprintf("lobject_seek: fd = %d, pos = %Ld, whence = %d", self->fd, pos, whence); Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); - where = lo_lseek(self->conn->pgconn, self->fd, pos, whence); - Dprintf("lobject_seek: where = %d", where); + where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence); + Dprintf("lobject_seek: where = %Ld", where); if (where < 0) collect_error(self->conn, &error); @@ -404,20 +404,20 @@ lobject_seek(lobjectObject *self, int pos, int whence) /* lobject_tell - tell the current position in the lo */ -RAISES_NEG int +RAISES_NEG long lobject_tell(lobjectObject *self) { PGresult *pgres = NULL; char *error = NULL; - int where; + long where; Dprintf("lobject_tell: fd = %d", self->fd); Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); - where = lo_tell(self->conn->pgconn, self->fd); - Dprintf("lobject_tell: where = %d", where); + where = lo_tell64(self->conn->pgconn, self->fd); + Dprintf("lobject_tell: where = %Ld", where); if (where < 0) collect_error(self->conn, &error); diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index ef442f90..ba27e4fb 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -121,7 +121,7 @@ static PyObject * psyco_lobj_read(lobjectObject *self, PyObject *args) { PyObject *res; - int where, end; + long where, end; Py_ssize_t size = -1; char *buffer; @@ -165,10 +165,10 @@ psyco_lobj_read(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_seek(lobjectObject *self, PyObject *args) { - int offset, whence=0; - int pos=0; + long offset, pos=0; + int whence=0; - if (!PyArg_ParseTuple(args, "i|i", &offset, &whence)) + if (!PyArg_ParseTuple(args, "l|i", &offset, &whence)) return NULL; EXC_IF_LOBJ_CLOSED(self); @@ -178,7 +178,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) if ((pos = lobject_seek(self, offset, whence)) < 0) return NULL; - return PyInt_FromLong((long)pos); + return PyLong_FromLong(pos); } /* tell method - tell current position in the lobject */ @@ -189,7 +189,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_tell(lobjectObject *self, PyObject *args) { - int pos; + long pos; EXC_IF_LOBJ_CLOSED(self); EXC_IF_LOBJ_LEVEL0(self); @@ -198,7 +198,7 @@ psyco_lobj_tell(lobjectObject *self, PyObject *args) if ((pos = lobject_tell(self)) < 0) return NULL; - return PyInt_FromLong((long)pos); + return PyLong_FromLong(pos); } /* unlink method - unlink (destroy) the lobject */ diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 66a3d8a1..36b24273 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -225,6 +225,24 @@ class LargeObjectTests(LargeObjectTestCase): self.assertEqual(lo.seek(-2, 2), length - 2) self.assertEqual(lo.read(), "ta") + def test_seek_tell_greater_than_2gb(self): + lo = self.conn.lobject() + + # write chunks until its 3gb + length = 0 + for _ in range(24): + # each chunk is written with 128mb + length += lo.write("data" * (1 << 25)) + self.assertEqual(lo.tell(), length) + lo.close() + lo = self.conn.lobject(lo.oid) + + # seek to 3gb - 4, last written text should be data + offset = (1 << 31) + (1 << 30) - 4 # 2gb + 1gb - 4 + self.assertEqual(lo.seek(offset, 0), offset) + self.assertEqual(lo.tell(), offset) + self.assertEqual(lo.read(), "data") + def test_unlink(self): lo = self.conn.lobject() lo.unlink() From cd67d3d2fe085b207268be649ef282fc6032a8cc Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 8 Sep 2014 12:05:28 -0400 Subject: [PATCH 059/132] Modify truncate to use lo_truncate64. Use HAVE_LO64 define to use new lo_*64 methods. Check size of offset and length for versions without LO64. --- psycopg/lobject_int.c | 12 ++++++++ psycopg/lobject_type.c | 20 +++++++++++-- setup.py | 7 +++++ tests/test_lobject.py | 67 ++++++++++++++++++++++++++++++------------ 4 files changed, 86 insertions(+), 20 deletions(-) diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 731d3da7..19d1b9d3 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -389,7 +389,11 @@ lobject_seek(lobjectObject *self, long pos, int whence) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); +#if HAVE_LO64 where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence); +#else + where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); +#endif Dprintf("lobject_seek: where = %Ld", where); if (where < 0) collect_error(self->conn, &error); @@ -416,7 +420,11 @@ lobject_tell(lobjectObject *self) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); +#if HAVE_LO64 where = lo_tell64(self->conn->pgconn, self->fd); +#else + where = (long)lo_tell(self->conn->pgconn, self->fd); +#endif Dprintf("lobject_tell: where = %Ld", where); if (where < 0) collect_error(self->conn, &error); @@ -473,7 +481,11 @@ lobject_truncate(lobjectObject *self, size_t len) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); +#if HAVE_LO64 + retvalue = lo_truncate64(self->conn->pgconn, self->fd, len); +#else retvalue = lo_truncate(self->conn->pgconn, self->fd, len); +#endif Dprintf("lobject_truncate: result = %d", retvalue); if (retvalue < 0) collect_error(self->conn, &error); diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index ba27e4fb..82aa186d 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -175,6 +175,14 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); +#if !HAVE_LO64 + if (offset > INT_MAX) { + psyco_set_error(InterfaceError, NULL, + "offset out of range"); + return NULL; + } +#endif + if ((pos = lobject_seek(self, offset, whence)) < 0) return NULL; @@ -255,15 +263,23 @@ psyco_lobj_get_closed(lobjectObject *self, void *closure) static PyObject * psyco_lobj_truncate(lobjectObject *self, PyObject *args) { - int len = 0; + long len = 0; - if (!PyArg_ParseTuple(args, "|i", &len)) + if (!PyArg_ParseTuple(args, "|l", &len)) return NULL; EXC_IF_LOBJ_CLOSED(self); EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); +#if !HAVE_LO64 + if (len > INT_MAX) { + psyco_set_error(InterfaceError, NULL, + "len out of range"); + return NULL; + } +#endif + if (lobject_truncate(self, len) < 0) return NULL; diff --git a/setup.py b/setup.py index 49069f71..9fdf0b56 100644 --- a/setup.py +++ b/setup.py @@ -415,6 +415,13 @@ class psycopg_build_ext(build_ext): define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % (int(pgmajor), int(pgminor), int(pgpatch)))) + + # enable lo64 if postgres >= 9.3 + if int(pgmajor) >= 9 and int(pgminor) >= 3: + define_macros.append(("HAVE_LO64", "1")) + else: + define_macros.append(("HAVE_LO64", "0")) + except Warning: w = sys.exc_info()[1] # work around py 2/3 different syntax sys.stderr.write("Error: %s\n" % w) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 36b24273..c692fcd0 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -225,24 +225,6 @@ class LargeObjectTests(LargeObjectTestCase): self.assertEqual(lo.seek(-2, 2), length - 2) self.assertEqual(lo.read(), "ta") - def test_seek_tell_greater_than_2gb(self): - lo = self.conn.lobject() - - # write chunks until its 3gb - length = 0 - for _ in range(24): - # each chunk is written with 128mb - length += lo.write("data" * (1 << 25)) - self.assertEqual(lo.tell(), length) - lo.close() - lo = self.conn.lobject(lo.oid) - - # seek to 3gb - 4, last written text should be data - offset = (1 << 31) + (1 << 30) - 4 # 2gb + 1gb - 4 - self.assertEqual(lo.seek(offset, 0), offset) - self.assertEqual(lo.tell(), offset) - self.assertEqual(lo.read(), "data") - def test_unlink(self): lo = self.conn.lobject() lo.unlink() @@ -458,6 +440,55 @@ decorate_all_tests(LargeObjectTruncateTests, skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate) +def skip_if_no_lo64(f): + @wraps(f) + def skip_if_no_lo64_(self): + if self.conn.server_version < 90300: + return self.skipTest("large objects 64bit only supported from PG 9.3") + else: + return f(self) + + return skip_if_no_lo64_ + +class LargeObject64Tests(LargeObjectTestCase): + def test_seek_tell_truncate_greater_than_2gb(self): + lo = self.conn.lobject() + + length = (1 << 31) + (1 << 30) # 2gb + 1gb = 3gb + lo.truncate(length) + + self.assertEqual(lo.seek(length, 0), length) + self.assertEqual(lo.tell(), length) + +decorate_all_tests(LargeObject64Tests, + skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate, skip_if_no_lo64) + + +def skip_if_lo64(f): + @wraps(f) + def skip_if_lo64_(self): + if self.conn.server_version >= 90300: + return self.skipTest("large objects 64bit only supported from PG 9.3") + else: + return f(self) + + return skip_if_lo64_ + +class LargeObjectNot64Tests(LargeObjectTestCase): + def test_seek_larger_than_2gb(self): + lo = self.conn.lobject() + offset = 1 << 32 # 4gb + self.assertRaises(psycopg2.InterfaceError, lo.seek, offset, 0) + + def test_truncate_larger_than_2gb(self): + lo = self.conn.lobject() + length = 1 << 32 # 4gb + self.assertRaises(psycopg2.InterfaceError, lo.truncate, length) + +decorate_all_tests(LargeObjectNot64Tests, + skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate, skip_if_lo64) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From e5bbde955494707f062a2af89bba6e86be5fcae6 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Wed, 10 Sep 2014 16:31:11 -0400 Subject: [PATCH 060/132] Check server_version before using lo_*64 functions. If less tahn 9.3 old none 64bit functions will be use. --- psycopg/lobject_int.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 19d1b9d3..f66d9498 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -390,7 +390,11 @@ lobject_seek(lobjectObject *self, long pos, int whence) pthread_mutex_lock(&(self->conn->lock)); #if HAVE_LO64 - where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence); + if (self->conn->server_version < 90300) { + where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); + } else { + where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence); + } #else where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); #endif @@ -421,7 +425,11 @@ lobject_tell(lobjectObject *self) pthread_mutex_lock(&(self->conn->lock)); #if HAVE_LO64 - where = lo_tell64(self->conn->pgconn, self->fd); + if (self->conn->server_version < 90300) { + where = (long)lo_tell(self->conn->pgconn, self->fd); + } else { + where = lo_tell64(self->conn->pgconn, self->fd); + } #else where = (long)lo_tell(self->conn->pgconn, self->fd); #endif @@ -482,7 +490,11 @@ lobject_truncate(lobjectObject *self, size_t len) pthread_mutex_lock(&(self->conn->lock)); #if HAVE_LO64 - retvalue = lo_truncate64(self->conn->pgconn, self->fd, len); + if (self->conn->server_version < 90300) { + retvalue = lo_truncate(self->conn->pgconn, self->fd, len); + } else { + retvalue = lo_truncate64(self->conn->pgconn, self->fd, len); + } #else retvalue = lo_truncate(self->conn->pgconn, self->fd, len); #endif From 91eabf5fcbf07a72b8b86f7d7da0d309db1af003 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Sep 2014 23:00:42 +0100 Subject: [PATCH 061/132] Fixed check for PG version It would have failed in PostgreSQL 10.0. --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9fdf0b56..9e797cda 100644 --- a/setup.py +++ b/setup.py @@ -407,6 +407,9 @@ class psycopg_build_ext(build_ext): pgmajor, pgminor, pgpatch = m.group(1, 2, 3) if pgpatch is None or not pgpatch.isdigit(): pgpatch = 0 + pgmajor = int(pgmajor) + pgminor = int(pgminor) + pgpatch = int(pgpatch) else: sys.stderr.write( "Error: could not determine PostgreSQL version from '%s'" @@ -414,10 +417,10 @@ class psycopg_build_ext(build_ext): sys.exit(1) define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % - (int(pgmajor), int(pgminor), int(pgpatch)))) + (pgmajor, pgminor, pgpatch))) # enable lo64 if postgres >= 9.3 - if int(pgmajor) >= 9 and int(pgminor) >= 3: + if (pgmajor, pgminor) >= (9, 3): define_macros.append(("HAVE_LO64", "1")) else: define_macros.append(("HAVE_LO64", "0")) From 0205d6ca2e0c358b8e6ab40be942dca3507e77ee Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Sep 2014 23:06:03 +0100 Subject: [PATCH 062/132] Use ifdef instead of if to check LO64 --- psycopg/lobject_int.c | 6 +++--- psycopg/lobject_type.c | 4 ++-- setup.py | 2 -- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index f66d9498..86aa9012 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -389,7 +389,7 @@ lobject_seek(lobjectObject *self, long pos, int whence) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); -#if HAVE_LO64 +#ifdef HAVE_LO64 if (self->conn->server_version < 90300) { where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); } else { @@ -424,7 +424,7 @@ lobject_tell(lobjectObject *self) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); -#if HAVE_LO64 +#ifdef HAVE_LO64 if (self->conn->server_version < 90300) { where = (long)lo_tell(self->conn->pgconn, self->fd); } else { @@ -489,7 +489,7 @@ lobject_truncate(lobjectObject *self, size_t len) Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); -#if HAVE_LO64 +#ifdef HAVE_LO64 if (self->conn->server_version < 90300) { retvalue = lo_truncate(self->conn->pgconn, self->fd, len); } else { diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 82aa186d..a25240c1 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -175,7 +175,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); -#if !HAVE_LO64 +#ifndef HAVE_LO64 if (offset > INT_MAX) { psyco_set_error(InterfaceError, NULL, "offset out of range"); @@ -272,7 +272,7 @@ psyco_lobj_truncate(lobjectObject *self, PyObject *args) EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); -#if !HAVE_LO64 +#ifndef HAVE_LO64 if (len > INT_MAX) { psyco_set_error(InterfaceError, NULL, "len out of range"); diff --git a/setup.py b/setup.py index 9e797cda..647a532f 100644 --- a/setup.py +++ b/setup.py @@ -422,8 +422,6 @@ class psycopg_build_ext(build_ext): # enable lo64 if postgres >= 9.3 if (pgmajor, pgminor) >= (9, 3): define_macros.append(("HAVE_LO64", "1")) - else: - define_macros.append(("HAVE_LO64", "0")) except Warning: w = sys.exc_info()[1] # work around py 2/3 different syntax From b2327b0fc729f4264e45dd26f9ebdbaf5a2b8fda Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Sep 2014 23:16:29 +0100 Subject: [PATCH 063/132] Add the flag 'lo64' to the version if psycopg supports the lo_*64 api --- setup.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/setup.py b/setup.py index 647a532f..22bba5b6 100644 --- a/setup.py +++ b/setup.py @@ -423,6 +423,16 @@ class psycopg_build_ext(build_ext): if (pgmajor, pgminor) >= (9, 3): define_macros.append(("HAVE_LO64", "1")) + # Inject the flag in the version string already packed up + # because we didn't know the version before. + # With distutils everything is complicated. + for i, t in enumerate(define_macros): + if t[0] == 'PSYCOPG_VERSION': + n = t[1].find(')') + if n > 0: + define_macros[i] = ( + t[0], t[1][:n] + ' lo64' + t[1][n:]) + except Warning: w = sys.exc_info()[1] # work around py 2/3 different syntax sys.stderr.write("Error: %s\n" % w) From 44219bf366477f11c365dc1df2334f811b677bdb Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Sep 2014 23:30:34 +0100 Subject: [PATCH 064/132] Don't try and compile lo64 support on 32 bits Python We can't fit more than 31 bits in a long anyway. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 22bba5b6..81b816bd 100644 --- a/setup.py +++ b/setup.py @@ -419,8 +419,8 @@ class psycopg_build_ext(build_ext): define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % (pgmajor, pgminor, pgpatch))) - # enable lo64 if postgres >= 9.3 - if (pgmajor, pgminor) >= (9, 3): + # enable lo64 if libpq >= 9.3 and Python 64 bits + if (pgmajor, pgminor) >= (9, 3) and sys.maxint > (1 << 32): define_macros.append(("HAVE_LO64", "1")) # Inject the flag in the version string already packed up From 79df47a146301207e95ba204c521888c93805092 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 10 Sep 2014 23:49:12 +0100 Subject: [PATCH 065/132] Fixed mismatched types in debug print --- psycopg/lobject_int.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 86aa9012..6b55d42b 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -383,7 +383,7 @@ lobject_seek(lobjectObject *self, long pos, int whence) char *error = NULL; long where; - Dprintf("lobject_seek: fd = %d, pos = %Ld, whence = %d", + Dprintf("lobject_seek: fd = %d, pos = %ld, whence = %d", self->fd, pos, whence); Py_BEGIN_ALLOW_THREADS; @@ -398,7 +398,7 @@ lobject_seek(lobjectObject *self, long pos, int whence) #else where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); #endif - Dprintf("lobject_seek: where = %Ld", where); + Dprintf("lobject_seek: where = %ld", where); if (where < 0) collect_error(self->conn, &error); @@ -433,7 +433,7 @@ lobject_tell(lobjectObject *self) #else where = (long)lo_tell(self->conn->pgconn, self->fd); #endif - Dprintf("lobject_tell: where = %Ld", where); + Dprintf("lobject_tell: where = %ld", where); if (where < 0) collect_error(self->conn, &error); From 2f862972c965192d35ad8a9d2a7543f351828161 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Sep 2014 01:06:57 +0100 Subject: [PATCH 066/132] Guard against overflows when using the lo32 api If psycopg supports lo64 but the server doesn't the user may pass values that would overflow the api range, resulting in: lo.seek((2<<30)) *** OperationalError: ERROR: invalid seek offset: -2147483648 Also improved the error messages and guard against INT_MIN for negative seek offsets. --- psycopg/lobject_type.c | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index a25240c1..ec95b5cf 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -175,10 +175,21 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); -#ifndef HAVE_LO64 - if (offset > INT_MAX) { - psyco_set_error(InterfaceError, NULL, - "offset out of range"); +#ifdef HAVE_LO64 + if ((offset < INT_MIN || offset > INT_MAX) + && self->conn->server_version < 90300) { + PyErr_Format(NotSupportedError, + "offset out of range (%ld): server version %d " + "does not support the lobject 64 API", + offset, self->conn->server_version); + return NULL; + } +#else + if (offset < INT_MIN || offset > INT_MAX) { + PyErr_Format(InterfaceError, + "offset out of range (%ld): this psycopg version was not built " + "with lobject 64 API support", + offset); return NULL; } #endif @@ -272,10 +283,20 @@ psyco_lobj_truncate(lobjectObject *self, PyObject *args) EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); -#ifndef HAVE_LO64 +#ifdef HAVE_LO64 + if (len > INT_MAX && self->conn->server_version < 90300) { + PyErr_Format(NotSupportedError, + "len out of range (%ld): server version %d " + "does not support the lobject 64 API", + len, self->conn->server_version); + return NULL; + } +#else if (len > INT_MAX) { - psyco_set_error(InterfaceError, NULL, - "len out of range"); + PyErr_Format(InterfaceError, + "len out of range (%ld): this psycopg version was not built " + "with lobject 64 API support", + len); return NULL; } #endif From bc5e2aeeadaccfcf798e8849d05b5dd7763d8540 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Sep 2014 01:38:45 +0100 Subject: [PATCH 067/132] Keep into account psycopg build in lo64 tests --- tests/test_lobject.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index c692fcd0..2323113e 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -440,13 +440,23 @@ decorate_all_tests(LargeObjectTruncateTests, skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate) +def _has_lo64(conn): + """Return (bool, msg) about the lo64 support""" + if conn.server_version < 90300: + return (False, "server version %s doesn'ts support the lo64 API" + % conn.server_version) + + if 'lo64' not in psycopg2.__version__: + return (False, "this psycopg build doesn't support the lo64 API") + + return (True, "this server and build support the lo64 API") + def skip_if_no_lo64(f): @wraps(f) def skip_if_no_lo64_(self): - if self.conn.server_version < 90300: - return self.skipTest("large objects 64bit only supported from PG 9.3") - else: - return f(self) + lo64, msg = _has_lo64(self.conn) + if not lo64: return self.skipTest(msg) + else: return f(self) return skip_if_no_lo64_ @@ -467,10 +477,9 @@ decorate_all_tests(LargeObject64Tests, def skip_if_lo64(f): @wraps(f) def skip_if_lo64_(self): - if self.conn.server_version >= 90300: - return self.skipTest("large objects 64bit only supported from PG 9.3") - else: - return f(self) + lo64, msg = _has_lo64(self.conn) + if lo64: return self.skipTest(msg) + else: return f(self) return skip_if_lo64_ @@ -478,12 +487,16 @@ class LargeObjectNot64Tests(LargeObjectTestCase): def test_seek_larger_than_2gb(self): lo = self.conn.lobject() offset = 1 << 32 # 4gb - self.assertRaises(psycopg2.InterfaceError, lo.seek, offset, 0) + self.assertRaises( + (psycopg2.InterfaceError, psycopg2.NotSupportedError), + lo.seek, offset, 0) def test_truncate_larger_than_2gb(self): lo = self.conn.lobject() length = 1 << 32 # 4gb - self.assertRaises(psycopg2.InterfaceError, lo.truncate, length) + self.assertRaises( + (psycopg2.InterfaceError, psycopg2.NotSupportedError), + lo.truncate, length) decorate_all_tests(LargeObjectNot64Tests, skip_if_no_lo, skip_lo_if_green, skip_if_no_truncate, skip_if_lo64) From c008f9d1ce8836157c883aaa7f5bbaa59177b157 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Sep 2014 02:40:05 +0100 Subject: [PATCH 068/132] Added documentation about the lo64 support --- doc/src/extensions.rst | 47 ++++++++++++++++++++++++++++++------------ doc/src/faq.rst | 15 ++++++++++---- doc/src/usage.rst | 12 +++++++++++ 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index ceedbc9c..dea10417 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -40,18 +40,20 @@ functionalities defined by the |DBAPI|_. The class can be subclassed: see the `connection.lobject()` to know how to specify a `!lobject` subclass. - + .. versionadded:: 2.0.8 .. attribute:: oid Database OID of the object. + .. attribute:: mode The mode the database was open. See `connection.lobject()` for a description of the available modes. + .. method:: read(bytes=-1) Read a chunk of data from the current file position. If -1 (default) @@ -64,6 +66,7 @@ functionalities defined by the |DBAPI|_. .. versionchanged:: 2.4 added Unicode support. + .. method:: write(str) Write a string to the large object. Return the number of bytes @@ -73,42 +76,60 @@ functionalities defined by the |DBAPI|_. .. versionchanged:: 2.4 added Unicode support. + .. method:: export(file_name) Export the large object content to the file system. - + The method uses the efficient |lo_export|_ libpq function. - + .. |lo_export| replace:: `!lo_export()` .. _lo_export: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-EXPORT + .. method:: seek(offset, whence=0) Set the lobject current position. + .. versionchanged:: 2.6.0 + added support for *offset* > 2GB. + + .. method:: tell() Return the lobject current position. - .. method:: truncate(len=0) - .. versionadded:: 2.2.0 + .. versionchanged:: 2.6.0 + added support for return value > 2GB. + + + .. method:: truncate(len=0) + Truncate the lobject to the given size. - The method will only be available if Psycopg has been built against libpq - from PostgreSQL 8.3 or later and can only be used with PostgreSQL servers - running these versions. It uses the |lo_truncate|_ libpq function. + The method will only be available if Psycopg has been built against + libpq from PostgreSQL 8.3 or later and can only be used with + PostgreSQL servers running these versions. It uses the |lo_truncate|_ + libpq function. .. |lo_truncate| replace:: `!lo_truncate()` .. _lo_truncate: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-TRUNCATE - .. warning:: + .. versionadded:: 2.2.0 + + .. versionchanged:: 2.6.0 + added support for *len* > 2GB. + + .. warning:: + + If Psycopg is built with |lo_truncate| support or with the 64 bits API + support (resp. from PostgreSQL versions 8.3 and 9.3) but at runtime an + older version of the dynamic library is found, the ``psycopg2`` module + will fail to import. See :ref:`the lo_truncate FAQ ` + about the problem. - If Psycopg is built with |lo_truncate| support (i.e. if the - :program:`pg_config` used during setup is version >= 8.3), but at - runtime an older libpq is found, Psycopg will fail to import. See - :ref:`the lo_truncate FAQ ` about the problem. .. method:: close() diff --git a/doc/src/faq.rst b/doc/src/faq.rst index 0646cdff..8f2f1ecc 100644 --- a/doc/src/faq.rst +++ b/doc/src/faq.rst @@ -248,13 +248,20 @@ I can't compile `!psycopg2`: the compiler says *error: libpq-fe.h: No such file .. cssclass:: faq `!psycopg2` raises `!ImportError` with message *_psycopg.so: undefined symbol: lo_truncate* when imported. - This means that Psycopg has been compiled with |lo_truncate|_ support, - which means that the libpq used at compile time was version >= 8.3, but at - runtime an older libpq library is found. You can use:: + This means that Psycopg was compiled with |lo_truncate|_ support (*i.e.* + the libpq used at compile time was version >= 8.3) but at runtime an older + libpq dynamic library is found. + + Fast-forward several years, if the message reports *undefined symbol: + lo_truncate64* it means that Psycopg was built with large objects 64 bits + API support (*i.e.* the libpq used at compile time was at least 9.3) but + at runtime an older libpq dynamic library is found. + + You can use:: $ ldd /path/to/packages/psycopg2/_psycopg.so | grep libpq - to find what is the version used at runtime. + to find what is the libpq dynamic library used at runtime. You can avoid the problem by using the same version of the :program:`pg_config` at install time and the libpq at runtime. diff --git a/doc/src/usage.rst b/doc/src/usage.rst index b87887d1..5cd4a0d7 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -899,6 +899,18 @@ using the |lo_import|_ and |lo_export|_ libpq functions. .. |lo_export| replace:: `!lo_export()` .. _lo_export: http://www.postgresql.org/docs/current/static/lo-interfaces.html#LO-EXPORT +.. versionchanged:: 2.6 + added support for large objects greated than 2GB. Note that the support is + enabled only if both these conditions are verified: + + - the extension was built against libpq at least 9.3 (you can check if + `psycopg2.__version__` contains the ``lo64`` flag); + - the server version is at least PostgreSQL 9.3 + (`~connection.server_version` must be >= ``90300``). + + If the contitions are not met several `!lobject` methods will fail if the + arguments exceed 2GB. + .. index:: From a338da9c198ae5a4fd1adbd5f6718085a17a6114 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 11 Sep 2014 10:14:22 +0100 Subject: [PATCH 069/132] Fixed typo in tests --- tests/test_lobject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 2323113e..9532770b 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -443,7 +443,7 @@ decorate_all_tests(LargeObjectTruncateTests, def _has_lo64(conn): """Return (bool, msg) about the lo64 support""" if conn.server_version < 90300: - return (False, "server version %s doesn'ts support the lo64 API" + return (False, "server version %s doesn't support the lo64 API" % conn.server_version) if 'lo64' not in psycopg2.__version__: From 713918738141b8806a08a35c88a0505b3e4ac5c0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 25 Dec 2014 15:01:28 +0100 Subject: [PATCH 070/132] Large object 64 bit API quoted in news file --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index 1db0ad80..6e81269e 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,10 @@ Current release What's new in psycopg 2.6 ------------------------- +New features: + +- Added support for large objects larger than 2GB. + Bug fixes: - Json apapter's `!str()` returns the adapted content instead of the `!repr()` From d3c1ad5945b7bed9e9f2696697768d731a7f80c2 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 00:41:50 +0000 Subject: [PATCH 071/132] Convert Postgres time 24:00 into 00:00 Fix ticket #278. --- NEWS | 1 + psycopg/typecast.c | 3 +++ tests/test_dates.py | 12 ++++++++++++ 3 files changed, 16 insertions(+) diff --git a/NEWS b/NEWS index 532540bc..d4ffc36d 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,7 @@ What's new in psycopg 2.5.5 - Named cursors used as context manager don't swallow the exception on exit (:ticket:`#262`). +- PostgreSQL time 24:00 is converted to Python 00:00 (:ticket:`#278`). What's new in psycopg 2.5.4 diff --git a/psycopg/typecast.c b/psycopg/typecast.c index 9678a36b..1cae869f 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -164,6 +164,9 @@ typecast_parse_time(const char* s, const char** t, Py_ssize_t* len, while (usd++ < 6) *us *= 10; } + /* 24:00:00 -> 00:00:00 (ticket #278) */ + if (*hh == 24) { *hh = 0; } + return cz; } diff --git a/tests/test_dates.py b/tests/test_dates.py index 9c4f840a..4ec131ab 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -319,6 +319,18 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): from datetime import timedelta self._test_type_roundtrip_array(timedelta(seconds=30)) + def test_time_24(self): + from datetime import time + + t = self.execute("select '24:00'::time;") + self.assertEqual(t, time(0, 0)) + + t = self.execute("select '24:00+05'::timetz;") + self.assertEqual(t, time(0, 0, tzinfo=FixedOffsetTimezone(300))) + + t = self.execute("select '24:00+05:30'::timetz;") + self.assertEqual(t, time(0, 0, tzinfo=FixedOffsetTimezone(330))) + # Only run the datetime tests if psycopg was compiled with support. if not hasattr(psycopg2.extensions, 'PYDATETIME'): From 7ce7fef322ae74f868b721315d9f26aeb0169d3c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 01:42:21 +0000 Subject: [PATCH 072/132] Propagate read error messages in COPY FROM Fix ticket #270. --- NEWS | 1 + psycopg/pqpath.c | 24 +++++++++++++++++++++--- tests/test_copy.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index d4ffc36d..632a4d48 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,7 @@ What's new in psycopg 2.5.5 - Named cursors used as context manager don't swallow the exception on exit (:ticket:`#262`). +- Propagate read error messages in COPY FROM (:ticket:`#270`). - PostgreSQL time 24:00 is converted to Python 00:00 (:ticket:`#278`). diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index ae6a18bb..c4b5528d 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -1370,9 +1370,27 @@ _pq_copy_in_v3(cursorObject *curs) res = PQputCopyEnd(curs->conn->pgconn, NULL); else if (error == 2) res = PQputCopyEnd(curs->conn->pgconn, "error in PQputCopyData() call"); - else - /* XXX would be nice to propagate the exception */ - res = PQputCopyEnd(curs->conn->pgconn, "error in .read() call"); + else { + char buf[1024]; + strcpy(buf, "error in .read() call"); + if (PyErr_Occurred()) { + PyObject *t, *ex, *tb; + PyErr_Fetch(&t, &ex, &tb); + if (ex) { + PyObject *str; + str = PyObject_Str(ex); + str = psycopg_ensure_bytes(str); + if (str) { + PyOS_snprintf(buf, sizeof(buf), + "error in .read() call: %s %s", + ((PyTypeObject *)t)->tp_name, Bytes_AsString(str)); + Py_DECREF(str); + } + } + PyErr_Restore(t, ex, tb); + } + res = PQputCopyEnd(curs->conn->pgconn, buf); + } CLEARPGRES(curs->pgres); diff --git a/tests/test_copy.py b/tests/test_copy.py index 5b28b20d..32134215 100755 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -340,6 +340,34 @@ conn.close() proc.communicate() self.assertEqual(0, proc.returncode) + def test_copy_from_propagate_error(self): + class BrokenRead(_base): + def read(self, size): + return 1/0 + + def readline(self): + return 1/0 + + curs = self.conn.cursor() + # It seems we cannot do this, but now at least we propagate the error + # self.assertRaises(ZeroDivisionError, + # curs.copy_from, BrokenRead(), "tcopy") + try: + curs.copy_from(BrokenRead(), "tcopy") + except Exception, e: + self.assert_('ZeroDivisionError' in str(e)) + + def test_copy_to_propagate_error(self): + class BrokenWrite(_base): + def write(self, data): + return 1/0 + + curs = self.conn.cursor() + curs.execute("insert into tcopy values (10, 'hi')") + self.assertRaises(ZeroDivisionError, + curs.copy_to, BrokenWrite(), "tcopy") + + decorate_all_tests(CopyTests, skip_copy_if_green) From e490e3bfa33b6a60606cb0a6b445f364dbe09ff3 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 02:04:41 +0000 Subject: [PATCH 073/132] Accept overflow errors testing for LO64 funcs It is raised on 32 bits by PyArg_ParseTuple. We may work around on truncate (maybe parsing a py_ssize_t) but we would have the same problem on seek as the offset is signed. --- tests/test_lobject.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 9532770b..fb2297fa 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -488,14 +488,14 @@ class LargeObjectNot64Tests(LargeObjectTestCase): lo = self.conn.lobject() offset = 1 << 32 # 4gb self.assertRaises( - (psycopg2.InterfaceError, psycopg2.NotSupportedError), + (OverflowError, psycopg2.InterfaceError, psycopg2.NotSupportedError), lo.seek, offset, 0) def test_truncate_larger_than_2gb(self): lo = self.conn.lobject() length = 1 << 32 # 4gb self.assertRaises( - (psycopg2.InterfaceError, psycopg2.NotSupportedError), + (OverflowError, psycopg2.InterfaceError, psycopg2.NotSupportedError), lo.truncate, length) decorate_all_tests(LargeObjectNot64Tests, From f15e9d0cc87094de965da495a337b1a549ca692f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 02:14:00 +0000 Subject: [PATCH 074/132] Fixed link in NEWS entry --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 632a4d48..219a42d3 100644 --- a/NEWS +++ b/NEWS @@ -8,7 +8,7 @@ New features: - Added support for large objects larger than 2GB. - Python `time` objects with a tzinfo specified and PostgreSQL :sql:`timetz` - data are converted into each other (ticket #272). + data are converted into each other (:ticket:`#272`). Bug fixes: From 54ebf90fc697774bf8c981f99091272e6a42ce82 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 02:21:22 +0000 Subject: [PATCH 075/132] Document that LO64 is only available on Py 64 builds --- doc/src/usage.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/src/usage.rst b/doc/src/usage.rst index da3f01fc..de80839c 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -902,15 +902,17 @@ using the |lo_import|_ and |lo_export|_ libpq functions. .. versionchanged:: 2.6 added support for large objects greated than 2GB. Note that the support is - enabled only if both these conditions are verified: + enabled only if all the following conditions are verified: - - the extension was built against libpq at least 9.3 (you can check if - `psycopg2.__version__` contains the ``lo64`` flag); + - the Python build is 64 bits; + - the extension was built against at least libpq 9.3; - the server version is at least PostgreSQL 9.3 (`~connection.server_version` must be >= ``90300``). - If the contitions are not met several `!lobject` methods will fail if the - arguments exceed 2GB. + If Psycopg was built with 64 bits large objects support (i.e. the first + two contidions above are verified), the `psycopg2.__version__` constant + will contain the ``lo64`` flag. If any of the contition is not met + several `!lobject` methods will fail if the arguments exceed 2GB. From 5af5fb4cc6c8a37e3f51b7ec87a8ab82a10c1262 Mon Sep 17 00:00:00 2001 From: Owen Raccuglia Date: Wed, 1 Oct 2014 15:57:47 -0400 Subject: [PATCH 076/132] Allow pickling of cursor.description This is for people using dtuple.py; a dtuple.DatabaseTuple instance keeps a reference to cursor.description, which is not picklable because psycopg2 doesn't export the Column namedtuple it uses. This commit exports the Column namedtuple, and includes a test to verify the pickle/unpickle works after exporting Column. --- lib/__init__.py | 1 + psycopg/psycopgmodule.c | 1 + tests/test_cursor.py | 11 +++++++++++ 3 files changed, 13 insertions(+) diff --git a/lib/__init__.py b/lib/__init__.py index cf8c06ae..a67c0930 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -55,6 +55,7 @@ from psycopg2._psycopg import DateFromTicks, TimeFromTicks, TimestampFromTicks from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, ProgrammingError from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError +from psycopg2._psycopg import Column from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle from psycopg2._psycopg import __version__ diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 745dd6e3..36f734c6 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -880,6 +880,7 @@ INIT_MODULE(_psycopg)(void) PyModule_AddObject(module, "List", (PyObject*)&listType); PyModule_AddObject(module, "QuotedString", (PyObject*)&qstringType); PyModule_AddObject(module, "lobject", (PyObject*)&lobjectType); + PyModule_AddObject(module, "Column", psyco_DescriptionType); /* encodings dictionary in module dictionary */ PyModule_AddObject(module, "encodings", psycoEncodings); diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 07956554..970cc37d 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -23,6 +23,7 @@ # License for more details. import time +import pickle import psycopg2 import psycopg2.extensions from psycopg2.extensions import b @@ -400,6 +401,16 @@ class CursorTests(ConnectingTestCase): self.assertEqual(c.precision, None) self.assertEqual(c.scale, None) + def test_pickle_description(self): + curs = self.conn.cursor() + curs.execute('SELECT 1 AS foo') + description = curs.description + + pickled = pickle.dumps(description, pickle.HIGHEST_PROTOCOL) + unpickled = pickle.loads(pickled) + + self.assertEqual(description, unpickled) + @skip_before_postgres(8, 0) def test_named_cursor_stealing(self): # you can use a named cursor to iterate on a refcursor created From 2a2f306f7b7cd6e74e3c591aaad6a131ef3b0b93 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 02:32:47 +0000 Subject: [PATCH 077/132] Added NEWS note about picklable cursor.desciption --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 219a42d3..7c45fa45 100644 --- a/NEWS +++ b/NEWS @@ -21,6 +21,7 @@ What's new in psycopg 2.5.5 - Named cursors used as context manager don't swallow the exception on exit (:ticket:`#262`). +- `cursor.description` can be pickled (:ticket:`#265`). - Propagate read error messages in COPY FROM (:ticket:`#270`). - PostgreSQL time 24:00 is converted to Python 00:00 (:ticket:`#278`). From 61778238117babcead7bd3aaabdd92a84d69fad8 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 10:30:48 +0000 Subject: [PATCH 078/132] Don't test date 24:00 before PG 8.4 --- tests/test_dates.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_dates.py b/tests/test_dates.py index 4ec131ab..bfa2d1b8 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -25,7 +25,7 @@ import math import psycopg2 from psycopg2.tz import FixedOffsetTimezone, ZERO -from testutils import unittest, ConnectingTestCase +from testutils import unittest, ConnectingTestCase, skip_before_postgres class CommonDatetimeTestsMixin: @@ -319,6 +319,7 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): from datetime import timedelta self._test_type_roundtrip_array(timedelta(seconds=30)) + @skip_before_postgres(8, 4) def test_time_24(self): from datetime import time From 7ea56b112e73081548e35ae27006e08e9189f896 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 11:27:10 +0000 Subject: [PATCH 079/132] Make Column picklable on Python >= 3.3 Also expose the type from the extensions module, not from the main module. --- lib/__init__.py | 1 - lib/extensions.py | 2 +- psycopg/psycopgmodule.c | 22 +++++++++++++++++++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/__init__.py b/lib/__init__.py index a67c0930..cf8c06ae 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -55,7 +55,6 @@ from psycopg2._psycopg import DateFromTicks, TimeFromTicks, TimestampFromTicks from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, ProgrammingError from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError -from psycopg2._psycopg import Column from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle from psycopg2._psycopg import __version__ diff --git a/lib/extensions.py b/lib/extensions.py index 71a92b93..216d8ad2 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -58,7 +58,7 @@ except ImportError: from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid from psycopg2._psycopg import string_types, binary_types, new_type, new_array_type, register_type -from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics +from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics, Column from psycopg2._psycopg import QueryCanceledError, TransactionRollbackError diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 36f734c6..2c345a3b 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -621,8 +621,10 @@ psyco_GetDecimalType(void) static PyObject * psyco_make_description_type(void) { - PyObject *nt = NULL; PyObject *coll = NULL; + PyObject *nt = NULL; + PyTypeObject *t = NULL; + PyObject *s = NULL; PyObject *rv = NULL; /* Try to import collections.namedtuple */ @@ -636,12 +638,26 @@ psyco_make_description_type(void) } /* Build the namedtuple */ - rv = PyObject_CallFunction(nt, "ss", "Column", - "name type_code display_size internal_size precision scale null_ok"); + if(!(t = (PyTypeObject *)PyObject_CallFunction(nt, "ss", "Column", + "name type_code display_size internal_size precision scale null_ok"))) { + goto exit; + } + + /* Export the tuple on the extensions module + * Required to guarantee picklability on Py > 3.3 (see Python issue 21374) + * for previous Py version the module is psycopg2 anyway but for consistency + * we'd rather expose it from the extensions module. */ + if (!(s = Text_FromUTF8("psycopg2.extensions"))) { goto exit; } + if (0 > PyDict_SetItemString(t->tp_dict, "__module__", s)) { goto exit; } + + rv = (PyObject *)t; + t = NULL; exit: Py_XDECREF(coll); Py_XDECREF(nt); + Py_XDECREF((PyObject *)t); + Py_XDECREF(s); return rv; From 296caa4556aa643601b7f63c7b08f97a6494db85 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 12:50:02 +0000 Subject: [PATCH 080/132] More tests tweaks Named cursors on old server versions have a different prefetch behaviour. This has hidden me the supported range of the 24:00 time format. Let's have another go at full testing... --- tests/test_dates.py | 2 +- tests/test_with.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_dates.py b/tests/test_dates.py index bfa2d1b8..d6ce3482 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -319,7 +319,7 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): from datetime import timedelta self._test_type_roundtrip_array(timedelta(seconds=30)) - @skip_before_postgres(8, 4) + @skip_before_postgres(8, 1) def test_time_24(self): from datetime import time diff --git a/tests/test_with.py b/tests/test_with.py index 13e4d7ef..2f018fc8 100755 --- a/tests/test_with.py +++ b/tests/test_with.py @@ -207,6 +207,7 @@ class WithCursorTestCase(WithTestCase): with self.conn as conn: with conn.cursor('named') as cur: cur.execute("select 1/0") + cur.fetchone() except psycopg2.DataError, e: self.assertEqual(e.pgcode, '22012') else: From 569fd0975b84f34228530028c23eb8612ac54bee Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 19:40:52 +0000 Subject: [PATCH 081/132] Python 3.1 bytes.decode() doesn't support keyword arguments Sucker. --- lib/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_json.py b/lib/_json.py index eef1436e..26e32f2f 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -102,7 +102,7 @@ class Json(object): else: def __str__(self): # getquoted is binary in Py3 - return self.getquoted().decode('ascii', errors='replace') + return self.getquoted().decode('ascii', 'replace') def register_json(conn_or_curs=None, globally=False, loads=None, From 6d63973e08cb0d9114d3e1832d89bf2201409a23 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 8 Feb 2015 22:52:50 +0000 Subject: [PATCH 082/132] More portable way to establish Python 32/64 build --- setup.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 81b816bd..ec5bd11a 100644 --- a/setup.py +++ b/setup.py @@ -420,7 +420,7 @@ class psycopg_build_ext(build_ext): (pgmajor, pgminor, pgpatch))) # enable lo64 if libpq >= 9.3 and Python 64 bits - if (pgmajor, pgminor) >= (9, 3) and sys.maxint > (1 << 32): + if (pgmajor, pgminor) >= (9, 3) and is_py_64(): define_macros.append(("HAVE_LO64", "1")) # Inject the flag in the version string already packed up @@ -441,6 +441,13 @@ class psycopg_build_ext(build_ext): if hasattr(self, "finalize_" + sys.platform): getattr(self, "finalize_" + sys.platform)() +def is_py_64(): + # sys.maxint not available since Py 3.1; + # sys.maxsize not available before Py 2.6; + # this is portable at least between Py 2.4 and 3.4. + import struct + return struct.calcsize("P") > 4 + # let's start with macro definitions (the ones not already in setup.cfg) define_macros = [] From 5efe7131ff86ba5adf08ead91393f4910e863f1e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 9 Feb 2015 09:43:55 +0000 Subject: [PATCH 083/132] Version bumped for release 2.6 --- NEWS | 3 ++- setup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 7c45fa45..2cc3c0cf 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,8 @@ What's new in psycopg 2.6 New features: -- Added support for large objects larger than 2GB. +- Added support for large objects larger than 2GB. Many thanks to Blake Rouse + and the MAAS Team for the feature development. - Python `time` objects with a tzinfo specified and PostgreSQL :sql:`timetz` data are converted into each other (:ticket:`#272`). diff --git a/setup.py b/setup.py index ec5bd11a..fcfd91fe 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ except ImportError: # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. -PSYCOPG_VERSION = '2.6.dev0' +PSYCOPG_VERSION = '2.6' version_flags = ['dt', 'dec'] From c62c292053d835bf6c6ba697aac3d3b6da78d6a6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 28 Apr 2015 08:38:41 +0100 Subject: [PATCH 084/132] Bump to next dev version number for the master branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fcfd91fe..fc4f1711 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ except ImportError: # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. -PSYCOPG_VERSION = '2.6' +PSYCOPG_VERSION = '2.7.dev0' version_flags = ['dt', 'dec'] From 1e8be5bd097ca29e779a05c11b3ced3195aaa842 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 23 Feb 2015 10:36:25 +0000 Subject: [PATCH 085/132] Added missing files needed to build the docs Fixes #291 --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 1603b1be..00e4fc32 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ recursive-include lib *.py recursive-include tests *.py recursive-include examples *.py somehackers.jpg whereareyou.jpg recursive-include doc README SUCCESS COPYING.LESSER pep-0249.txt +recursive-include doc Makefile requirements.txt recursive-include doc/src *.rst *.py *.css Makefile recursive-include scripts *.py *.sh include scripts/maketypes.sh scripts/buildtypes.py From 0b523927bcb533fe7450f547c89f5b4a195e9b79 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 28 Apr 2015 09:30:59 +0100 Subject: [PATCH 086/132] Docs build process and docs cleaned up --- Makefile | 4 ++-- doc/Makefile | 17 +++++++++++------ doc/README | 42 ------------------------------------------ doc/README.rst | 26 ++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 50 deletions(-) delete mode 100644 doc/README create mode 100644 doc/README.rst diff --git a/Makefile b/Makefile index 68eaf14a..232f0d0b 100644 --- a/Makefile +++ b/Makefile @@ -102,10 +102,10 @@ MANIFEST: MANIFEST.in $(SOURCE) # docs depend on the build as it partly use introspection. doc/html/genindex.html: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) - PYTHONPATH=:$(BUILD_DIR):$$PYTHONPATH $(MAKE) -C doc html + $(MAKE) -C doc html doc/psycopg2.txt: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) - PYTHONPATH=$(BUILD_DIR):$$PYTHONPATH $(MAKE) -C doc text + $(MAKE) -C doc text doc/docs.zip: doc/html/genindex.html (cd doc/html && zip -r ../docs.zip *) diff --git a/doc/Makefile b/doc/Makefile index 6d639526..4457914c 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -7,22 +7,27 @@ check: doctest # The environment is currently required to build the documentation. # It is not clean by 'make clean' -help: - $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ +PYTHON := python$(PYTHON_VERSION) +PYTHON_VERSION ?= $(shell $(PYTHON) -c 'import sys; print ("%d.%d" % sys.version_info[:2])') + +SPHOPTS=PYTHONPATH=$$(pwd)/../build/lib.$(PYTHON_VERSION)/ SPHINXBUILD=$$(pwd)/env/bin/sphinx-build html: - $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ + $(MAKE) PYTHON=$(PYTHON) -C .. package + $(MAKE) $(SPHOPTS) -C src $@ cp -r src/_build/html . text: - $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ + $(MAKE) PYTHON=$(PYTHON) -C .. package + $(MAKE) $(SPHOPTS) -C src $@ cd src && tools/stitch_text.py index.rst _build/text > ../psycopg2.txt doctest: - $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ + $(MAKE) PYTHON=$(PYTHON) -C .. package + $(MAKE) $(SPHOPTS) -C src $@ clean: - $(MAKE) SPHINXBUILD=$$(pwd)/env/bin/sphinx-build -C src $@ + $(MAKE) $(SPHOPTS) -C src $@ rm -rf html psycopg2.txt env: requirements.txt diff --git a/doc/README b/doc/README deleted file mode 100644 index f74f7d9a..00000000 --- a/doc/README +++ /dev/null @@ -1,42 +0,0 @@ -How to build psycopg documentation ----------------------------------- - -- Install Sphinx, maybe in a virtualenv. Tested with Sphinx 0.6.4:: - - ~$ virtualenv pd - New python executable in pd/bin/python - Installing setuptools............done. - ~$ cd pd - ~/pd$ source bin/activate - (pd)~/pd$ - -- Install Sphinx in the env:: - - (pd)~/pd$ easy_install sphinx - Searching for sphinx - Reading http://pypi.python.org/simple/sphinx/ - Reading http://sphinx.pocoo.org/ - Best match: Sphinx 0.6.4 - ... - Finished processing dependencies for sphinx - -- Build psycopg2 and ensure the package can be imported (it will be used for - reading the version number, autodocs etc.):: - - (pd)~/pd/psycopg2$ python setup.py build - (pd)~/pd/psycopg2$ python setup.py install - running install - ... - creating ~/pd/lib/python2.6/site-packages/psycopg2 - ... - -- Move to the ``doc`` dir and run ``make`` from there:: - - (pd)~/pd/psycopg2$ cd doc/ - (pd)~/pd/psycopg2/doc$ make - Running Sphinx v0.6.4 - ... - -You should have the rendered documentation in ``./html`` and the text file -``psycopg2.txt`` now. - diff --git a/doc/README.rst b/doc/README.rst new file mode 100644 index 00000000..d64794c6 --- /dev/null +++ b/doc/README.rst @@ -0,0 +1,26 @@ +How to build psycopg documentation +---------------------------------- + +Building the documentation usually requires building the library too for +introspection, so you will need the same prerequisites_. The only extra +prerequisite is virtualenv_: the packages needed to build the docs will be +installed when building the env. + +.. _prerequisites: http://initd.org/psycopg/docs/install.html#install-from-source +.. _virtualenv: https://virtualenv.pypa.io/en/latest/ + +Build the env once with:: + + make env + +Then you can build the documentation with:: + + make + +Or the single targets:: + + make html + make text + +You should find the rendered documentation in the ``html`` dir and the text +file ``psycopg2.txt``. From 70fbc8bf1fe6e82b04083f7bf5973a0916f2c59e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 3 May 2015 09:30:04 +0100 Subject: [PATCH 087/132] Fixed connection.poll() docstring Fixes #312 --- psycopg/connection_type.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 1f54ccfe..43abe8a3 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -840,6 +840,10 @@ psyco_conn_get_exception(PyObject *self, void *closure) return exception; } + +#define psyco_conn_poll_doc \ +"poll() -> int -- Advance the connection or query process without blocking." + static PyObject * psyco_conn_poll(connectionObject *self) { @@ -980,7 +984,7 @@ static struct PyMethodDef connectionObject_methods[] = { {"reset", (PyCFunction)psyco_conn_reset, METH_NOARGS, psyco_conn_reset_doc}, {"poll", (PyCFunction)psyco_conn_poll, - METH_NOARGS, psyco_conn_lobject_doc}, + METH_NOARGS, psyco_conn_poll_doc}, {"fileno", (PyCFunction)psyco_conn_fileno, METH_NOARGS, psyco_conn_fileno_doc}, {"isexecuting", (PyCFunction)psyco_conn_isexecuting, From f27ca25d2e1cf2236444ecad9cf6eafb01efb77e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 3 May 2015 09:42:32 +0100 Subject: [PATCH 088/132] Wordsmith on connection.set_session() Fixes #310. --- doc/src/connection.rst | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index c6492632..07e494a2 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -351,17 +351,14 @@ The ``connection`` class .. method:: set_session(isolation_level=None, readonly=None, deferrable=None, autocommit=None) Set one or more parameters for the next transactions or statements in - the current session. See |SET TRANSACTION|_ for further details. - - .. |SET TRANSACTION| replace:: :sql:`SET TRANSACTION` - .. _SET TRANSACTION: http://www.postgresql.org/docs/current/static/sql-set-transaction.html + the current session. :param isolation_level: set the `isolation level`_ for the next - transactions/statements. The value can be one of the - :ref:`constants ` defined in the - `~psycopg2.extensions` module or one of the literal values - ``READ UNCOMMITTED``, ``READ COMMITTED``, ``REPEATABLE READ``, - ``SERIALIZABLE``. + transactions/statements. The value can be one of the literal + values ``READ UNCOMMITTED``, ``READ COMMITTED``, ``REPEATABLE + READ``, ``SERIALIZABLE`` or the equivalent :ref:`constant + ` defined in the `~psycopg2.extensions` + module. :param readonly: if `!True`, set the connection to read only; read/write if `!False`. :param deferrable: if `!True`, set the connection to deferrable; @@ -370,19 +367,14 @@ The ``connection`` class PostgreSQL session setting but an alias for setting the `autocommit` attribute. - Parameter passed as `!None` (the default for all) will not be changed. - The parameters *isolation_level*, *readonly* and *deferrable* also - accept the string ``DEFAULT`` as a value: the effect is to reset the - parameter to the server default. - .. _isolation level: http://www.postgresql.org/docs/current/static/transaction-iso.html - The function must be invoked with no transaction in progress. At every - function invocation, only the specified parameters are changed. - - The default for the values are defined by the server configuration: - see values for |default_transaction_isolation|__, + Arguments set to `!None` (the default for all) will not be changed. + The parameters *isolation_level*, *readonly* and *deferrable* also + accept the string ``DEFAULT`` as a value: the effect is to reset the + parameter to the server default. Defaults are defined by the server + configuration: see values for |default_transaction_isolation|__, |default_transaction_read_only|__, |default_transaction_deferrable|__. .. |default_transaction_isolation| replace:: :sql:`default_transaction_isolation` @@ -392,12 +384,20 @@ The ``connection`` class .. |default_transaction_deferrable| replace:: :sql:`default_transaction_deferrable` .. __: http://www.postgresql.org/docs/current/static/runtime-config-client.html#GUC-DEFAULT-TRANSACTION-DEFERRABLE + The function must be invoked with no transaction in progress. + .. note:: There is currently no builtin method to read the current value for the parameters: use :sql:`SHOW default_transaction_...` to read the values from the backend. + .. seealso:: |SET TRANSACTION|_ for further details about the behaviour + of the transaction parameters in the server. + + .. |SET TRANSACTION| replace:: :sql:`SET TRANSACTION` + .. _SET TRANSACTION: http://www.postgresql.org/docs/current/static/sql-set-transaction.html + .. versionadded:: 2.4.2 From 41639c76100e5f475b59e2e6a0e2685f1bb39948 Mon Sep 17 00:00:00 2001 From: andrew deryabin Date: Thu, 23 Apr 2015 17:00:16 +0300 Subject: [PATCH 089/132] Fixed MinTimeLoggingCursor.callproc() --- lib/extras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/extras.py b/lib/extras.py index a873c4ef..c9f1cbcd 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -434,7 +434,7 @@ class MinTimeLoggingCursor(LoggingCursor): def callproc(self, procname, vars=None): self.timestamp = _time.time() - return LoggingCursor.execute(self, procname, vars) + return LoggingCursor.callproc(self, procname, vars) # a dbtype and adapter for Python UUID type From 2cab752443ba8face8d01ce5032275cfe37003d1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 3 May 2015 09:58:45 +0100 Subject: [PATCH 090/132] Fix to MinTimeLoggingCursor.callproc() noted in NEWSfile --- NEWS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS b/NEWS index 2cc3c0cf..0b71653f 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,12 @@ Current release --------------- +What's new in psycopg 2.6.1 +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`). + + What's new in psycopg 2.6 ------------------------- From 4eee1207f94483a53104da4d7868aa7401b2453e Mon Sep 17 00:00:00 2001 From: Hyunjun Kim Date: Mon, 16 Mar 2015 17:50:20 +0900 Subject: [PATCH 091/132] Fix several typos --- psycopg/cursor_type.c | 2 +- tests/dbapi20.py | 4 ++-- tests/testutils.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index fd019812..cd8d5ca3 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1546,7 +1546,7 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs) if (sql == NULL) { goto exit; } /* This validation of file is rather weak, in that it doesn't enforce the - assocation between "COPY FROM" -> "read" and "COPY TO" -> "write". + association between "COPY FROM" -> "read" and "COPY TO" -> "write". However, the error handling in _pq_copy_[in|out] must be able to handle the case where the attempt to call file.read|write fails, so no harm done. */ diff --git a/tests/dbapi20.py b/tests/dbapi20.py index b8d6a399..f707c090 100644 --- a/tests/dbapi20.py +++ b/tests/dbapi20.py @@ -62,7 +62,7 @@ import sys # - Reversed the polarity of buggy test in test_description # - Test exception hierarchy correctly # - self.populate is now self._populate(), so if a driver stub -# overrides self.ddl1 this change propogates +# overrides self.ddl1 this change propagates # - VARCHAR columns now have a width, which will hopefully make the # DDL even more portible (this will be reversed if it causes more problems) # - cursor.rowcount being checked after various execute and fetchXXX methods @@ -804,7 +804,7 @@ class DatabaseAPI20Test(unittest.TestCase): con.close() def test_setoutputsize(self): - # Real test for setoutputsize is driver dependant + # Real test for setoutputsize is driver dependent raise NotImplementedError('Driver needed to override this test') def test_None(self): diff --git a/tests/testutils.py b/tests/testutils.py index 12732ac6..6a784320 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -65,7 +65,8 @@ else: unittest.TestCase.skipTest = skipTest -# Silence warnings caused by the stubborness of the Python unittest maintainers +# Silence warnings caused by the stubbornness of the Python unittest +# maintainers # http://bugs.python.org/issue9424 if not hasattr(unittest.TestCase, 'assert_') \ or unittest.TestCase.assert_ is not unittest.TestCase.assertTrue: From 1d006ccbe9960ecb2c91d50210229b9afaa5c534 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 3 May 2015 10:22:14 +0100 Subject: [PATCH 092/132] Added note about table names to be escaped in copy_*() methods. --- doc/src/cursor.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/src/cursor.rst b/doc/src/cursor.rst index ebfa90ad..73bb5375 100644 --- a/doc/src/cursor.rst +++ b/doc/src/cursor.rst @@ -524,6 +524,13 @@ The ``cursor`` class >>> cur.fetchall() [(6, 42, 'foo'), (7, 74, 'bar')] + .. note:: the name of the table is not quoted: if the table name + contains uppercase letters or special characters it must be quoted + with double quotes:: + + cur.copy_from(f, '"TABLE"') + + .. versionchanged:: 2.0.6 added the *columns* parameter. @@ -553,6 +560,12 @@ The ``cursor`` class 2|\N|dada ... + .. note:: the name of the table is not quoted: if the table name + contains uppercase letters or special characters it must be quoted + with double quotes:: + + cur.copy_to(f, '"TABLE"') + .. versionchanged:: 2.0.6 added the *columns* parameter. From 4078b895216b9104cd4c045df2d1af9de9e0bd7b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 20 Mar 2013 12:45:37 +0000 Subject: [PATCH 093/132] Added makefile target to upload docs --- doc/Makefile | 6 ++ doc/src/tools/pypi_docs_upload.py | 166 ++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100755 doc/src/tools/pypi_docs_upload.py diff --git a/doc/Makefile b/doc/Makefile index 4457914c..2f5e8e69 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -26,6 +26,12 @@ doctest: $(MAKE) PYTHON=$(PYTHON) -C .. package $(MAKE) $(SPHOPTS) -C src $@ +upload: + # this command requires ssh configured to the proper target + tar czf - -C html . | ssh psycoweb tar xzvf - -C docs/current + # this command requires a .pypirc with the right privileges + python src/tools/pypi_docs_upload.py psycopg2 $$(pwd)/html + clean: $(MAKE) $(SPHOPTS) -C src $@ rm -rf html psycopg2.txt diff --git a/doc/src/tools/pypi_docs_upload.py b/doc/src/tools/pypi_docs_upload.py new file mode 100755 index 00000000..197ec87d --- /dev/null +++ b/doc/src/tools/pypi_docs_upload.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +""" + Standalone script to upload a project docs on PyPI + + Hacked together from the following distutils extension, avaliable from + https://bitbucket.org/jezdez/sphinx-pypi-upload/overview (ver. 0.2.1) + + sphinx_pypi_upload + ~~~~~~~~~~~~~~~~~~ + + setuptools command for uploading Sphinx documentation to PyPI + + :author: Jannis Leidel + :contact: jannis@leidel.info + :copyright: Copyright 2009, Jannis Leidel. + :license: BSD, see LICENSE for details. +""" + +import os +import sys +import socket +import zipfile +import httplib +import base64 +import urlparse +import tempfile +import cStringIO as StringIO +from ConfigParser import ConfigParser + +from distutils import log +from distutils.command.upload import upload +from distutils.errors import DistutilsOptionError + +class UploadDoc(object): + """Distutils command to upload Sphinx documentation.""" + def __init__(self, name, upload_dir, repository=None): + self.name = name + self.upload_dir = upload_dir + + p = ConfigParser() + p.read(os.path.expanduser('~/.pypirc')) + self.username = p.get('pypi', 'username') + self.password = p.get('pypi', 'password') + + self.show_response = False + self.repository = repository or upload.DEFAULT_REPOSITORY + + def create_zipfile(self): + # name = self.distribution.metadata.get_name() + name = self.name + tmp_dir = tempfile.mkdtemp() + tmp_file = os.path.join(tmp_dir, "%s.zip" % name) + zip_file = zipfile.ZipFile(tmp_file, "w") + for root, dirs, files in os.walk(self.upload_dir): + if not files: + raise DistutilsOptionError, \ + "no files found in upload directory '%s'" % self.upload_dir + for name in files: + full = os.path.join(root, name) + relative = root[len(self.upload_dir):].lstrip(os.path.sep) + dest = os.path.join(relative, name) + zip_file.write(full, dest) + zip_file.close() + return tmp_file + + def upload_file(self, filename): + content = open(filename,'rb').read() + # meta = self.distribution.metadata + data = { + ':action': 'doc_upload', + 'name': self.name, # meta.get_name(), + 'content': (os.path.basename(filename),content), + } + # set up the authentication + auth = "Basic " + base64.encodestring(self.username + ":" + self.password).strip() + + # Build up the MIME payload for the POST data + boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' + sep_boundary = '\n--' + boundary + end_boundary = sep_boundary + '--' + body = StringIO.StringIO() + for key, value in data.items(): + # handle multiple entries for the same name + if type(value) != type([]): + value = [value] + for value in value: + if type(value) is tuple: + fn = ';filename="%s"' % value[0] + value = value[1] + else: + fn = "" + value = str(value) + body.write(sep_boundary) + body.write('\nContent-Disposition: form-data; name="%s"'%key) + body.write(fn) + body.write("\n\n") + body.write(value) + if value and value[-1] == '\r': + body.write('\n') # write an extra newline (lurve Macs) + body.write(end_boundary) + body.write("\n") + body = body.getvalue() + + self.announce("Submitting documentation to %s" % (self.repository), log.INFO) + + # build the Request + # We can't use urllib2 since we need to send the Basic + # auth right with the first request + schema, netloc, url, params, query, fragments = \ + urlparse.urlparse(self.repository) + assert not params and not query and not fragments + if schema == 'http': + http = httplib.HTTPConnection(netloc) + elif schema == 'https': + http = httplib.HTTPSConnection(netloc) + else: + raise AssertionError, "unsupported schema "+schema + + data = '' + loglevel = log.INFO + try: + http.connect() + http.putrequest("POST", url) + http.putheader('Content-type', + 'multipart/form-data; boundary=%s'%boundary) + http.putheader('Content-length', str(len(body))) + http.putheader('Authorization', auth) + http.endheaders() + http.send(body) + except socket.error, e: + self.announce(str(e), log.ERROR) + return + + response = http.getresponse() + if response.status == 200: + self.announce('Server response (%s): %s' % (response.status, response.reason), + log.INFO) + elif response.status == 301: + location = response.getheader('Location') + if location is None: + location = 'http://packages.python.org/%s/' % self.name # meta.get_name() + self.announce('Upload successful. Visit %s' % location, + log.INFO) + else: + self.announce('Upload failed (%s): %s' % (response.status, response.reason), + log.ERROR) + if self.show_response: + print '-'*75, response.read(), '-'*75 + + def run(self): + zip_file = self.create_zipfile() + self.upload_file(zip_file) + os.remove(zip_file) + + def announce(self, msg, *args, **kwargs): + print msg + +if __name__ == '__main__': + if len(sys.argv) != 3: + print >>sys.stderr, "usage: %s PROJECT UPLOAD_DIR" % sys.argv[0] + sys.exit(2) + + project, upload_dir = sys.argv[1:] + up = UploadDoc(project, upload_dir=upload_dir) + up.run() + From ad3e91a56f7436362ba9bb1b2e790cdfa2eef123 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 3 May 2015 11:44:10 +0100 Subject: [PATCH 094/132] Unlock the connection after PQflush error Apparently this has never happened... anyway the code path was wrong. Fixes #294. --- NEWS | 1 + psycopg/pqpath.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/NEWS b/NEWS index 0b71653f..86751c4f 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Current release What's new in psycopg 2.6.1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Correctly unlock the connection after error in flush (:ticket:`#294`). - Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`). diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index c4b5528d..5e1974be 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -980,6 +980,10 @@ pq_execute(cursorObject *curs, const char *query, int async, int no_result, int } else { /* there was an error */ + pthread_mutex_unlock(&(curs->conn->lock)); + Py_BLOCK_THREADS; + PyErr_SetString(OperationalError, + PQerrorMessage(curs->conn->pgconn)); return -1; } } From 1b7e3c6da4017fb40eff5d2d59531d7254e13fd0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 3 May 2015 11:59:06 +0100 Subject: [PATCH 095/132] Improve docs about connection's with only closing the transaction --- doc/src/usage.rst | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/src/usage.rst b/doc/src/usage.rst index de80839c..e83b1280 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -705,13 +705,28 @@ managers* and can be used with the ``with`` statement:: When a connection exits the ``with`` block, if no exception has been raised by the block, the transaction is committed. In case of exception the transaction -is rolled back. In no case the connection is closed: a connection can be used -in more than a ``with`` statement and each ``with`` block is effectively -wrapped in a transaction. +is rolled back. When a cursor exits the ``with`` block it is closed, releasing any resource eventually associated with it. The state of the transaction is not affected. +Note that, unlike file objects or other resources, exiting the connection's +``with`` block *doesn't close the connection* but only the transaction +associated with it: a connection can be used in more than a ``with`` statement +and each ``with`` block is effectively wrapped in a separate transaction:: + + conn = psycopg2.connect(DSN) + + with conn: + with conn.cursor() as curs: + curs.execute(SQL1) + + with conn: + with conn.cursor() as curs: + curs.execute(SQL2) + + conn.close() + .. index:: From a59704cf93e8594dfe59cf12d416e82a816953a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Urba=C5=84ski?= Date: Tue, 17 Feb 2015 10:48:18 +0100 Subject: [PATCH 096/132] Make sure libcrypto threadsafety callbacks are properly set up Multithreaded programs using libcrypto (part of OpenSSL) need to set up callbacks to ensure safe execution. Both Python and libpq set up those callbacks, which might lead to a conflict. To avoid leaving dangling function pointers when being unloaded, libpq sets up and removes the callbacks every time a SSL connection it opened and closed. If another Python thread is performing unrelated SSL operations (like connecting to a HTTPS server), this might lead to deadlocks, as described in http://www.postgresql.org/message-id/871tlzrlkq.fsf@wulczer.org Even if the problem will be remediated in libpq, it's still useful to have it fixed in psycopg2. The solution is to use Python's own libcrypto callbacks and completely disable handling them in libpq. --- psycopg/psycopgmodule.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 2c345a3b..61e2de57 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -176,6 +176,26 @@ psyco_register_type(PyObject *self, PyObject *args) } + +/* Make sure libcrypto thread callbacks are set up. */ +static void +psyco_libcrypto_threads_init(void) +{ + /* importing the ssl module sets up Python's libcrypto callbacks */ + if (PyImport_ImportModule("ssl") != NULL) { + /* disable libcrypto setup in libpq, so it won't stomp on the callbacks + that have already been set up */ +#if PG_VERSION_HEX >= 0x080400 + PQinitOpenSSL(1, 0); +#endif + } + else { + /* might mean that Python has been compiled without OpenSSL support, + fall back to relying on libpq's libcrypto locking */ + PyErr_Clear(); + } +} + /* Initialize the default adapters map * * Return 0 on success, else -1 and set an exception. @@ -814,6 +834,9 @@ INIT_MODULE(_psycopg)(void) Py_TYPE(&lobjectType) = &PyType_Type; if (PyType_Ready(&lobjectType) == -1) goto exit; + /* initialize libcrypto threading callbacks */ + psyco_libcrypto_threads_init(); + /* import mx.DateTime module, if necessary */ #ifdef HAVE_MXDATETIME Py_TYPE(&mxdatetimeType) = &PyType_Type; From d66165232e5fa146bc968680d958c38ebe5b2880 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 3 May 2015 12:25:39 +0100 Subject: [PATCH 097/132] OpenSSL deadlock fix noted in NEWSfile --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 86751c4f..3e8864f0 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,7 @@ Current release What's new in psycopg 2.6.1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Fixed deadlock in multithread programs using OpenSSL (:ticket:`#290`). - Correctly unlock the connection after error in flush (:ticket:`#294`). - Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`). From 6c57e4a648cbd5ff81db425978e007f5961e2e77 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 10:16:07 +0200 Subject: [PATCH 098/132] Add parse_dsn module function Calls PQconninfoParse to parse the dsn into a list of keyword and value structs, then constructs a dictionary from that. Can be useful when one needs to alter some part of the the connection string reliably, but doesn't want to get into all the details of parsing a dsn string: quoting, URL format, etc. --- doc/src/module.rst | 12 ++++++++++++ lib/__init__.py | 2 +- psycopg/psycopgmodule.c | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/doc/src/module.rst b/doc/src/module.rst index 8de9f87e..36073a23 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -78,6 +78,7 @@ The module interface respects the standard defined in the |DBAPI|_. .. seealso:: + - `parse_dsn` - libpq `connection string syntax`__ - libpq supported `connection parameters`__ - libpq supported `environment variables`__ @@ -91,6 +92,17 @@ The module interface respects the standard defined in the |DBAPI|_. The parameters *connection_factory* and *async* are Psycopg extensions to the |DBAPI|. +.. function:: parse_dsn(dsn) + + Parse connection string into a dictionary of keywords and values. + + Uses libpq's ``PQconninfoParse`` to parse the string according to + accepted format(s) and check for supported keywords. + + Example:: + + >>> psycopg2.parse_dsn('dbname=test user=postgres password=secret') + {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} .. data:: apilevel diff --git a/lib/__init__.py b/lib/__init__.py index cf8c06ae..27b9d172 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -56,7 +56,7 @@ from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, Programm from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError -from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle +from psycopg2._psycopg import _connect, parse_dsn, apilevel, threadsafety, paramstyle from psycopg2._psycopg import __version__ from psycopg2 import tz diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 61e2de57..d8f893cc 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -112,6 +112,44 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) return conn; } +#define psyco_parse_dsn_doc \ +"parse_dsn(dsn) -- Parse database connection string.\n\n" + +static PyObject * +psyco_parse_dsn(PyObject *self, PyObject *args) +{ + char *dsn, *err; + PQconninfoOption *options = NULL, *o; + PyObject *res = NULL, *value; + + if (!PyArg_ParseTuple(args, "s", &dsn)) { + return NULL; + } + + options = PQconninfoParse(dsn, &err); + if (!options) { + PyErr_Format(PyExc_RuntimeError, "PQconninfoParse: %s: %s", dsn, err); + PQfreemem(err); + return NULL; + } + + res = PyDict_New(); + for (o = options; o->keyword != NULL; o++) { + if (o->val != NULL) { + value = PyString_FromString(o->val); + if (value == NULL || PyDict_SetItemString(res, o->keyword, value) != 0) { + Py_DECREF(res); + res = NULL; + break; + } + } + } + + PQconninfoFree(options); + + return res; +} + /** type registration **/ #define psyco_register_type_doc \ "register_type(obj, conn_or_curs) -> None -- register obj with psycopg type system\n\n" \ @@ -695,6 +733,8 @@ error: static PyMethodDef psycopgMethods[] = { {"_connect", (PyCFunction)psyco_connect, METH_VARARGS|METH_KEYWORDS, psyco_connect_doc}, + {"parse_dsn", (PyCFunction)psyco_parse_dsn, + METH_VARARGS, psyco_parse_dsn_doc}, {"adapt", (PyCFunction)psyco_microprotocols_adapt, METH_VARARGS, psyco_microprotocols_adapt_doc}, From 6a2f21aa14eec258178b6a90255ebfe7bbe0acdc Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 15:11:12 +0200 Subject: [PATCH 099/132] Move parse_dsn to extensions, add tests --- doc/src/extensions.rst | 11 +++++++++++ doc/src/module.rst | 14 +------------- lib/__init__.py | 2 +- lib/extensions.py | 2 +- psycopg/psycopgmodule.c | 38 +++++++++++++++++++++++++------------- tests/test_connection.py | 31 +++++++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 28 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index dea10417..3cdd4c4f 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -12,6 +12,17 @@ The module contains a few objects and function extending the minimum set of functionalities defined by the |DBAPI|_. +.. function:: parse_dsn(dsn) + + Parse connection string into a dictionary of keywords and values. + + Uses libpq's ``PQconninfoParse`` to parse the string according to + accepted format(s) and check for supported keywords. + + Example:: + + >>> psycopg2.extensions.parse_dsn('dbname=test user=postgres password=secret') + {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} .. class:: connection(dsn, async=False) diff --git a/doc/src/module.rst b/doc/src/module.rst index 36073a23..ad19aa7b 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -78,7 +78,7 @@ The module interface respects the standard defined in the |DBAPI|_. .. seealso:: - - `parse_dsn` + - `~psycopg2.extensions.parse_dsn` - libpq `connection string syntax`__ - libpq supported `connection parameters`__ - libpq supported `environment variables`__ @@ -92,18 +92,6 @@ The module interface respects the standard defined in the |DBAPI|_. The parameters *connection_factory* and *async* are Psycopg extensions to the |DBAPI|. -.. function:: parse_dsn(dsn) - - Parse connection string into a dictionary of keywords and values. - - Uses libpq's ``PQconninfoParse`` to parse the string according to - accepted format(s) and check for supported keywords. - - Example:: - - >>> psycopg2.parse_dsn('dbname=test user=postgres password=secret') - {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} - .. data:: apilevel String constant stating the supported DB API level. For `psycopg2` is diff --git a/lib/__init__.py b/lib/__init__.py index 27b9d172..cf8c06ae 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -56,7 +56,7 @@ from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, Programm from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError -from psycopg2._psycopg import _connect, parse_dsn, apilevel, threadsafety, paramstyle +from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle from psycopg2._psycopg import __version__ from psycopg2 import tz diff --git a/lib/extensions.py b/lib/extensions.py index 216d8ad2..f951c519 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -56,7 +56,7 @@ try: except ImportError: pass -from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid +from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid, parse_dsn from psycopg2._psycopg import string_types, binary_types, new_type, new_array_type, register_type from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics, Column diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index d8f893cc..f36fbf42 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -112,13 +112,12 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) return conn; } -#define psyco_parse_dsn_doc \ -"parse_dsn(dsn) -- Parse database connection string.\n\n" +#define psyco_parse_dsn_doc "parse_dsn(dsn) -> dict" static PyObject * psyco_parse_dsn(PyObject *self, PyObject *args) { - char *dsn, *err; + char *dsn, *err = NULL; PQconninfoOption *options = NULL, *o; PyObject *res = NULL, *value; @@ -127,20 +126,33 @@ psyco_parse_dsn(PyObject *self, PyObject *args) } options = PQconninfoParse(dsn, &err); - if (!options) { - PyErr_Format(PyExc_RuntimeError, "PQconninfoParse: %s: %s", dsn, err); - PQfreemem(err); + if (options == NULL) { + if (err != NULL) { + PyErr_Format(ProgrammingError, "error parsing the dsn: %s", err); + PQfreemem(err); + } else { + PyErr_SetString(OperationalError, "PQconninfoParse() failed"); + } return NULL; } res = PyDict_New(); - for (o = options; o->keyword != NULL; o++) { - if (o->val != NULL) { - value = PyString_FromString(o->val); - if (value == NULL || PyDict_SetItemString(res, o->keyword, value) != 0) { - Py_DECREF(res); - res = NULL; - break; + if (res != NULL) { + for (o = options; o->keyword != NULL; o++) { + if (o->val != NULL) { + value = Text_FromUTF8(o->val); + if (value == NULL) { + Py_DECREF(res); + res = NULL; + break; + } + if (PyDict_SetItemString(res, o->keyword, value) != 0) { + Py_DECREF(value); + Py_DECREF(res); + res = NULL; + break; + } + Py_DECREF(value); } } } diff --git a/tests/test_connection.py b/tests/test_connection.py index 340693e2..eb80d4c4 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -270,6 +270,37 @@ class ConnectionTests(ConnectingTestCase): self.assert_(c.closed, "connection failed so it must be closed") self.assert_('foobar' not in c.dsn, "password was not obscured") + def test_parse_dsn(self): + from psycopg2 import ProgrammingError + from psycopg2.extensions import parse_dsn + + self.assertEqual(parse_dsn('dbname=test user=tester password=secret'), + dict(user='tester', password='secret', dbname='test'), + "simple DSN parsed") + + self.assertEqual(parse_dsn("dbname='test 2' user=tester password=secret"), + dict(user='tester', password='secret', dbname='test 2'), + "DSN with quoting parsed") + + self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), + dict(user='tester', password='secret', dbname='test'), + "simple URI dsn parsed") + + # Can't really use assertRaisesRegexp() here since we need to + # make sure that secret is *not* exposed in the error messgage + # (and it also requires python >= 2.7). + raised = False + try: + # unterminated quote after dbname: + parse_dsn("dbname='test 2 user=tester password=secret") + except ProgrammingError, e: + raised = True + self.assertTrue(e.message.find('secret') < 0, + "DSN was not exposed in error message") + except e: + self.fail("unexpected error condition: " + repr(e)) + self.assertTrue(raised, "ProgrammingError raised due to invalid DSN") + class IsolationLevelsTestCase(ConnectingTestCase): From 3200cd77bf07c5de3375dece678eb58f714956b9 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 15:18:03 +0200 Subject: [PATCH 100/132] One more parse_dsn test for unquoted space --- tests/test_connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_connection.py b/tests/test_connection.py index eb80d4c4..a5fabf9f 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -278,6 +278,9 @@ class ConnectionTests(ConnectingTestCase): dict(user='tester', password='secret', dbname='test'), "simple DSN parsed") + self.assertRaises(ProgrammingError, parse_dsn, + "dbname=test 2 user=tester password=secret") + self.assertEqual(parse_dsn("dbname='test 2' user=tester password=secret"), dict(user='tester', password='secret', dbname='test 2'), "DSN with quoting parsed") From 4bb6f9cef2ac2631b2af881d945b770b64fa23bd Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 1 Jun 2015 18:05:11 +0200 Subject: [PATCH 101/132] Add libpq version discovery --- doc/src/extensions.rst | 10 ++++++++++ doc/src/module.rst | 6 ++++++ lib/__init__.py | 2 +- lib/extensions.py | 2 +- psycopg/psycopgmodule.c | 16 ++++++++++++++++ setup.py | 3 +++ tests/testutils.py | 37 +++++++++++++++++++++++++++++++++++++ 7 files changed, 74 insertions(+), 2 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index dea10417..3f010665 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -197,6 +197,16 @@ functionalities defined by the |DBAPI|_. .. versionadded:: 2.2.0 +.. function:: libpq_version() + + Query actual ``libpq`` version loaded. + + This function throws `NotSupportedError` if it was compiled with + ``libpq < 9.1``. + + .. seealso:: libpq docs for `PQlibVersion()`__ + + .. __: http://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQLIBVERSION .. _sql-adaptation-objects: diff --git a/doc/src/module.rst b/doc/src/module.rst index 8de9f87e..bd6bcf45 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -109,6 +109,12 @@ The module interface respects the standard defined in the |DBAPI|_. by the interface. For `psycopg2` is ``pyformat``. See also :ref:`query-parameters`. +.. data:: __libpq_version__ + + Integer contant containing the version of ``libpq`` this `psycopg2` + module was compiled with. If this value is ``>= 90100`` then you + may query for the actually loaded version of libpq using + `~psycopg2.extensions.libpq_version()`. .. index:: diff --git a/lib/__init__.py b/lib/__init__.py index cf8c06ae..994b15a8 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -57,7 +57,7 @@ from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError from psycopg2._psycopg import NotSupportedError, OperationalError from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle -from psycopg2._psycopg import __version__ +from psycopg2._psycopg import __version__, __libpq_version__ from psycopg2 import tz diff --git a/lib/extensions.py b/lib/extensions.py index 216d8ad2..c40e3369 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -56,7 +56,7 @@ try: except ImportError: pass -from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid +from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid, libpq_version from psycopg2._psycopg import string_types, binary_types, new_type, new_array_type, register_type from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics, Column diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 61e2de57..63abb03d 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -300,6 +300,19 @@ exit: return rv; } +#define psyco_libpq_version_doc "Query actual libpq version loaded." + +static PyObject* +psyco_libpq_version(PyObject *self) +{ +#if PG_VERSION_HEX >= 0x090100 + return PyInt_FromLong(PQlibVersion()); +#else + PyErr_SetString(NotSupportedError, "version discovery is not supported in libpq < 9.1"); + return NULL; +#endif +} + /* psyco_encodings_fill Fill the module's postgresql<->python encoding table */ @@ -704,6 +717,8 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS|METH_KEYWORDS, typecast_from_python_doc}, {"new_array_type", (PyCFunction)typecast_array_from_python, METH_VARARGS|METH_KEYWORDS, typecast_array_from_python_doc}, + {"libpq_version", (PyCFunction)psyco_libpq_version, + METH_NOARGS, psyco_libpq_version_doc}, {"Date", (PyCFunction)psyco_Date, METH_VARARGS, psyco_Date_doc}, @@ -899,6 +914,7 @@ INIT_MODULE(_psycopg)(void) /* set some module's parameters */ PyModule_AddStringConstant(module, "__version__", PSYCOPG_VERSION); PyModule_AddStringConstant(module, "__doc__", "psycopg PostgreSQL driver"); + PyModule_AddIntConstant(module, "__libpq_version__", PG_VERSION_NUM); PyModule_AddObject(module, "apilevel", Text_FromUTF8(APILEVEL)); PyModule_AddObject(module, "threadsafety", PyInt_FromLong(THREADSAFETY)); PyModule_AddObject(module, "paramstyle", Text_FromUTF8(PARAMSTYLE)); diff --git a/setup.py b/setup.py index fc4f1711..e42a5c1a 100644 --- a/setup.py +++ b/setup.py @@ -416,6 +416,9 @@ class psycopg_build_ext(build_ext): % pgversion) sys.exit(1) + define_macros.append(("PG_VERSION_NUM", "%d%02d%02d" % + (pgmajor, pgminor, pgpatch))) + define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % (pgmajor, pgminor, pgpatch))) diff --git a/tests/testutils.py b/tests/testutils.py index 6a784320..987bd7b6 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -236,6 +236,43 @@ def skip_after_postgres(*ver): return skip_after_postgres__ return skip_after_postgres_ +def libpq_version(): + import psycopg2 + v = psycopg2.__libpq_version__ + if v >= 90100: + v = psycopg2.extensions.libpq_version() + return v + +def skip_before_libpq(*ver): + """Skip a test if libpq we're linked to is older than a certain version.""" + ver = ver + (0,) * (3 - len(ver)) + def skip_before_libpq_(f): + @wraps(f) + def skip_before_libpq__(self): + v = libpq_version() + if v < int("%d%02d%02d" % ver): + return self.skipTest("skipped because libpq %d" % v) + else: + return f(self) + + return skip_before_libpq__ + return skip_before_libpq_ + +def skip_after_libpq(*ver): + """Skip a test if libpq we're linked to is newer than a certain version.""" + ver = ver + (0,) * (3 - len(ver)) + def skip_after_libpq_(f): + @wraps(f) + def skip_after_libpq__(self): + v = libpq_version() + if v >= int("%d%02d%02d" % ver): + return self.skipTest("skipped because libpq %s" % v) + else: + return f(self) + + return skip_after_libpq__ + return skip_after_libpq_ + def skip_before_python(*ver): """Skip a test on Python before a certain version.""" def skip_before_python_(f): From ffd98a82c04642b73d76a4e60f8f58c355ae1126 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 2 Jun 2015 11:12:16 +0200 Subject: [PATCH 102/132] Add test for libpq_version --- tests/test_module.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_module.py b/tests/test_module.py index 608f703d..62b85ee2 100755 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -320,6 +320,15 @@ import _psycopg self.assertEqual(0, proc.returncode) +class TestVersionDiscovery(unittest.TestCase): + def test_libpq_version(self): + self.assertTrue(type(psycopg2.__libpq_version__) is int) + try: + self.assertTrue(type(psycopg2.extensions.libpq_version()) is int) + except NotSupportedError: + self.assertTrue(psycopg2.__libpq_version__ < 90100) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 73d17e3c5e80c66738435cd6733e999cb9c21ca7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 10:54:08 +0100 Subject: [PATCH 103/132] Dropped PG_VERSION_HEX constant At PostgreSQL 10.0 it would have become awkward. --- psycopg/adapter_binary.c | 2 +- psycopg/lobject_int.c | 4 ++-- psycopg/lobject_type.c | 6 +++--- psycopg/psycopgmodule.c | 4 ++-- psycopg/utils.c | 2 +- setup.py | 3 --- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 485dc5a4..597048d2 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -39,7 +39,7 @@ static unsigned char * binary_escape(unsigned char *from, size_t from_length, size_t *to_length, PGconn *conn) { -#if PG_VERSION_HEX >= 0x080104 +#if PG_VERSION_NUM >= 80104 if (conn) return PQescapeByteaConn(conn, from, from_length, to_length); else diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 6b55d42b..8788c100 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -474,7 +474,7 @@ lobject_export(lobjectObject *self, const char *filename) return retvalue; } -#if PG_VERSION_HEX >= 0x080300 +#if PG_VERSION_NUM >= 80300 RAISES_NEG int lobject_truncate(lobjectObject *self, size_t len) @@ -511,4 +511,4 @@ lobject_truncate(lobjectObject *self, size_t len) } -#endif /* PG_VERSION_HEX >= 0x080300 */ +#endif /* PG_VERSION_NUM >= 80300 */ diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index ec95b5cf..a43325d4 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -266,7 +266,7 @@ psyco_lobj_get_closed(lobjectObject *self, void *closure) return closed; } -#if PG_VERSION_HEX >= 0x080300 +#if PG_VERSION_NUM >= 80300 #define psyco_lobj_truncate_doc \ "truncate(len=0) -- Truncate large object to given size." @@ -327,10 +327,10 @@ static struct PyMethodDef lobjectObject_methods[] = { METH_NOARGS, psyco_lobj_unlink_doc}, {"export",(PyCFunction)psyco_lobj_export, METH_VARARGS, psyco_lobj_export_doc}, -#if PG_VERSION_HEX >= 0x080300 +#if PG_VERSION_NUM >= 80300 {"truncate",(PyCFunction)psyco_lobj_truncate, METH_VARARGS, psyco_lobj_truncate_doc}, -#endif /* PG_VERSION_HEX >= 0x080300 */ +#endif /* PG_VERSION_NUM >= 80300 */ {NULL} }; diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 63abb03d..34fc25e8 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -185,7 +185,7 @@ psyco_libcrypto_threads_init(void) if (PyImport_ImportModule("ssl") != NULL) { /* disable libcrypto setup in libpq, so it won't stomp on the callbacks that have already been set up */ -#if PG_VERSION_HEX >= 0x080400 +#if PG_VERSION_NUM >= 80400 PQinitOpenSSL(1, 0); #endif } @@ -305,7 +305,7 @@ exit: static PyObject* psyco_libpq_version(PyObject *self) { -#if PG_VERSION_HEX >= 0x090100 +#if PG_VERSION_NUM >= 90100 return PyInt_FromLong(PQlibVersion()); #else PyErr_SetString(NotSupportedError, "version discovery is not supported in libpq < 9.1"); diff --git a/psycopg/utils.c b/psycopg/utils.c index 6b035cfa..836f6129 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -62,7 +62,7 @@ psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, } { - #if PG_VERSION_HEX >= 0x080104 + #if PG_VERSION_NUM >= 80104 int err; if (conn && conn->pgconn) ql = PQescapeStringConn(conn->pgconn, to+eq+1, from, len, &err); diff --git a/setup.py b/setup.py index e42a5c1a..2de8c5ef 100644 --- a/setup.py +++ b/setup.py @@ -419,9 +419,6 @@ class psycopg_build_ext(build_ext): define_macros.append(("PG_VERSION_NUM", "%d%02d%02d" % (pgmajor, pgminor, pgpatch))) - define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % - (pgmajor, pgminor, pgpatch))) - # enable lo64 if libpq >= 9.3 and Python 64 bits if (pgmajor, pgminor) >= (9, 3) and is_py_64(): define_macros.append(("HAVE_LO64", "1")) From 5a21da43ee240d66d5fdefd3476a5329e6d730a9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 11:01:10 +0100 Subject: [PATCH 104/132] Mention libpq version inspection in news file --- NEWS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NEWS b/NEWS index 3e8864f0..de5ead30 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,17 @@ Current release --------------- +What's new in psycopg 2.7 +------------------------- + +New features: + +- Added `~psycopg2.__libpq_version__` and + `~psycopg2.extensions.libpq_version()` to inspect the version of the + ``libpq`` library the module was compiled/loaded with + (:tickets:`#35, #323`). + + What's new in psycopg 2.6.1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From c2955fb8fc95afd288af22444be8fa11e4132717 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 11:14:22 +0100 Subject: [PATCH 105/132] Version function/constant docs improved --- doc/src/extensions.rst | 10 ++++++---- doc/src/module.rst | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index 3f010665..84e12412 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -199,12 +199,14 @@ functionalities defined by the |DBAPI|_. .. function:: libpq_version() - Query actual ``libpq`` version loaded. + Return the version number of the ``libpq`` dynamic library loaded as an + integer, in the same format of `~connection.server_version`. - This function throws `NotSupportedError` if it was compiled with - ``libpq < 9.1``. + Raise `~psycopg2.NotSupportedError` if the ``psycopg2`` module was + compiled with a ``libpq`` version lesser than 9.1 (which can be detected + by the `~psycopg2.__libpq_version__` constant). - .. seealso:: libpq docs for `PQlibVersion()`__ + .. seealso:: libpq docs for `PQlibVersion()`__. .. __: http://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQLIBVERSION diff --git a/doc/src/module.rst b/doc/src/module.rst index bd6bcf45..bd121e9d 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -111,10 +111,11 @@ The module interface respects the standard defined in the |DBAPI|_. .. data:: __libpq_version__ - Integer contant containing the version of ``libpq`` this `psycopg2` - module was compiled with. If this value is ``>= 90100`` then you - may query for the actually loaded version of libpq using - `~psycopg2.extensions.libpq_version()`. + Integer constant reporting the version of the ``libpq`` library this + ``psycopg2`` module was compiled with (in the same format of + `~connection.server_version`). If this value is lesser than ``90100`` + then you may query the version of the actually loaded library using the + `~psycopg2.extensions.libpq_version()` function. .. index:: From b0058c0cc85a0e9ce88ccacef0bc34a26d8e5d89 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 12:38:59 +0100 Subject: [PATCH 106/132] Fixed adaptation of lists of None Note: lists of lists of None are not supported yet. --- NEWS | 1 + psycopg/adapter_list.c | 16 +++++++++++++++- tests/test_types_basic.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index de5ead30..4aebbff3 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,7 @@ New features: What's new in psycopg 2.6.1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- Lists consisting of only `None` are escaped correctly (:ticket:`#285`). - Fixed deadlock in multithread programs using OpenSSL (:ticket:`#290`). - Correctly unlock the connection after error in flush (:ticket:`#294`). - Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`). diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index e68b1978..dec17b4c 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -39,6 +39,14 @@ list_quote(listObject *self) /* adapt the list by calling adapt() recursively and then wrapping everything into "ARRAY[]" */ PyObject *tmp = NULL, *str = NULL, *joined = NULL, *res = NULL; + + /* list consisting of only NULL don't work with the ARRAY[] construct + * so we use the {NULL,...} syntax. Note however that list of lists where + * some element is a list of only null still fails: for that we should use + * the '{...}' syntax uniformly but we cannot do it in the current + * infrastructure. TODO in psycopg3 */ + int all_nulls = 1; + Py_ssize_t i, len; len = PyList_GET_SIZE(self->wrapped); @@ -60,6 +68,7 @@ list_quote(listObject *self) quoted = microprotocol_getquoted(wrapped, (connectionObject*)self->connection); if (quoted == NULL) goto error; + all_nulls = 0; } /* here we don't loose a refcnt: SET_ITEM does not change the @@ -74,7 +83,12 @@ list_quote(listObject *self) joined = PyObject_CallMethod(str, "join", "(O)", tmp); if (joined == NULL) goto error; - res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined)); + /* PG doesn't like ARRAY[NULL..] */ + if (!all_nulls) { + res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined)); + } else { + res = Bytes_FromFormat("'{%s}'", Bytes_AsString(joined)); + } error: Py_XDECREF(tmp); diff --git a/tests/test_types_basic.py b/tests/test_types_basic.py index 6c4cc970..199dc1b6 100755 --- a/tests/test_types_basic.py +++ b/tests/test_types_basic.py @@ -192,6 +192,40 @@ class TypesBasicTests(ConnectingTestCase): self.assertRaises(psycopg2.DataError, psycopg2.extensions.STRINGARRAY, b(s), curs) + def testArrayOfNulls(self): + curs = self.conn.cursor() + curs.execute(""" + create table na ( + texta text[], + inta int[], + boola boolean[], + + textaa text[][], + intaa int[][], + boolaa boolean[][] + )""") + + curs.execute("insert into na (texta) values (%s)", ([None],)) + curs.execute("insert into na (texta) values (%s)", (['a', None],)) + curs.execute("insert into na (texta) values (%s)", ([None, None],)) + curs.execute("insert into na (inta) values (%s)", ([None],)) + curs.execute("insert into na (inta) values (%s)", ([42, None],)) + curs.execute("insert into na (inta) values (%s)", ([None, None],)) + curs.execute("insert into na (boola) values (%s)", ([None],)) + curs.execute("insert into na (boola) values (%s)", ([True, None],)) + curs.execute("insert into na (boola) values (%s)", ([None, None],)) + + # TODO: array of array of nulls are not supported yet + # curs.execute("insert into na (textaa) values (%s)", ([[None]],)) + curs.execute("insert into na (textaa) values (%s)", ([['a', None]],)) + # curs.execute("insert into na (textaa) values (%s)", ([[None, None]],)) + # curs.execute("insert into na (intaa) values (%s)", ([[None]],)) + curs.execute("insert into na (intaa) values (%s)", ([[42, None]],)) + # curs.execute("insert into na (intaa) values (%s)", ([[None, None]],)) + # curs.execute("insert into na (boolaa) values (%s)", ([[None]],)) + curs.execute("insert into na (boolaa) values (%s)", ([[True, None]],)) + # curs.execute("insert into na (boolaa) values (%s)", ([[None, None]],)) + @testutils.skip_from_python(3) def testTypeRoundtripBuffer(self): o1 = buffer("".join(map(chr, range(256)))) From 5712f3016970948d17df8b790a2146d1c166c032 Mon Sep 17 00:00:00 2001 From: Pete Hollobon Date: Tue, 19 May 2015 14:06:43 +0100 Subject: [PATCH 107/132] Change "non desiderable" to "undesirable" in docs "desiderable" is considered obsolete --- doc/src/connection.rst | 4 ++-- doc/src/usage.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 07e494a2..92178f34 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -419,8 +419,8 @@ The ``connection`` class By default, any query execution, including a simple :sql:`SELECT` will start a transaction: for long-running programs, if no further - action is taken, the session will remain "idle in transaction", a - condition non desiderable for several reasons (locks are held by + action is taken, the session will remain "idle in transaction", an + undesirable condition for several reasons (locks are held by the session, tables bloat...). For long lived scripts, either ensure to terminate a transaction as soon as possible or use an autocommit connection. diff --git a/doc/src/usage.rst b/doc/src/usage.rst index e83b1280..9dd31df2 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -679,7 +679,7 @@ older versions). By default even a simple :sql:`SELECT` will start a transaction: in long-running programs, if no further action is taken, the session will - remain "idle in transaction", a condition non desiderable for several + remain "idle in transaction", an undesirable condition for several reasons (locks are held by the session, tables bloat...). For long lived scripts, either make sure to terminate a transaction as soon as possible or use an autocommit connection. From d6041271bc7fc539d46198b2603cf7fba2f98bd4 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 2 Jun 2015 14:02:29 +0200 Subject: [PATCH 108/132] Separate parse_dsn test on URI, for libpq >= 9.2 --- tests/test_connection.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index a5fabf9f..6a060ab2 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -32,7 +32,7 @@ import psycopg2.errorcodes import psycopg2.extensions from testutils import unittest, decorate_all_tests, skip_if_no_superuser -from testutils import skip_before_postgres, skip_after_postgres +from testutils import skip_before_postgres, skip_after_postgres, skip_before_libpq from testutils import ConnectingTestCase, skip_if_tpc_disabled from testutils import skip_if_windows from testconfig import dsn, dbname @@ -285,10 +285,6 @@ class ConnectionTests(ConnectingTestCase): dict(user='tester', password='secret', dbname='test 2'), "DSN with quoting parsed") - self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), - dict(user='tester', password='secret', dbname='test'), - "simple URI dsn parsed") - # Can't really use assertRaisesRegexp() here since we need to # make sure that secret is *not* exposed in the error messgage # (and it also requires python >= 2.7). @@ -304,6 +300,27 @@ class ConnectionTests(ConnectingTestCase): self.fail("unexpected error condition: " + repr(e)) self.assertTrue(raised, "ProgrammingError raised due to invalid DSN") + @skip_before_libpq(9, 2) + def test_parse_dsn_uri(self): + from psycopg2 import ProgrammingError + from psycopg2.extensions import parse_dsn + + self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), + dict(user='tester', password='secret', dbname='test'), + "valid URI dsn parsed") + + raised = False + try: + # extra '=' after port value + parse_dsn('postgresql://tester:secret@/test?port=1111=x') + except ProgrammingError, e: + raised = True + self.assertTrue(e.message.find('secret') < 0, + "URI was not exposed in error message") + except e: + self.fail("unexpected error condition: " + repr(e)) + self.assertTrue(raised, "ProgrammingError raised due to invalid URI") + class IsolationLevelsTestCase(ConnectingTestCase): From 0a7261268bcfd225cbd66a3b93a730d69a32f047 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 13:11:46 +0100 Subject: [PATCH 109/132] Dropped unused notice_filter connection member --- psycopg/connection.h | 1 - psycopg/connection_type.c | 2 -- 2 files changed, 3 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index 07dfe2e7..c52abc9f 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -106,7 +106,6 @@ struct connectionObject { /* notice processing */ PyObject *notice_list; - PyObject *notice_filter; struct connectionObject_notice *notice_pending; /* notifies */ diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 43abe8a3..9931399b 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -1105,7 +1105,6 @@ connection_clear(connectionObject *self) Py_CLEAR(self->tpc_xid); Py_CLEAR(self->async_cursor); Py_CLEAR(self->notice_list); - Py_CLEAR(self->notice_filter); Py_CLEAR(self->notifies); Py_CLEAR(self->string_types); Py_CLEAR(self->binary_types); @@ -1181,7 +1180,6 @@ connection_traverse(connectionObject *self, visitproc visit, void *arg) Py_VISIT((PyObject *)(self->tpc_xid)); Py_VISIT(self->async_cursor); Py_VISIT(self->notice_list); - Py_VISIT(self->notice_filter); Py_VISIT(self->notifies); Py_VISIT(self->string_types); Py_VISIT(self->binary_types); From b326a277743b80dde7e5263a232db8df99149164 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 14:24:48 +0100 Subject: [PATCH 110/132] Dropped unneeded constness on the notice message That's a strdup result, we 0wn it. --- psycopg/connection.h | 2 +- psycopg/connection_int.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index c52abc9f..d15c9c64 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -71,7 +71,7 @@ extern HIDDEN PyTypeObject connectionType; struct connectionObject_notice { struct connectionObject_notice *next; - const char *message; + char *message; }; /* the typedef is forward-declared in psycopg.h */ diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index aea2841c..8fce908d 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -154,7 +154,7 @@ conn_notice_clean(connectionObject *self) while (notice != NULL) { tmp = notice; notice = notice->next; - free((void*)tmp->message); + free(tmp->message); free(tmp); } From 2ad82b973b86fa71126657aacee89a30d2211894 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 14:25:46 +0100 Subject: [PATCH 111/132] Pending notice list converted into a forward list This allows inserting the elements in order without using list.insert(). --- psycopg/connection.h | 1 + psycopg/connection_int.c | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index d15c9c64..ec107429 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -107,6 +107,7 @@ struct connectionObject { /* notice processing */ PyObject *notice_list; struct connectionObject_notice *notice_pending; + struct connectionObject_notice *last_notice; /* notifies */ PyObject *notifies; diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 8fce908d..40f7e6ca 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -87,13 +87,20 @@ conn_notice_callback(void *args, const char *message) /* Discard the notice in case of failed allocation. */ return; } + notice->next = NULL; notice->message = strdup(message); if (NULL == notice->message) { free(notice); return; } - notice->next = self->notice_pending; - self->notice_pending = notice; + + if (NULL == self->last_notice) { + self->notice_pending = self->last_notice = notice; + } + else { + self->last_notice->next = notice; + self->last_notice = notice; + } } /* Expose the notices received as Python objects. @@ -111,17 +118,14 @@ conn_notice_process(connectionObject *self) } notice = self->notice_pending; - nnotices = PyList_GET_SIZE(self->notice_list); while (notice != NULL) { PyObject *msg; msg = conn_text_from_chars(self, notice->message); Dprintf("conn_notice_process: %s", notice->message); - /* Respect the order in which notices were produced, - because in notice_list they are reversed (see ticket #9) */ if (msg) { - PyList_Insert(self->notice_list, nnotices, msg); + PyList_Append(self->notice_list, msg); Py_DECREF(msg); } else { @@ -158,7 +162,7 @@ conn_notice_clean(connectionObject *self) free(tmp); } - self->notice_pending = NULL; + self->last_notice = self->notice_pending = NULL; } From 1f330e9cac9c5d40c33f4f58d0dbfc0109c62edc Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 17:02:04 +0100 Subject: [PATCH 112/132] Allow connection.notices and notifies to be replaced. Close #326 --- NEWS | 3 ++ doc/src/advanced.rst | 4 +++ doc/src/connection.rst | 18 ++++++++-- psycopg/connection_int.c | 69 +++++++++++++++++++++++++++++---------- psycopg/connection_type.c | 4 +-- tests/test_connection.py | 36 ++++++++++++++++++++ tests/test_notify.py | 22 +++++++++++++ 7 files changed, 134 insertions(+), 22 deletions(-) diff --git a/NEWS b/NEWS index 4aebbff3..fe6cea41 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,9 @@ New features: `~psycopg2.extensions.libpq_version()` to inspect the version of the ``libpq`` library the module was compiled/loaded with (:tickets:`#35, #323`). +- The attributes `~connection.notices` and `~connection.notifies` can be + customized replacing them with any object exposing an `!append()` method + (:ticket:`#326`). What's new in psycopg 2.6.1 diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index eecbcfda..f0483cea 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -312,6 +312,10 @@ received from a previous version server will have the Added `~psycopg2.extensions.Notify` object and handling notification payload. +.. versionchanged:: 2.7 + The `~connection.notifies` attribute is writable: it is possible to + replace it with any object exposing an `!append()` method. An useful + example would be to use a `~collections.deque` object. .. index:: diff --git a/doc/src/connection.rst b/doc/src/connection.rst index 92178f34..cceef1e5 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -483,13 +483,21 @@ The ``connection`` class ['NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "foo_pkey" for table "foo"\n', 'NOTICE: CREATE TABLE will create implicit sequence "foo_id_seq" for serial column "foo.id"\n'] + .. versionchanged:: 2.7 + The `!notices` attribute is writable: the user may replace it + with any Python object exposing an `!append()` method. If + appending raises an exception the notice is silently + dropped. + To avoid a leak in case excessive notices are generated, only the last - 50 messages are kept. + 50 messages are kept. This check is only in place if the `!notices` + attribute is a list: if any other object is used it will be up to the + user to guard from leakage. You can configure what messages to receive using `PostgreSQL logging configuration parameters`__ such as ``log_statement``, ``client_min_messages``, ``log_min_duration_statement`` etc. - + .. __: http://www.postgresql.org/docs/current/static/runtime-config-logging.html @@ -506,6 +514,12 @@ The ``connection`` class the payload was not accessible. To keep backward compatibility, `!Notify` objects can still be accessed as 2 items tuples. + .. versionchanged:: 2.7 + The `!notifies` attribute is writable: the user may replace it + with any Python object exposing an `!append()` method. If + appending raises an exception the notification is silently + dropped. + .. attribute:: cursor_factory diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 40f7e6ca..43d0fdae 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -111,41 +111,60 @@ void conn_notice_process(connectionObject *self) { struct connectionObject_notice *notice; - Py_ssize_t nnotices; + PyObject *msg = NULL; + PyObject *tmp = NULL; + static PyObject *append; if (NULL == self->notice_pending) { return; } - notice = self->notice_pending; + if (!append) { + if (!(append = Text_FromUTF8("append"))) { + goto error; + } + } + notice = self->notice_pending; while (notice != NULL) { - PyObject *msg; - msg = conn_text_from_chars(self, notice->message); Dprintf("conn_notice_process: %s", notice->message); - if (msg) { - PyList_Append(self->notice_list, msg); - Py_DECREF(msg); - } - else { - /* We don't really have a way to report errors, so gulp it. - * The function should only fail for out of memory, so we are - * likely going to die anyway. */ - PyErr_Clear(); + if (!(msg = conn_text_from_chars(self, notice->message))) { goto error; } + + if (!(tmp = PyObject_CallMethodObjArgs( + self->notice_list, append, msg, NULL))) { + + goto error; } + Py_DECREF(tmp); tmp = NULL; + Py_DECREF(msg); msg = NULL; + notice = notice->next; } /* Remove the oldest item if the queue is getting too long. */ - nnotices = PyList_GET_SIZE(self->notice_list); - if (nnotices > CONN_NOTICES_LIMIT) { - PySequence_DelSlice(self->notice_list, - 0, nnotices - CONN_NOTICES_LIMIT); + if (PyList_Check(self->notice_list)) { + Py_ssize_t nnotices; + nnotices = PyList_GET_SIZE(self->notice_list); + if (nnotices > CONN_NOTICES_LIMIT) { + if (-1 == PySequence_DelSlice(self->notice_list, + 0, nnotices - CONN_NOTICES_LIMIT)) { + PyErr_Clear(); + } + } } conn_notice_clean(self); + return; + +error: + Py_XDECREF(tmp); + Py_XDECREF(msg); + conn_notice_clean(self); + + /* TODO: the caller doesn't expects errors from us */ + PyErr_Clear(); } void @@ -177,6 +196,15 @@ conn_notifies_process(connectionObject *self) PGnotify *pgn = NULL; PyObject *notify = NULL; PyObject *pid = NULL, *channel = NULL, *payload = NULL; + PyObject *tmp = NULL; + + static PyObject *append; + + if (!append) { + if (!(append = Text_FromUTF8("append"))) { + goto error; + } + } while ((pgn = PQnotifies(self->pgconn)) != NULL) { @@ -196,7 +224,11 @@ conn_notifies_process(connectionObject *self) Py_DECREF(channel); channel = NULL; Py_DECREF(payload); payload = NULL; - PyList_Append(self->notifies, (PyObject *)notify); + if (!(tmp = PyObject_CallMethodObjArgs( + self->notifies, append, notify, NULL))) { + goto error; + } + Py_DECREF(tmp); tmp = NULL; Py_DECREF(notify); notify = NULL; PQfreemem(pgn); pgn = NULL; @@ -205,6 +237,7 @@ conn_notifies_process(connectionObject *self) error: if (pgn) { PQfreemem(pgn); } + Py_XDECREF(tmp); Py_XDECREF(notify); Py_XDECREF(pid); Py_XDECREF(channel); diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 9931399b..2c1dddf2 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -1001,8 +1001,8 @@ static struct PyMemberDef connectionObject_members[] = { "True if the connection is closed."}, {"encoding", T_STRING, offsetof(connectionObject, encoding), READONLY, "The current client encoding."}, - {"notices", T_OBJECT, offsetof(connectionObject, notice_list), READONLY}, - {"notifies", T_OBJECT, offsetof(connectionObject, notifies), READONLY}, + {"notices", T_OBJECT, offsetof(connectionObject, notice_list), 0}, + {"notifies", T_OBJECT, offsetof(connectionObject, notifies), 0}, {"dsn", T_STRING, offsetof(connectionObject, dsn), READONLY, "The current connection string."}, {"async", T_LONG, offsetof(connectionObject, async), READONLY, diff --git a/tests/test_connection.py b/tests/test_connection.py index 340693e2..fa78eb37 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -129,6 +129,42 @@ class ConnectionTests(ConnectingTestCase): self.assertEqual(50, len(conn.notices)) self.assert_('table99' in conn.notices[-1], conn.notices[-1]) + def test_notices_deque(self): + from collections import deque + + conn = self.conn + self.conn.notices = deque() + cur = conn.cursor() + if self.conn.server_version >= 90300: + cur.execute("set client_min_messages=debug1") + + cur.execute("create temp table table1 (id serial); create temp table table2 (id serial);") + cur.execute("create temp table table3 (id serial); create temp table table4 (id serial);") + self.assertEqual(len(conn.notices), 4) + self.assert_('table1' in conn.notices.popleft()) + self.assert_('table2' in conn.notices.popleft()) + self.assert_('table3' in conn.notices.popleft()) + self.assert_('table4' in conn.notices.popleft()) + self.assertEqual(len(conn.notices), 0) + + # not limited, but no error + for i in range(0, 100, 10): + sql = " ".join(["create temp table table2_%d (id serial);" % j for j in range(i, i+10)]) + cur.execute(sql) + + self.assertEqual(100, len(conn.notices)) + + def test_notices_noappend(self): + conn = self.conn + self.conn.notices = None # will make an error swallowes ok + cur = conn.cursor() + if self.conn.server_version >= 90300: + cur.execute("set client_min_messages=debug1") + + cur.execute("create temp table table1 (id serial);") + + self.assertEqual(self.conn.notices, None) + def test_server_version(self): self.assert_(self.conn.server_version) diff --git a/tests/test_notify.py b/tests/test_notify.py index f8383899..fc6224d7 100755 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -155,6 +155,27 @@ conn.close() self.assertEqual('foo', notify.channel) self.assertEqual('Hello, world!', notify.payload) + def test_notify_deque(self): + from collections import deque + self.autocommit(self.conn) + self.conn.notifies = deque() + self.listen('foo') + self.notify('foo').communicate() + time.sleep(0.5) + self.conn.poll() + notify = self.conn.notifies.popleft() + self.assert_(isinstance(notify, psycopg2.extensions.Notify)) + self.assertEqual(len(self.conn.notifies), 0) + + def test_notify_noappend(self): + self.autocommit(self.conn) + self.conn.notifies = None + self.listen('foo') + self.notify('foo').communicate() + time.sleep(0.5) + self.conn.poll() + self.assertEqual(self.conn.notifies, None) + def test_notify_init(self): n = psycopg2.extensions.Notify(10, 'foo') self.assertEqual(10, n.pid) @@ -192,6 +213,7 @@ conn.close() self.assertNotEqual(hash(Notify(10, 'foo', 'bar')), hash(Notify(10, 'foo'))) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 06b4b1de94a1ae9365b94069f7680f87e5e9b87b Mon Sep 17 00:00:00 2001 From: btubbs Date: Thu, 21 May 2015 01:24:00 -0600 Subject: [PATCH 113/132] Notify example should pop the oldest message in conn.notifies, not the newest. --- doc/src/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index f0483cea..82754ee0 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -291,7 +291,7 @@ something to read:: else: conn.poll() while conn.notifies: - notify = conn.notifies.pop() + notify = conn.notifies.pop(0) print "Got NOTIFY:", notify.pid, notify.channel, notify.payload Running the script and executing a command such as :sql:`NOTIFY test, 'hello'` From 925fdf57311b7168202e83d3d8e6040b986e5691 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 2 Jun 2015 17:11:09 +0100 Subject: [PATCH 114/132] Fixed doc about libpq version availability --- doc/src/module.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/src/module.rst b/doc/src/module.rst index bd121e9d..7f8a29b6 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -113,9 +113,9 @@ The module interface respects the standard defined in the |DBAPI|_. Integer constant reporting the version of the ``libpq`` library this ``psycopg2`` module was compiled with (in the same format of - `~connection.server_version`). If this value is lesser than ``90100`` - then you may query the version of the actually loaded library using the - `~psycopg2.extensions.libpq_version()` function. + `~connection.server_version`). If this value is greater or equal than + ``90100`` then you may query the version of the actually loaded library + using the `~psycopg2.extensions.libpq_version()` function. .. index:: From ac25ba0a3f1ffb797e91e335654e91b8778aff4a Mon Sep 17 00:00:00 2001 From: Photonios Date: Tue, 22 Sep 2015 18:25:53 +0200 Subject: [PATCH 115/132] Fix for MSVC 2015: isnan is supported in this version --- psycopg/config.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psycopg/config.h b/psycopg/config.h index b6cd4190..255fc3a7 100644 --- a/psycopg/config.h +++ b/psycopg/config.h @@ -129,14 +129,15 @@ static int pthread_mutex_init(pthread_mutex_t *mutex, void* fake) /* remove the inline keyword, since it doesn't work unless C++ file */ #define inline -/* Hmmm, MSVC doesn't have a isnan/isinf function, but has _isnan function */ +/* Hmmm, MSVC <2015 doesn't have a isnan/isinf function, but has _isnan function */ #if defined (_MSC_VER) +#if !defined(isnan) #define isnan(x) (_isnan(x)) /* The following line was hacked together from simliar code by Bjorn Reese * in libxml2 code */ #define isinf(x) ((_fpclass(x) == _FPCLASS_PINF) ? 1 \ : ((_fpclass(x) == _FPCLASS_NINF) ? -1 : 0)) - +#endif #define strcasecmp(x, y) lstrcmpi(x, y) #endif #endif From 71925fcc00a3e41c255c244e124b58144cecb6a1 Mon Sep 17 00:00:00 2001 From: Photonios Date: Tue, 22 Sep 2015 18:26:14 +0200 Subject: [PATCH 116/132] Fix for MSVC 2015: round has been added to this version --- psycopg/config.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/psycopg/config.h b/psycopg/config.h index 255fc3a7..f8e17a9a 100644 --- a/psycopg/config.h +++ b/psycopg/config.h @@ -142,16 +142,20 @@ static int pthread_mutex_init(pthread_mutex_t *mutex, void* fake) #endif #endif +/* what's this, we have no round function either? */ #if (defined(__FreeBSD__) && __FreeBSD_version < 503000) \ || (defined(_WIN32) && !defined(__GNUC__)) \ || (defined(sun) || defined(__sun__)) \ && (defined(__SunOS_5_8) || defined(__SunOS_5_9)) -/* what's this, we have no round function either? */ + +/* round has been added in the standard library with MSVC 2015 */ +#if _MSC_VER < 1900 static double round(double num) { return (num >= 0) ? floor(num + 0.5) : ceil(num - 0.5); } #endif +#endif /* resolve missing isinf() function for Solaris */ #if defined (__SVR4) && defined (__sun) From 0e3f5214c5f292212e10b1d87b68069c9ab3ce22 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 30 Sep 2015 12:00:13 +0100 Subject: [PATCH 117/132] Report the server response status on errors with no message Suggested by Craig Ringer in pull request #353, should also give more information for other cases we were reported on flaky servers (AWS, digital ocean...), see bug #281. --- NEWS | 6 ++++++ psycopg/pqpath.c | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index fe6cea41..df68b4ea 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,12 @@ New features: (:ticket:`#326`). +What's new in psycopg 2.6.2 +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Report the server response status on errors (such as :ticket:`#281`). + + What's new in psycopg 2.6.1 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 5e1974be..5a128382 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -190,8 +190,10 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult **pgres) raise and a meaningful message is better than an empty one. Note: it can happen without it being our error: see ticket #82 */ if (err == NULL || err[0] == '\0') { - PyErr_SetString(DatabaseError, - "error with no message from the libpq"); + PyErr_Format(DatabaseError, + "error with status %s and no message from the libpq", + PQresStatus(pgres == NULL ? + PQstatus(conn->pgconn) : PQresultStatus(*pgres))); return; } From 6803341f21e71bf1c77470b1a5b262503f5b7a39 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 30 Sep 2015 12:15:35 +0100 Subject: [PATCH 118/132] Report NotSupportedError for PGRES_COPY_BOTH and PGRES_SINGLE_TUPLE Fixes #352. --- NEWS | 2 ++ psycopg/pqpath.c | 19 +++++++++++++++++-- tests/test_connection.py | 12 ++++++++++++ tests/testconfig.py | 2 ++ tests/testutils.py | 29 ++++++++++++++++++++++++++--- 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index df68b4ea..287e5fa4 100644 --- a/NEWS +++ b/NEWS @@ -19,6 +19,8 @@ What's new in psycopg 2.6.2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Report the server response status on errors (such as :ticket:`#281`). +- Raise NotSupportedError on unhandled server response status + (:ticket:`#352`). What's new in psycopg 2.6.1 diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 5a128382..6e788058 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -1597,11 +1597,26 @@ pq_fetch(cursorObject *curs, int no_result) ex = -1; break; - default: - Dprintf("pq_fetch: uh-oh, something FAILED: pgconn = %p", curs->conn); + case PGRES_BAD_RESPONSE: + case PGRES_NONFATAL_ERROR: + case PGRES_FATAL_ERROR: + Dprintf("pq_fetch: uh-oh, something FAILED: status = %d pgconn = %p", + status, curs->conn); pq_raise(curs->conn, curs, NULL); ex = -1; break; + + default: + /* PGRES_COPY_BOTH, PGRES_SINGLE_TUPLE, future statuses */ + Dprintf("pq_fetch: got unsupported result: status = %d pgconn = %p", + status, curs->conn); + PyErr_Format(NotSupportedError, + "got server response with unsupported status %s", + PQresStatus(curs->pgres == NULL ? + PQstatus(curs->conn->pgconn) : PQresultStatus(curs->pgres))); + CLEARPGRES(curs->pgres); + ex = -1; + break; } /* error checking, close the connection if necessary (some critical errors diff --git a/tests/test_connection.py b/tests/test_connection.py index fa78eb37..d0a74773 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -26,6 +26,7 @@ import os import time import threading from operator import attrgetter +from StringIO import StringIO import psycopg2 import psycopg2.errorcodes @@ -1103,6 +1104,17 @@ class AutocommitTests(ConnectingTestCase): self.assertEqual(cur.fetchone()[0], 'on') +class ReplicationTest(ConnectingTestCase): + @skip_before_postgres(9, 0) + def test_replication_not_supported(self): + conn = self.repl_connect() + if conn is None: return + cur = conn.cursor() + f = StringIO() + self.assertRaises(psycopg2.NotSupportedError, + cur.copy_expert, "START_REPLICATION 0/0", f) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/tests/testconfig.py b/tests/testconfig.py index f83ded84..0f995fbf 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -7,6 +7,8 @@ dbhost = os.environ.get('PSYCOPG2_TESTDB_HOST', None) dbport = os.environ.get('PSYCOPG2_TESTDB_PORT', None) dbuser = os.environ.get('PSYCOPG2_TESTDB_USER', None) dbpass = os.environ.get('PSYCOPG2_TESTDB_PASSWORD', None) +repl_dsn = os.environ.get('PSYCOPG2_TEST_REPL_DSN', + "dbname=psycopg2_test replication=1") # Check if we want to test psycopg's green path. green = os.environ.get('PSYCOPG2_TEST_GREEN', None) diff --git a/tests/testutils.py b/tests/testutils.py index 987bd7b6..76671d99 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -28,7 +28,7 @@ import os import platform import sys from functools import wraps -from testconfig import dsn +from testconfig import dsn, repl_dsn try: import unittest2 @@ -103,11 +103,35 @@ class ConnectingTestCase(unittest.TestCase): "%s (did you remember calling ConnectingTestCase.setUp()?)" % e) + if 'dsn' in kwargs: + conninfo = kwargs.pop('dsn') + else: + conninfo = dsn import psycopg2 - conn = psycopg2.connect(dsn, **kwargs) + conn = psycopg2.connect(conninfo, **kwargs) self._conns.append(conn) return conn + def repl_connect(self, **kwargs): + """Return a connection set up for replication + + The connection is on "PSYCOPG2_TEST_REPL_DSN" unless overridden by + a *dsn* kwarg. + + Should raise a skip test if not available, but guard for None on + old Python versions. + """ + if 'dsn' not in kwargs: + kwargs['dsn'] = repl_dsn + import psycopg2 + try: + conn = self.connect(**kwargs) + except psycopg2.OperationalError, e: + return self.skipTest("replication db not configured: %s" % e) + + conn.autocommit = True + return conn + def _get_conn(self): if not hasattr(self, '_the_conn'): self._the_conn = self.connect() @@ -388,4 +412,3 @@ class py3_raises_typeerror(object): if sys.version_info[0] >= 3: assert type is TypeError return True - From d3bbd19ccb26123a310c7e4c9869410ecf5f6144 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 11:52:42 +0100 Subject: [PATCH 119/132] Separate parse_dsn test in a test case of their own --- tests/test_connection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_connection.py b/tests/test_connection.py index 950c8056..cd60b170 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -307,6 +307,8 @@ class ConnectionTests(ConnectingTestCase): self.assert_(c.closed, "connection failed so it must be closed") self.assert_('foobar' not in c.dsn, "password was not obscured") + +class ParseDsnTestCase(ConnectingTestCase): def test_parse_dsn(self): from psycopg2 import ProgrammingError from psycopg2.extensions import parse_dsn From 71d96293ab61b8554c9f608905c70aa44a6c008d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 12:00:33 +0100 Subject: [PATCH 120/132] Fixed parse_dsn tests on Python 3 On Python 3 there is no Exception.message attribute. --- tests/test_connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index cd60b170..4cf50622 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -333,7 +333,7 @@ class ParseDsnTestCase(ConnectingTestCase): parse_dsn("dbname='test 2 user=tester password=secret") except ProgrammingError, e: raised = True - self.assertTrue(e.message.find('secret') < 0, + self.assertTrue(str(e).find('secret') < 0, "DSN was not exposed in error message") except e: self.fail("unexpected error condition: " + repr(e)) @@ -354,7 +354,7 @@ class ParseDsnTestCase(ConnectingTestCase): parse_dsn('postgresql://tester:secret@/test?port=1111=x') except ProgrammingError, e: raised = True - self.assertTrue(e.message.find('secret') < 0, + self.assertTrue(str(e).find('secret') < 0, "URI was not exposed in error message") except e: self.fail("unexpected error condition: " + repr(e)) From 5afeee3613de284fd00c3e8b2228bdbe94d74572 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 13:20:11 +0100 Subject: [PATCH 121/132] Added unicode support to parse_dsn Also added support for the argument as a keyword. --- psycopg/psycopgmodule.c | 51 +++++++++++++++++++++------------------- tests/test_connection.py | 26 +++++++++++++++++--- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 0736d260..737a7811 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -115,17 +115,21 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) #define psyco_parse_dsn_doc "parse_dsn(dsn) -> dict" static PyObject * -psyco_parse_dsn(PyObject *self, PyObject *args) +psyco_parse_dsn(PyObject *self, PyObject *args, PyObject *kwargs) { - char *dsn, *err = NULL; + char *err = NULL; PQconninfoOption *options = NULL, *o; - PyObject *res = NULL, *value; + PyObject *dict = NULL, *res = NULL, *dsn; - if (!PyArg_ParseTuple(args, "s", &dsn)) { + static char *kwlist[] = {"dsn", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kwlist, &dsn)) { return NULL; } - options = PQconninfoParse(dsn, &err); + Py_INCREF(dsn); /* for ensure_bytes */ + if (!(dsn = psycopg_ensure_bytes(dsn))) { goto exit; } + + options = PQconninfoParse(Bytes_AS_STRING(dsn), &err); if (options == NULL) { if (err != NULL) { PyErr_Format(ProgrammingError, "error parsing the dsn: %s", err); @@ -133,31 +137,30 @@ psyco_parse_dsn(PyObject *self, PyObject *args) } else { PyErr_SetString(OperationalError, "PQconninfoParse() failed"); } - return NULL; + goto exit; } - res = PyDict_New(); - if (res != NULL) { - for (o = options; o->keyword != NULL; o++) { - if (o->val != NULL) { - value = Text_FromUTF8(o->val); - if (value == NULL) { - Py_DECREF(res); - res = NULL; - break; - } - if (PyDict_SetItemString(res, o->keyword, value) != 0) { - Py_DECREF(value); - Py_DECREF(res); - res = NULL; - break; - } + if (!(dict = PyDict_New())) { goto exit; } + for (o = options; o->keyword != NULL; o++) { + if (o->val != NULL) { + PyObject *value; + if (!(value = Text_FromUTF8(o->val))) { goto exit; } + if (PyDict_SetItemString(dict, o->keyword, value) != 0) { Py_DECREF(value); + goto exit; } + Py_DECREF(value); } } - PQconninfoFree(options); + /* success */ + res = dict; + dict = NULL; + +exit: + PQconninfoFree(options); /* safe on null */ + Py_XDECREF(dict); + Py_XDECREF(dsn); return res; } @@ -759,7 +762,7 @@ static PyMethodDef psycopgMethods[] = { {"_connect", (PyCFunction)psyco_connect, METH_VARARGS|METH_KEYWORDS, psyco_connect_doc}, {"parse_dsn", (PyCFunction)psyco_parse_dsn, - METH_VARARGS, psyco_parse_dsn_doc}, + METH_VARARGS|METH_KEYWORDS, psyco_parse_dsn_doc}, {"adapt", (PyCFunction)psyco_microprotocols_adapt, METH_VARARGS, psyco_microprotocols_adapt_doc}, diff --git a/tests/test_connection.py b/tests/test_connection.py index 4cf50622..ee742580 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -23,6 +23,7 @@ # License for more details. import os +import sys import time import threading from operator import attrgetter @@ -341,7 +342,6 @@ class ParseDsnTestCase(ConnectingTestCase): @skip_before_libpq(9, 2) def test_parse_dsn_uri(self): - from psycopg2 import ProgrammingError from psycopg2.extensions import parse_dsn self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), @@ -351,8 +351,8 @@ class ParseDsnTestCase(ConnectingTestCase): raised = False try: # extra '=' after port value - parse_dsn('postgresql://tester:secret@/test?port=1111=x') - except ProgrammingError, e: + parse_dsn(dsn='postgresql://tester:secret@/test?port=1111=x') + except psycopg2.ProgrammingError, e: raised = True self.assertTrue(str(e).find('secret') < 0, "URI was not exposed in error message") @@ -360,6 +360,26 @@ class ParseDsnTestCase(ConnectingTestCase): self.fail("unexpected error condition: " + repr(e)) self.assertTrue(raised, "ProgrammingError raised due to invalid URI") + def test_unicode_value(self): + from psycopg2.extensions import parse_dsn + snowman = u"\u2603" + d = parse_dsn('dbname=' + snowman) + if sys.version_info[0] < 3: + self.assertEqual(d['dbname'], snowman.encode('utf8')) + else: + self.assertEqual(d['dbname'], snowman) + + def test_unicode_key(self): + from psycopg2.extensions import parse_dsn + snowman = u"\u2603" + self.assertRaises(psycopg2.ProgrammingError, parse_dsn, + snowman + '=' + snowman) + + def test_bad_param(self): + from psycopg2.extensions import parse_dsn + self.assertRaises(TypeError, parse_dsn, None) + self.assertRaises(TypeError, parse_dsn, 42) + class IsolationLevelsTestCase(ConnectingTestCase): From 58918801d65751270b370d97e6c958fcb23193ab Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 13:26:11 +0100 Subject: [PATCH 122/132] Added parse_dsn() to news file --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 287e5fa4..fd4fc6ba 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,7 @@ What's new in psycopg 2.7 New features: +- Added `~psycopg2.extensions.parse_dsn()` function (:ticket:`#321`). - Added `~psycopg2.__libpq_version__` and `~psycopg2.extensions.libpq_version()` to inspect the version of the ``libpq`` library the module was compiled/loaded with From 7e94ce1f14ae25eb6ff901e8c6822d858525780d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 13:31:13 +0100 Subject: [PATCH 123/132] Drop spurious notices in test Getting some "rehashing catalog" debug messages in PG 9.4 --- tests/test_connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index ee742580..68bb6f05 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -154,7 +154,8 @@ class ConnectionTests(ConnectingTestCase): sql = " ".join(["create temp table table2_%d (id serial);" % j for j in range(i, i+10)]) cur.execute(sql) - self.assertEqual(100, len(conn.notices)) + self.assertEqual(len([n for n in conn.notices if 'CREATE TABLE' in n]), + 100) def test_notices_noappend(self): conn = self.conn From ade7dba27cb856c3d2e3f6284aa3e909f8033e10 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 14:29:56 +0100 Subject: [PATCH 124/132] MSVC 2015 compiler support added to news file --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index fd4fc6ba..81d24583 100644 --- a/NEWS +++ b/NEWS @@ -31,6 +31,7 @@ What's new in psycopg 2.6.1 - Fixed deadlock in multithread programs using OpenSSL (:ticket:`#290`). - Correctly unlock the connection after error in flush (:ticket:`#294`). - Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`). +- Added support for MSVC 2015 compiler (:ticket:`#350`). What's new in psycopg 2.6 From 98f2aad4baea7472a6643d6da7986e9b016a6f54 Mon Sep 17 00:00:00 2001 From: ClodoaldoPinto Date: Wed, 9 Sep 2015 11:16:59 -0300 Subject: [PATCH 125/132] Typo correction --- NEWS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 81d24583..3ba60dea 100644 --- a/NEWS +++ b/NEWS @@ -46,7 +46,7 @@ New features: Bug fixes: -- Json apapter's `!str()` returns the adapted content instead of the `!repr()` +- Json adapter's `!str()` returns the adapted content instead of the `!repr()` (:ticket:`#191`). From 9e6c3322d8640bca7007a222973d87d8ea60057c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 14:44:14 +0100 Subject: [PATCH 126/132] Fixed PersistentConnectionPool on Python 3 Fixes ticket #348. --- NEWS | 5 +++-- lib/pool.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/NEWS b/NEWS index 3ba60dea..651b74c4 100644 --- a/NEWS +++ b/NEWS @@ -20,8 +20,9 @@ What's new in psycopg 2.6.2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Report the server response status on errors (such as :ticket:`#281`). -- Raise NotSupportedError on unhandled server response status +- Raise `!NotSupportedError` on unhandled server response status (:ticket:`#352`). +- Fixed `!PersistentConnectionPool` on Python 3 (:ticket:`#348`). What's new in psycopg 2.6.1 @@ -30,7 +31,7 @@ What's new in psycopg 2.6.1 - Lists consisting of only `None` are escaped correctly (:ticket:`#285`). - Fixed deadlock in multithread programs using OpenSSL (:ticket:`#290`). - Correctly unlock the connection after error in flush (:ticket:`#294`). -- Fixed ``MinTimeLoggingCursor.callproc()`` (:ticket:`#309`). +- Fixed `!MinTimeLoggingCursor.callproc()` (:ticket:`#309`). - Added support for MSVC 2015 compiler (:ticket:`#350`). diff --git a/lib/pool.py b/lib/pool.py index 4f858ab1..8d7c4afb 100644 --- a/lib/pool.py +++ b/lib/pool.py @@ -204,8 +204,8 @@ class PersistentConnectionPool(AbstractConnectionPool): # we we'll need the thread module, to determine thread ids, so we # import it here and copy it in an instance variable - import thread - self.__thread = thread + import thread as _thread # work around for 2to3 bug - see ticket #348 + self.__thread = _thread def getconn(self): """Generate thread id and return a connection.""" From f635547ec690cc76b02da8e22772d41008fcc46e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 15:26:13 +0100 Subject: [PATCH 127/132] The wait_select callback can cancel a query using Ctrl-C Fixes #333. --- NEWS | 3 +++ doc/src/extras.rst | 3 +++ doc/src/faq.rst | 31 +++++++++++++++++++++++++++++++ lib/extras.py | 23 ++++++++++++++--------- 4 files changed, 51 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 651b74c4..83ba166e 100644 --- a/NEWS +++ b/NEWS @@ -20,6 +20,9 @@ What's new in psycopg 2.6.2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Report the server response status on errors (such as :ticket:`#281`). +- The `~psycopg2.extras.wait_select` callback allows interrupting a + long-running query in an interactive shell using :kbd:`Ctrl-C` + (:ticket:`#333`). - Raise `!NotSupportedError` on unhandled server response status (:ticket:`#352`). - Fixed `!PersistentConnectionPool` on Python 3 (:ticket:`#348`). diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 36ef0132..0e21ae58 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -611,3 +611,6 @@ Coroutine support .. autofunction:: wait_select(conn) + .. versionchanged:: 2.6.2 + allow to cancel a query using :kbd:`Ctrl-C`, see + :ref:`the FAQ ` for an example. diff --git a/doc/src/faq.rst b/doc/src/faq.rst index 8f2f1ecc..69273ba5 100644 --- a/doc/src/faq.rst +++ b/doc/src/faq.rst @@ -223,6 +223,37 @@ What are the advantages or disadvantages of using named cursors? little memory on the client and to skip or discard parts of the result set. +.. _faq-interrupt-query: +.. cssclass:: faq + +How do I interrupt a long-running query in an interactive shell? + Normally the interactive shell becomes unresponsive to :kbd:`Ctrl-C` when + running a query. Using a connection in green mode allows Python to + receive and handle the interrupt, although it may leave the connection + broken, if the async callback doesn't handle the `!KeyboardInterrupt` + correctly. + + Starting from psycopg 2.6.2, the `~psycopg2.extras.wait_select` callback + can handle a :kbd:`Ctrl-C` correctly. For previous versions, you can use + `this implementation`__. + + .. __: http://initd.org/psycopg/articles/2014/07/20/cancelling-postgresql-statements-python/ + + .. code-block:: pycon + + >>> psycopg2.extensions.set_wait_callback(psycopg2.extensions.wait_select) + >>> cnn = psycopg2.connect('') + >>> cur = cnn.cursor() + >>> cur.execute("select pg_sleep(10)") + ^C + Traceback (most recent call last): + File "", line 1, in + QueryCanceledError: canceling statement due to user request + + >>> cnn.rollback() + >>> # You can use the connection and cursor again from here + + .. _faq-compile: Problems compiling and deploying psycopg2 diff --git a/lib/extras.py b/lib/extras.py index c9f1cbcd..2713d6fc 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -575,15 +575,20 @@ def wait_select(conn): from psycopg2.extensions import POLL_OK, POLL_READ, POLL_WRITE while 1: - state = conn.poll() - if state == POLL_OK: - break - elif state == POLL_READ: - select.select([conn.fileno()], [], []) - elif state == POLL_WRITE: - select.select([], [conn.fileno()], []) - else: - raise conn.OperationalError("bad state from poll: %s" % state) + try: + state = conn.poll() + if state == POLL_OK: + break + elif state == POLL_READ: + select.select([conn.fileno()], [], []) + elif state == POLL_WRITE: + select.select([], [conn.fileno()], []) + else: + raise conn.OperationalError("bad state from poll: %s" % state) + except KeyboardInterrupt: + conn.cancel() + # the loop will be broken by a server error + continue def _solve_conn_curs(conn_or_curs): From c73c1c577170f51c2dfadcdb61c34e4def82d709 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 1 Oct 2015 17:02:43 +0100 Subject: [PATCH 128/132] Decref the ssl module after importing --- psycopg/psycopgmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 737a7811..c77dce5b 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -234,13 +234,16 @@ psyco_register_type(PyObject *self, PyObject *args) static void psyco_libcrypto_threads_init(void) { + PyObject *m; + /* importing the ssl module sets up Python's libcrypto callbacks */ - if (PyImport_ImportModule("ssl") != NULL) { + if ((m = PyImport_ImportModule("ssl"))) { /* disable libcrypto setup in libpq, so it won't stomp on the callbacks that have already been set up */ #if PG_VERSION_NUM >= 80400 PQinitOpenSSL(1, 0); #endif + Py_DECREF(m); } else { /* might mean that Python has been compiled without OpenSSL support, From 9295bce154182863e19342b6a4c2e80a58187120 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Tue, 13 Oct 2015 17:29:55 +0200 Subject: [PATCH 129/132] Add psycopg2.extensions.quote_ident. --- doc/src/extensions.rst | 13 +++++++++++++ lib/extensions.py | 2 +- psycopg/psycopgmodule.c | 38 ++++++++++++++++++++++++++++++++++++++ psycopg/utils.c | 4 ++-- tests/test_quote.py | 7 +++++++ 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index 4db76b01..d96cca4f 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -221,6 +221,19 @@ functionalities defined by the |DBAPI|_. .. __: http://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQLIBVERSION +.. function:: quote_ident(str, scope) + + Return quoted identifier according to PostgreSQL quoting rules. + + The *scope* must be a `connection` or a `cursor`, the underlying + connection encoding is used for any necessary character conversion. + + Requires libpq >= 9.0. + + .. seealso:: libpq docs for `PQescapeIdentifier()`__ + + .. __: http://www.postgresql.org/docs/current/static/libpq-exec.html#LIBPQ-PQESCAPEIDENTIFIER + .. _sql-adaptation-objects: SQL adaptation protocol objects diff --git a/lib/extensions.py b/lib/extensions.py index d10e8ac6..b40e28b8 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -56,7 +56,7 @@ try: except ImportError: pass -from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid, libpq_version, parse_dsn +from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid, libpq_version, parse_dsn, quote_ident from psycopg2._psycopg import string_types, binary_types, new_type, new_array_type, register_type from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics, Column diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index c77dce5b..9906b7be 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -165,6 +165,42 @@ exit: return res; } + +#define psyco_quote_ident_doc "quote_ident(str, conn_or_curs) -> str" + +static PyObject * +psyco_quote_ident(PyObject *self, PyObject *args) +{ + const char *str = NULL; + char *quoted; + PyObject *obj, *result; + connectionObject *conn; + + if (!PyArg_ParseTuple(args, "sO", &str, &obj)) return NULL; + + if (PyObject_TypeCheck(obj, &cursorType)) { + conn = ((cursorObject*)obj)->conn; + } + else if (PyObject_TypeCheck(obj, &connectionType)) { + conn = (connectionObject*)obj; + } + else { + PyErr_SetString(PyExc_TypeError, + "argument 2 must be a connection or a cursor"); + return NULL; + } + + quoted = PQescapeIdentifier(conn->pgconn, str, strlen(str)); + if (!quoted) { + PyErr_NoMemory(); + return NULL; + } + result = conn_text_from_chars(conn, quoted); + PQfreemem(quoted); + + return result; +} + /** type registration **/ #define psyco_register_type_doc \ "register_type(obj, conn_or_curs) -> None -- register obj with psycopg type system\n\n" \ @@ -768,6 +804,8 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS|METH_KEYWORDS, psyco_parse_dsn_doc}, {"adapt", (PyCFunction)psyco_microprotocols_adapt, METH_VARARGS, psyco_microprotocols_adapt_doc}, + {"quote_ident", (PyCFunction)psyco_quote_ident, + METH_VARARGS, psyco_quote_ident_doc}, {"register_type", (PyCFunction)psyco_register_type, METH_VARARGS, psyco_register_type_doc}, diff --git a/psycopg/utils.c b/psycopg/utils.c index 836f6129..ec8e47c8 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -87,7 +87,7 @@ psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, return to; } -/* Escape a string to build a valid PostgreSQL identifier +/* Escape a string to build a valid PostgreSQL identifier. * * Allocate a new buffer on the Python heap containing the new string. * 'len' is optional: if 0 the length is calculated. @@ -96,7 +96,7 @@ psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, * * WARNING: this function is not so safe to allow untrusted input: it does no * check for multibyte chars. Such a function should be built on - * PQescapeIndentifier, which is only available from PostgreSQL 9.0. + * PQescapeIdentifier, which is only available from PostgreSQL 9.0. */ char * psycopg_escape_identifier_easy(const char *from, Py_ssize_t len) diff --git a/tests/test_quote.py b/tests/test_quote.py index e7b3c316..a24ab6d4 100755 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -165,6 +165,13 @@ class TestQuotedString(ConnectingTestCase): self.assertEqual(q.encoding, 'utf_8') +class TestQuotedIdentifier(ConnectingTestCase): + def test_identifier(self): + from psycopg2.extensions import quote_ident + self.assertEqual(quote_ident('blah-blah', self.conn), '"blah-blah"') + self.assertEqual(quote_ident('quote"inside', self.conn), '"quote""inside"') + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 89bb6b0711f8ba59f7b8e81339ddaa53356233a2 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Thu, 15 Oct 2015 11:52:18 +0200 Subject: [PATCH 130/132] Proper unicode handling in quote_ident. --- psycopg/psycopgmodule.c | 38 +++++++++++++++++++++++++++++--------- tests/test_quote.py | 13 ++++++++++++- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 9906b7be..cf70a4ad 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -166,17 +166,25 @@ exit: } -#define psyco_quote_ident_doc "quote_ident(str, conn_or_curs) -> str" +#define psyco_quote_ident_doc \ +"quote_ident(str, conn_or_curs) -> str -- wrapper around PQescapeIdentifier\n\n" \ +":Parameters:\n" \ +" * `str`: A bytes or unicode object\n" \ +" * `conn_or_curs`: A connection or cursor, required" static PyObject * -psyco_quote_ident(PyObject *self, PyObject *args) +psyco_quote_ident(PyObject *self, PyObject *args, PyObject *kwargs) { - const char *str = NULL; - char *quoted; - PyObject *obj, *result; +#if PG_VERSION_NUM >= 90000 + PyObject *ident = NULL, *obj = NULL, *result = NULL; connectionObject *conn; + const char *str; + char *quoted = NULL; - if (!PyArg_ParseTuple(args, "sO", &str, &obj)) return NULL; + static char *kwlist[] = {"ident", "scope", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO", kwlist, &ident, &obj)) { + return NULL; + } if (PyObject_TypeCheck(obj, &cursorType)) { conn = ((cursorObject*)obj)->conn; @@ -190,15 +198,27 @@ psyco_quote_ident(PyObject *self, PyObject *args) return NULL; } + Py_INCREF(ident); /* for ensure_bytes */ + if (!(ident = psycopg_ensure_bytes(ident))) { goto exit; } + + str = Bytes_AS_STRING(ident); + quoted = PQescapeIdentifier(conn->pgconn, str, strlen(str)); if (!quoted) { PyErr_NoMemory(); - return NULL; + goto exit; } result = conn_text_from_chars(conn, quoted); + +exit: PQfreemem(quoted); + Py_XDECREF(ident); return result; +#else + PyErr_SetString(NotSupportedError, "PQescapeIdentifier not available in libpq < 9.0"); + return NULL; +#endif } /** type registration **/ @@ -802,10 +822,10 @@ static PyMethodDef psycopgMethods[] = { METH_VARARGS|METH_KEYWORDS, psyco_connect_doc}, {"parse_dsn", (PyCFunction)psyco_parse_dsn, METH_VARARGS|METH_KEYWORDS, psyco_parse_dsn_doc}, + {"quote_ident", (PyCFunction)psyco_quote_ident, + METH_VARARGS|METH_KEYWORDS, psyco_quote_ident_doc}, {"adapt", (PyCFunction)psyco_microprotocols_adapt, METH_VARARGS, psyco_microprotocols_adapt_doc}, - {"quote_ident", (PyCFunction)psyco_quote_ident, - METH_VARARGS, psyco_quote_ident_doc}, {"register_type", (PyCFunction)psyco_register_type, METH_VARARGS, psyco_register_type_doc}, diff --git a/tests/test_quote.py b/tests/test_quote.py index a24ab6d4..6e945624 100755 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -23,7 +23,7 @@ # License for more details. import sys -from testutils import unittest, ConnectingTestCase +from testutils import unittest, ConnectingTestCase, skip_before_libpq import psycopg2 import psycopg2.extensions @@ -166,11 +166,22 @@ class TestQuotedString(ConnectingTestCase): class TestQuotedIdentifier(ConnectingTestCase): + @skip_before_libpq(9, 0) def test_identifier(self): from psycopg2.extensions import quote_ident self.assertEqual(quote_ident('blah-blah', self.conn), '"blah-blah"') self.assertEqual(quote_ident('quote"inside', self.conn), '"quote""inside"') + @skip_before_libpq(9, 0) + def test_unicode_ident(self): + from psycopg2.extensions import quote_ident + snowman = u"\u2603" + quoted = '"' + snowman + '"' + if sys.version_info[0] < 3: + self.assertEqual(quote_ident(snowman, self.conn), quoted.encode('utf8')) + else: + self.assertEqual(quote_ident(snowman, self.conn), quoted) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 109409bc951b7dd3e61712a65289e3458430656a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 15 Oct 2015 11:06:44 +0100 Subject: [PATCH 131/132] Mention quote_ident() in NEWS file --- NEWS | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS b/NEWS index 83ba166e..5200c4dd 100644 --- a/NEWS +++ b/NEWS @@ -14,6 +14,7 @@ New features: - The attributes `~connection.notices` and `~connection.notifies` can be customized replacing them with any object exposing an `!append()` method (:ticket:`#326`). +- Added `~psycopg2.extensions.quote_ident()` function (:ticket:`#359`). What's new in psycopg 2.6.2 From fe4cb0d49353f56328b9981a5140ecda65e972b4 Mon Sep 17 00:00:00 2001 From: Oleksandr Shulgin Date: Mon, 26 Oct 2015 17:39:39 +0100 Subject: [PATCH 132/132] Fix stale Dprintfs in pqpath.c referring to 'status' --- psycopg/pqpath.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 6e788058..b643512d 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -1601,7 +1601,7 @@ pq_fetch(cursorObject *curs, int no_result) case PGRES_NONFATAL_ERROR: case PGRES_FATAL_ERROR: Dprintf("pq_fetch: uh-oh, something FAILED: status = %d pgconn = %p", - status, curs->conn); + pgstatus, curs->conn); pq_raise(curs->conn, curs, NULL); ex = -1; break; @@ -1609,7 +1609,7 @@ pq_fetch(cursorObject *curs, int no_result) default: /* PGRES_COPY_BOTH, PGRES_SINGLE_TUPLE, future statuses */ Dprintf("pq_fetch: got unsupported result: status = %d pgconn = %p", - status, curs->conn); + pgstatus, curs->conn); PyErr_Format(NotSupportedError, "got server response with unsupported status %s", PQresStatus(curs->pgres == NULL ?