diff --git a/NEWS b/NEWS index fb2e4cc8..ca91abef 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,16 @@ What's new in psycopg 2.4.4 --------------------------- + - 'register_composite()' also works with the types implicitly defined + after a table row, not only with the ones created by 'CREATE TYPE'. + - Values for the isolation level symbolic constants restored to what + they were before release 2.4.2 to avoid breaking apps using the + values instead of the constants. - Named DictCursor/RealDictCursor honour itersize (ticket #80). - Fixed rollback on error on Zope (ticket #73). + - Raise 'DatabaseError' instead of 'Error' with empty libpq errors, + consistently with other disconnection-related errors: regression + introduced in release 2.4.1 (ticket #82). What's new in psycopg 2.4.3 diff --git a/README b/README index 93d25d0c..7466bf91 100644 --- a/README +++ b/README @@ -8,7 +8,7 @@ and stable as a rock. psycopg2 is different from the other database adapter because it was designed for heavily multi-threaded applications that create and destroy lots of cursors and make a conspicuous number of concurrent INSERTs or -UPDATEs. psycopg2 also provide full asycronous operations and support +UPDATEs. psycopg2 also provides full asynchronous operations and support for coroutine libraries. psycopg2 can compile and run on Linux, FreeBSD, Solaris, MacOS X and diff --git a/doc/src/errorcodes.rst b/doc/src/errorcodes.rst index c6ec089d..499881d1 100644 --- a/doc/src/errorcodes.rst +++ b/doc/src/errorcodes.rst @@ -50,7 +50,7 @@ An example of the available constants defined in the module: '42P01' Constants representing all the error values documented by PostgreSQL versions -between 8.1 and 9.0 are included in the module. +between 8.1 and 9.1 are included in the module. .. autofunction:: lookup(code) diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 5f85d616..710e9b5c 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -128,6 +128,8 @@ Additional data types --------------------- +.. _adapt-hstore: + .. index:: pair: hstore; Data types pair: dict; Adaptation @@ -157,6 +159,8 @@ can be enabled using the `register_hstore()` function. +.. _adapt-composite: + .. index:: pair: Composite types; Data types pair: tuple; Adaptation @@ -168,8 +172,9 @@ Composite types casting .. versionadded:: 2.4 Using `register_composite()` it is possible to cast a PostgreSQL composite -type (e.g. created with |CREATE TYPE|_ command) into a Python named tuple, or -into a regular tuple if :py:func:`collections.namedtuple` is not found. +type (either created with the |CREATE TYPE|_ command or implicitly defined +after a table row type) into a Python named tuple, or into a regular tuple if +:py:func:`collections.namedtuple` is not found. .. |CREATE TYPE| replace:: :sql:`CREATE TYPE` .. _CREATE TYPE: http://www.postgresql.org/docs/9.0/static/sql-createtype.html diff --git a/doc/src/index.rst b/doc/src/index.rst index 52e7dd2f..906753e0 100644 --- a/doc/src/index.rst +++ b/doc/src/index.rst @@ -4,20 +4,31 @@ Psycopg -- PostgreSQL database adapter for Python .. sectionauthor:: Daniele Varrazzo -Psycopg is a PostgreSQL_ database adapter for the Python_ programming -language. Its main advantages are that it supports the full Python |DBAPI|_ +Psycopg_ is a PostgreSQL_ database adapter for the Python_ programming +language. Its main features are that it supports the full Python |DBAPI|_ and it is thread safe (threads can share the connections). It was designed for heavily multi-threaded applications that create and destroy lots of cursors and -make a conspicuous number of concurrent :sql:`INSERT`\ s or :sql:`UPDATE`\ s. -The psycopg distribution includes ZPsycopgDA, a Zope_ Database Adapter. +make a large number of concurrent :sql:`INSERT`\ s or :sql:`UPDATE`\ s. +The Psycopg distribution includes ZPsycopgDA, a Zope_ Database Adapter. -Psycopg 2 is an almost complete rewrite of the Psycopg 1.1.x branch. Psycopg 2 -features complete libpq_ v3 protocol, |COPY-TO-FROM|__ and full :ref:`object -adaptation ` for all basic Python types: strings (including unicode), ints, -longs, floats, buffers (binary objects), booleans, `mx.DateTime`_ and builtin -datetime types. It also supports unicode queries and Python lists mapped to -PostgreSQL arrays. +Psycopg 2 is mostly implemented in C as a libpq_ wrapper, resulting in being +both efficient and secure. It features client-side and :ref:`server-side +` cursors, :ref:`asynchronous communication +` and :ref:`notifications `, |COPY-TO-FROM|__ +support, and a flexible :ref:`objects adaptation system +`. Many basic Python types are supported +out-of-the-box and mapped to matching PostgreSQL data types, such as strings +(both bytes and Unicode), numbers (ints, longs, floats, decimals), booleans and +datetime objects (both built-in and `mx.DateTime`_), several types of +:ref:`binary objects `. Also available are mappings between lists +and PostgreSQL arrays of any supported type, between :ref:`dictionaries and +PostgreSQL hstores `, and between :ref:`tuples/namedtuples and +PostgreSQL composite types `. +Psycopg 2 is both Unicode and Python 3 friendly. + + +.. _Psycopg: http://initd.org/psycopg/ .. _PostgreSQL: http://www.postgresql.org/ .. _Python: http://www.python.org/ .. _Zope: http://www.zope.org/ diff --git a/doc/src/usage.rst b/doc/src/usage.rst index e0595d28..d480adef 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -256,11 +256,11 @@ the SQL string that would be sent to the database. single: memoryview; Adaptation single: Binary string -- Binary types: Python types representing binary objects are converted in +- Binary types: Python types representing binary objects are converted into PostgreSQL binary string syntax, suitable for :sql:`bytea` fields. Such types are `buffer` (only available in Python 2), `memoryview` (available from Python 2.7), `bytearray` (available from Python 2.6) and `bytes` - (only form Python 3: the name is available from Python 2.6 but it's only an + (only from Python 3: the name is available from Python 2.6 but it's only an alias for the type `!str`). Any object implementing the `Revised Buffer Protocol`__ should be usable as binary type where the protocol is supported (i.e. from Python 2.6). Received data is returned as `!buffer` (in Python 2) diff --git a/examples/binary.py b/examples/binary.py index 89d5c739..804735b6 100644 --- a/examples/binary.py +++ b/examples/binary.py @@ -16,7 +16,7 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys import psycopg2 @@ -79,7 +79,7 @@ for row in curs.fetchall(): print "done" print " python type of image data is", type(row[0]) -# this rollback is requires because we can't drop a table with a binary cusor +# this rollback is required because we can't drop a table with a binary cusor # declared and still open conn.rollback() diff --git a/examples/copy_from.py b/examples/copy_from.py index 8dd7efd8..861dfdb8 100644 --- a/examples/copy_from.py +++ b/examples/copy_from.py @@ -17,7 +17,7 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys import os @@ -165,7 +165,7 @@ try: curs.copy_from(data, 'test_copy') except StandardError, err: conn.rollback() - print " Catched error (as expected):\n", err + print " Caught error (as expected):\n", err conn.rollback() diff --git a/examples/copy_to.py b/examples/copy_to.py index 4c1398fd..ec647945 100644 --- a/examples/copy_to.py +++ b/examples/copy_to.py @@ -17,7 +17,7 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys import os diff --git a/examples/cursor.py b/examples/cursor.py index f3b2addb..2d56fd73 100644 --- a/examples/cursor.py +++ b/examples/cursor.py @@ -58,6 +58,6 @@ print "Result of fetchone():", curs.fetchone() try: curs.fetchone() except NoDataError, err: - print "Exception caugth:", err + print "Exception caught:", err conn.rollback() diff --git a/examples/dialtone.py b/examples/dialtone.py index 3a55686d..a89021ce 100644 --- a/examples/dialtone.py +++ b/examples/dialtone.py @@ -104,10 +104,10 @@ print adapt(Order()).generateInsert() - Discussion Psycopg 2 has a great new feature: adaptation. The big thing about -adaptation is that it enable the programmer to glue most of the +adaptation is that it enables the programmer to glue most of the code out there without many difficulties. -This recipe tries to focus the attention on a way to generate SQL queries to +This recipe tries to focus attention on a way to generate SQL queries to insert completely new objects inside a database. As you can see objects do not know anything about the code that is handling them. We specify all the fields that we need for each object through the persistent_fields dict. @@ -116,7 +116,7 @@ The most important lines of this recipe are: register_adapter(Album, ObjectMapper) register_adapter(Order, ObjectMapper) -In these line we notify the system that when we call adapt with an Album instance +In these lines we notify the system that when we call adapt with an Album instance as an argument we want it to istantiate ObjectMapper passing the Album instance as argument (self.orig in the ObjectMapper class). diff --git a/examples/dt.py b/examples/dt.py index 3ab5ee8d..c876590a 100644 --- a/examples/dt.py +++ b/examples/dt.py @@ -16,7 +16,7 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys import psycopg2 @@ -73,7 +73,7 @@ print "Extracting values inserted with mx.DateTime wrappers:" curs.execute("SELECT d, t, dt, z FROM test_dt WHERE k = 1") for n, x in zip(mx1[1:], curs.fetchone()): try: - # this will work only is psycopg has been compiled with datetime + # this will work only if psycopg has been compiled with datetime # as the default typecaster for date/time values s = repr(n) + "\n -> " + str(adapt(n)) + \ "\n -> " + repr(x) + "\n -> " + x.isoformat() @@ -87,7 +87,7 @@ print "Extracting values inserted with Python datetime wrappers:" curs.execute("SELECT d, t, dt, z FROM test_dt WHERE k = 2") for n, x in zip(dt1[1:], curs.fetchone()): try: - # this will work only is psycopg has been compiled with datetime + # this will work only if psycopg has been compiled with datetime # as the default typecaster for date/time values s = repr(n) + "\n -> " + repr(x) + "\n -> " + x.isoformat() except: diff --git a/examples/fetch.py b/examples/fetch.py index 996f7b74..b47ed3f1 100644 --- a/examples/fetch.py +++ b/examples/fetch.py @@ -16,7 +16,7 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys import psycopg2 @@ -52,7 +52,7 @@ conn.commit() # does some nice tricks with the transaction and postgres cursors # (remember to always commit or rollback before a DECLARE) # -# we don't need to DECLARE ourselves, psycopg now support named +# we don't need to DECLARE ourselves, psycopg now supports named # cursors (but we leave the code here, comments, as an example of # what psycopg is doing under the hood) # diff --git a/examples/lastrowid.py b/examples/lastrowid.py index 827f7c22..12c9174c 100644 --- a/examples/lastrowid.py +++ b/examples/lastrowid.py @@ -16,7 +16,7 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys, psycopg2 diff --git a/examples/notify.py b/examples/notify.py index 7dc15d2a..9b9a2d79 100644 --- a/examples/notify.py +++ b/examples/notify.py @@ -16,11 +16,12 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys -import psycopg2 import select +import psycopg2 +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT if len(sys.argv) > 1: DSN = sys.argv[1] @@ -29,7 +30,7 @@ print "Opening connection using dns:", DSN conn = psycopg2.connect(DSN) print "Encoding for this connection is", conn.encoding -conn.set_isolation_level(0) +conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) curs = conn.cursor() curs.execute("listen test") diff --git a/examples/threads.py b/examples/threads.py index 41ff21f3..3f2c05fd 100644 --- a/examples/threads.py +++ b/examples/threads.py @@ -29,15 +29,16 @@ SELECT_STEP = 500 SELECT_DIV = 250 # the available modes are: -# 0 - one connection for all insert and one for all select threads +# 0 - one connection for all inserts and one for all select threads # 1 - connections generated using the connection pool MODE = 1 -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys, psycopg2, threading from psycopg2.pool import ThreadedConnectionPool +from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT if len(sys.argv) > 1: DSN = sys.argv[1] @@ -89,21 +90,21 @@ def insert_func(conn_or_pool, rows): conn.commit() ## a nice select function that prints the current number of rows in the -## database (and transefer them, putting some pressure on the network) +## database (and transfer them, putting some pressure on the network) def select_func(conn_or_pool, z): name = threading.currentThread().getName() if MODE == 0: conn = conn_or_pool - conn.set_isolation_level(0) + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) for i in range(SELECT_SIZE): if divmod(i, SELECT_STEP)[1] == 0: try: if MODE == 1: conn = conn_or_pool.getconn() - conn.set_isolation_level(0) + conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) c = conn.cursor() c.execute("SELECT * FROM test_threads WHERE value2 < %s", (int(i/z),)) diff --git a/examples/usercast.py b/examples/usercast.py index 9540b28f..a0210f77 100644 --- a/examples/usercast.py +++ b/examples/usercast.py @@ -17,7 +17,7 @@ DSN = 'dbname=test' -## don't modify anything below tis line (except for experimenting) +## don't modify anything below this line (except for experimenting) import sys import psycopg2 diff --git a/lib/extras.py b/lib/extras.py index 491a3908..a6fcbc04 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -704,7 +704,7 @@ WHERE typname = 'hstore'; # revert the status of the connection as before the command if (conn_status != _ext.STATUS_IN_TRANSACTION - and conn.isolation_level != _ext.ISOLATION_LEVEL_AUTOCOMMIT): + and not conn.autocommit): conn.rollback() return tuple(rv0), tuple(rv1) @@ -841,8 +841,8 @@ class CompositeCaster(object): tokens = self.tokenize(s) if len(tokens) != len(self.atttypes): raise psycopg2.DataError( - "expecting %d components for the type %s, %d found instead", - (len(self.atttypes), self.name, len(self.tokens))) + "expecting %d components for the type %s, %d found instead" % + (len(self.atttypes), self.name, len(tokens))) attrs = [ curs.cast(oid, token) for oid, token in zip(self.atttypes, tokens) ] @@ -913,7 +913,8 @@ SELECT t.oid, %s, attname, atttypid FROM pg_type t JOIN pg_namespace ns ON typnamespace = ns.oid JOIN pg_attribute a ON attrelid = typrelid -WHERE typname = %%s and nspname = %%s +WHERE typname = %%s AND nspname = %%s + AND attnum > 0 AND NOT attisdropped ORDER BY attnum; """ % typarray, (tname, schema)) @@ -921,7 +922,7 @@ ORDER BY attnum; # revert the status of the connection as before the command if (conn_status != _ext.STATUS_IN_TRANSACTION - and conn.isolation_level != _ext.ISOLATION_LEVEL_AUTOCOMMIT): + and not conn.autocommit): conn.rollback() if not recs: diff --git a/lib/psycopg1.py b/lib/psycopg1.py index 2e5aa826..7a24c5f2 100644 --- a/lib/psycopg1.py +++ b/lib/psycopg1.py @@ -33,14 +33,14 @@ from psycopg2.extensions import cursor as _2cursor from psycopg2.extensions import connection as _2connection from psycopg2 import * -del connect - +import psycopg2.extensions as _ext +_2connect = connect def connect(*args, **kwargs): """connect(dsn, ...) -> new psycopg 1.1.x compatible connection object""" kwargs['connection_factory'] = connection - conn = _2psycopg.connect(*args, **kwargs) - conn.set_isolation_level(2) + conn = _2connect(*args, **kwargs) + conn.set_isolation_level(_ext.ISOLATION_LEVEL_READ_COMMITTED) return conn class connection(_2connection): @@ -53,9 +53,9 @@ class connection(_2connection): def autocommit(self, on_off=1): """autocommit(on_off=1) -> switch autocommit on (1) or off (0)""" if on_off > 0: - self.set_isolation_level(0) + self.set_isolation_level(_ext.ISOLATION_LEVEL_AUTOCOMMIT) else: - self.set_isolation_level(2) + self.set_isolation_level(_ext.ISOLATION_LEVEL_READ_COMMITTED) class cursor(_2cursor): diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 1f0d5da9..1734eceb 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -76,6 +76,7 @@ exception_from_sqlstate(const char *sqlstate) break; case '2': switch (sqlstate[1]) { + case '0': /* Class 20 - Case Not Found */ case '1': /* Class 21 - Cardinality Violation */ return ProgrammingError; case '2': /* Class 22 - Data Exception */ @@ -135,6 +136,8 @@ exception_from_sqlstate(const char *sqlstate) return OperationalError; case 'F': /* Class F0 - Configuration File Error */ return InternalError; + case 'H': /* Class HV - Foreign Data Wrapper Error (SQL/MED) */ + return OperationalError; case 'P': /* Class P0 - PL/pgSQL Error */ return InternalError; case 'X': /* Class XX - Internal Error */ @@ -157,7 +160,8 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult *pgres) const char *code = NULL; if (conn == NULL) { - PyErr_SetString(Error, "psycopg went psycotic and raised a null error"); + PyErr_SetString(DatabaseError, + "psycopg went psycotic and raised a null error"); return; } @@ -183,9 +187,11 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult *pgres) /* if the is no error message we probably called pq_raise without reason: we need to set an exception anyway because the caller will probably - raise and a meaningful message is better than an empty one */ + raise and a meaningful message is better than an empty one. + Note: it can happen without it being our error: see ticket #82 */ if (err == NULL || err[0] == '\0') { - PyErr_SetString(Error, "psycopg went psycotic without error set"); + PyErr_SetString(DatabaseError, + "error with no message from the libpq"); return; } diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py index e59f399e..01fbf90c 100755 --- a/scripts/make_errorcodes.py +++ b/scripts/make_errorcodes.py @@ -48,11 +48,7 @@ def read_base_file(filename): raise ValueError("can't find the separator. Is this the right file?") def parse_errors(url): - page = urllib2.urlopen(url).read() - page = page.replace( # make things easier - 'PostgreSQL', - 'PostgreSQL') - page = BS(page) + page = BS(urllib2.urlopen(url)) table = page('table')[1]('tbody')[0] classes = {} @@ -60,9 +56,9 @@ def parse_errors(url): for tr in table('tr'): if tr.td.get('colspan'): # it's a class - label = tr.b.string.encode("ascii") + label = ' '.join(' '.join(tr(text=True)).split()) \ + .replace(u'\u2014', '-').encode('ascii') assert label.startswith('Class') - label = label.replace("—", "-") class_ = label.split()[1] assert len(class_) == 2 classes[class_] = label @@ -73,14 +69,14 @@ def parse_errors(url): tds = tr('td') if len(tds) == 3: - errlabel = tds[1].string.replace(" ", "_").encode("ascii") + errlabel = '_'.join(tds[1].string.split()).encode('ascii') # double check the columns are equal - cond_name = tds[2].string.upper().encode("ascii") + cond_name = tds[2].string.strip().upper().encode("ascii") assert errlabel == cond_name, tr elif len(tds) == 2: - # found in PG 9.1 beta3 docs + # found in PG 9.1 docs errlabel = tds[1].tt.string.upper().encode("ascii") else: diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index 9936305b..29a935ee 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -648,6 +648,61 @@ class AdaptTypeTestCase(unittest.TestCase): self.assertEqual(v[1][1], "world") self.assertEqual(v[1][2], date(2011,1,3)) + @skip_if_no_composite + def test_wrong_schema(self): + oid = self._create_type("type_ii", [("a", "integer"), ("b", "integer")]) + from psycopg2.extras import CompositeCaster + c = CompositeCaster('type_ii', oid, [('a', 23), ('b', 23), ('c', 23)]) + curs = self.conn.cursor() + psycopg2.extensions.register_type(c.typecaster, curs) + curs.execute("select (1,2)::type_ii") + self.assertRaises(psycopg2.DataError, curs.fetchone) + + @skip_if_no_composite + @skip_before_postgres(8, 4) + def test_from_tables(self): + curs = self.conn.cursor() + curs.execute("""create table ctest1 ( + id integer primary key, + temp int, + label varchar + );""") + + curs.execute("""alter table ctest1 drop temp;""") + + curs.execute("""create table ctest2 ( + id serial primary key, + label varchar, + test_id integer references ctest1(id) + );""") + + curs.execute("""insert into ctest1 (id, label) values + (1, 'test1'), + (2, 'test2');""") + curs.execute("""insert into ctest2 (label, test_id) values + ('testa', 1), + ('testb', 1), + ('testc', 2), + ('testd', 2);""") + + psycopg2.extras.register_composite("ctest1", curs) + psycopg2.extras.register_composite("ctest2", curs) + + curs.execute(""" + select ctest1, array_agg(ctest2) as test2s + from ( + select ctest1, ctest2 + from ctest1 inner join ctest2 on ctest1.id = ctest2.test_id + order by ctest1.id, ctest2.label + ) x group by ctest1;""") + + r = curs.fetchone() + self.assertEqual(r[0], (1, 'test1')) + self.assertEqual(r[1], [(1, 'testa', 1), (2, 'testb', 1)]) + r = curs.fetchone() + self.assertEqual(r[0], (2, 'test2')) + self.assertEqual(r[1], [(3, 'testc', 2), (4, 'testd', 2)]) + def _create_type(self, name, fields): curs = self.conn.cursor() try: