diff --git a/ChangeLog b/ChangeLog index b12c675f..57b803b6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,11 +1,27 @@ 2010-12-18 Daniele Varrazzo + * connection.h: added codec attribute to avoid repeated codec name + lookups during unicode query/params manipulations. + * setup.py: bumped to version 2.3.2.dev0 * psycopg/connection_int.c: applied patch from Marti Raudsepp to close ticket #24. Fixed segfault in connection when DateStyle not available (e.g. pgbouncer appars not passing it to the client) +2010-12-15 Daniele Varrazzo + + * psycopg/utils.c: Added psycopg_strdup function. + +2010-12-14 Daniele Varrazzo + + * psycopg/connection_type.c: No need to put connection fields to zero. + + * lib/extensions.py: Improved mapping from PG to Py encodings. + + * psycopg/psycopgmodule.c: Added a few missing encodings: EUC_CN, + EUC_JIS_2004, ISO885910, ISO885916, LATIN10, SHIFT_JIS_2004. + 2010-12-04 Daniele Varrazzo * setup.py: bumped to version 2.3.1.dev0 diff --git a/Makefile b/Makefile index f4a5ecee..858a0b10 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ # make check # this requires setting up a test database with the correct user PYTHON := python$(PYTHON_VERSION) -PYTHON_VERSION ?= $(shell $(PYTHON) -c 'import sys; print "%d.%d" % sys.version_info[:2]') +PYTHON_VERSION ?= $(shell $(PYTHON) -c 'import sys; print ("%d.%d" % sys.version_info[:2])') BUILD_DIR = $(shell pwd)/build/lib.$(PYTHON_VERSION) ENV_DIR = $(shell pwd)/env/py-$(PYTHON_VERSION) ENV_BIN = $(ENV_DIR)/bin @@ -50,11 +50,11 @@ SDIST := dist/psycopg2-$(VERSION).tar.gz EASY_INSTALL = PYTHONPATH=$(ENV_LIB) $(ENV_BIN)/easy_install-$(PYTHON_VERSION) -d $(ENV_LIB) -s $(ENV_BIN) EZ_SETUP = $(ENV_BIN)/ez_setup.py -.PHONY: env check runtests clean +.PHONY: env check clean default: package -all: package runtests sdist +all: package sdist package: $(PLATLIB) $(PURELIB) @@ -87,7 +87,7 @@ ez_setup: wget -O $(EZ_SETUP) http://peak.telecommunity.com/dist/ez_setup.py check: - PYTHONPATH=$(BUILD_DIR):.:$(PYTHONPATH) $(PYTHON) tests/__init__.py --verbose + PYTHONPATH=$(BUILD_DIR):$(PYTHONPATH) $(PYTHON) -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" --verbose testdb: @echo "* Creating $(TESTDB)" diff --git a/NEWS-2.3 b/NEWS-2.3 index 5e168f19..40d049a2 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -8,6 +8,16 @@ What's new in psycopg 2.3.3 - More efficient iteration on named cursors. - The build script refuses to guess values if pg_config is not found. - Connections and cursors are weakly referenceable. + - Added 'b' and 't' mode to large objects: write can deal with both bytes + strings and unicode; read can return either bytes strings or decoded + unicode. + - COPY sends Unicode data to files implementing io.TextIOBase. + - The build script refuses to guess values if pg_config is not found. + - Improved PostgreSQL-Python encodings mapping. Added a few + missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, + LATIN10, SHIFT_JIS_2004. + - Dropped repeated dictionary lookups with unicode query/parameters. + - Empty lists correctly roundtrip Python -> PostgreSQL -> Python. * Bug fixes: diff --git a/doc/src/connection.rst b/doc/src/connection.rst index bf6a6f26..d0f5e1a9 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -490,13 +490,14 @@ The ``connection`` class .. method:: lobject([oid [, mode [, new_oid [, new_file [, lobject_factory]]]]]) - Return a new database large object. See :ref:`large-objects` for an - overview. + Return a new database large object as a `~psycopg2.extensions.lobject` + instance. + + See :ref:`large-objects` for an overview. :param oid: The OID of the object to read or write. 0 to create a new large object and and have its OID assigned automatically. - :param mode: Access mode to the object: can be ``r``, ``w``, - ``rw`` or ``n`` (meaning don't open it). + :param mode: Access mode to the object, see below. :param new_oid: Create a new object using the specified OID. The function raises `OperationalError` if the OID is already in use. Default is 0, meaning assign a new one automatically. @@ -504,13 +505,31 @@ The ``connection`` class (using the |lo_import|_ function) :param lobject_factory: Subclass of `~psycopg2.extensions.lobject` to be instantiated. - :rtype: `~psycopg2.extensions.lobject` .. |lo_import| replace:: `!lo_import()` .. _lo_import: http://www.postgresql.org/docs/9.0/static/lo-interfaces.html#LO-IMPORT + Available values for *mode* are: + + ======= ========= + *mode* meaning + ======= ========= + ``r`` Open for read only + ``w`` Open for write only + ``rw`` Open for read/write + ``n`` Don't open the file + ``b`` Don't decode read data (return data as `str` in Python 2 or `bytes` in Python 3) + ``t`` Decode read data according to `connection.encoding` (return data as `unicode` in Python 2 or `str` in Python 3) + ======= ========= + + ``b`` and ``t`` can be specified together with a read/write mode. If + neither ``b`` nor ``t`` is specified, the default is ``b`` in Python 2 + and ``t`` in Python 3. + .. versionadded:: 2.0.8 + .. versionchanged:: 2.3.3 added ``b`` and ``t`` mode and unicode + support. .. rubric:: Methods related to asynchronous support. diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index 73b05db1..8fee890e 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -51,17 +51,29 @@ functionalities defined by the |DBAPI|_. .. attribute:: mode - The mode the database was open (``r``, ``w``, ``rw`` or ``n``). + The mode the database was open. See `connection.lobject()` for a + description of the available modes. .. method:: read(bytes=-1) Read a chunk of data from the current file position. If -1 (default) read all the remaining data. + The result is an Unicode string (decoded according to + `connection.encoding`) if the file was open in ``t`` mode, a bytes + string for ``b`` mode. + + .. versionchanged:: 2.3.3 + added Unicode support. + .. method:: write(str) Write a string to the large object. Return the number of bytes - written. + written. Unicode strings are encoded in the `connection.encoding` + before writing. + + .. versionchanged:: 2.3.3 + added Unicode support. .. method:: export(file_name) diff --git a/doc/src/usage.rst b/doc/src/usage.rst index a5efaa45..36bd36be 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -574,7 +574,8 @@ whole. Psycopg allows access to the large object using the `~psycopg2.extensions.lobject` class. Objects are generated using the -`connection.lobject()` factory method. +`connection.lobject()` factory method. Data can be retrieved either as bytes +or as Unicode strings. Psycopg large object support efficient import/export with file system files using the |lo_import|_ and |lo_export|_ libpq functions. diff --git a/lib/__init__.py b/lib/__init__.py index 065b2a70..4b9e22a2 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -66,17 +66,17 @@ from psycopg2 import tz # Import the DBAPI-2.0 stuff into top-level module. -from _psycopg import BINARY, NUMBER, STRING, DATETIME, ROWID +from psycopg2._psycopg import BINARY, NUMBER, STRING, DATETIME, ROWID -from _psycopg import Binary, Date, Time, Timestamp -from _psycopg import DateFromTicks, TimeFromTicks, TimestampFromTicks +from psycopg2._psycopg import Binary, Date, Time, Timestamp +from psycopg2._psycopg import DateFromTicks, TimeFromTicks, TimestampFromTicks -from _psycopg import Error, Warning, DataError, DatabaseError, ProgrammingError -from _psycopg import IntegrityError, InterfaceError, InternalError -from _psycopg import NotSupportedError, OperationalError +from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, ProgrammingError +from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError +from psycopg2._psycopg import NotSupportedError, OperationalError -from _psycopg import connect, apilevel, threadsafety, paramstyle -from _psycopg import __version__ +from psycopg2._psycopg import connect, apilevel, threadsafety, paramstyle +from psycopg2._psycopg import __version__ # Register default adapters. diff --git a/lib/extensions.py b/lib/extensions.py index 24a9a6b8..82e17fa4 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -32,38 +32,38 @@ This module holds all the extensions to the DBAPI-2.0 provided by psycopg. # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. -from _psycopg import UNICODE, INTEGER, LONGINTEGER, BOOLEAN, FLOAT -from _psycopg import TIME, DATE, INTERVAL, DECIMAL -from _psycopg import BINARYARRAY, BOOLEANARRAY, DATEARRAY, DATETIMEARRAY -from _psycopg import DECIMALARRAY, FLOATARRAY, INTEGERARRAY, INTERVALARRAY -from _psycopg import LONGINTEGERARRAY, ROWIDARRAY, STRINGARRAY, TIMEARRAY -from _psycopg import UNICODEARRAY +from psycopg2._psycopg import UNICODE, INTEGER, LONGINTEGER, BOOLEAN, FLOAT +from psycopg2._psycopg import TIME, DATE, INTERVAL, DECIMAL +from psycopg2._psycopg import BINARYARRAY, BOOLEANARRAY, DATEARRAY, DATETIMEARRAY +from psycopg2._psycopg import DECIMALARRAY, FLOATARRAY, INTEGERARRAY, INTERVALARRAY +from psycopg2._psycopg import LONGINTEGERARRAY, ROWIDARRAY, STRINGARRAY, TIMEARRAY +from psycopg2._psycopg import UNICODEARRAY -from _psycopg import Binary, Boolean, Float, QuotedString, AsIs +from psycopg2._psycopg import Binary, Boolean, Float, QuotedString, AsIs try: - from _psycopg import MXDATE, MXDATETIME, MXINTERVAL, MXTIME - from _psycopg import MXDATEARRAY, MXDATETIMEARRAY, MXINTERVALARRAY, MXTIMEARRAY - from _psycopg import DateFromMx, TimeFromMx, TimestampFromMx - from _psycopg import IntervalFromMx -except: + from psycopg2._psycopg import MXDATE, MXDATETIME, MXINTERVAL, MXTIME + from psycopg2._psycopg import MXDATEARRAY, MXDATETIMEARRAY, MXINTERVALARRAY, MXTIMEARRAY + from psycopg2._psycopg import DateFromMx, TimeFromMx, TimestampFromMx + from psycopg2._psycopg import IntervalFromMx +except ImportError: pass try: - from _psycopg import PYDATE, PYDATETIME, PYINTERVAL, PYTIME - from _psycopg import PYDATEARRAY, PYDATETIMEARRAY, PYINTERVALARRAY, PYTIMEARRAY - from _psycopg import DateFromPy, TimeFromPy, TimestampFromPy - from _psycopg import IntervalFromPy -except: + from psycopg2._psycopg import PYDATE, PYDATETIME, PYINTERVAL, PYTIME + from psycopg2._psycopg import PYDATEARRAY, PYDATETIMEARRAY, PYINTERVALARRAY, PYTIMEARRAY + from psycopg2._psycopg import DateFromPy, TimeFromPy, TimestampFromPy + from psycopg2._psycopg import IntervalFromPy +except ImportError: pass -from _psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid -from _psycopg import string_types, binary_types, new_type, register_type -from _psycopg import ISQLQuote, Notify +from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid +from psycopg2._psycopg import string_types, binary_types, new_type, register_type +from psycopg2._psycopg import ISQLQuote, Notify -from _psycopg import QueryCanceledError, TransactionRollbackError +from psycopg2._psycopg import QueryCanceledError, TransactionRollbackError try: - from _psycopg import set_wait_callback, get_wait_callback + from psycopg2._psycopg import set_wait_callback, get_wait_callback except ImportError: pass @@ -100,6 +100,16 @@ TRANSACTION_STATUS_INTRANS = 2 TRANSACTION_STATUS_INERROR = 3 TRANSACTION_STATUS_UNKNOWN = 4 +import sys as _sys + +# Return bytes from a string +if _sys.version_info[0] < 3: + def b(s): + return s +else: + def b(s): + return s.encode('utf8') + def register_adapter(typ, callable): """Register 'callable' as an ISQLQuote adapter for type 'typ'.""" adapters[(typ, ISQLQuote)] = callable @@ -122,10 +132,11 @@ class SQL_IN(object): for obj in pobjs: if hasattr(obj, 'prepare'): obj.prepare(self._conn) - qobjs = [str(o.getquoted()) for o in pobjs] - return '(' + ', '.join(qobjs) + ')' + qobjs = [o.getquoted() for o in pobjs] + return b('(') + b(', ').join(qobjs) + b(')') - __str__ = getquoted + def __str__(self): + return str(self.getquoted()) class NoneAdapter(object): @@ -137,8 +148,17 @@ class NoneAdapter(object): def __init__(self, obj): pass - def getquoted(self): - return "NULL" + def getquoted(self, _null=b("NULL")): + return _null + + +# Add the "cleaned" version of the encodings to the key. +# When the encoding is set its name is cleaned up from - and _ and turned +# uppercase, so an encoding not respecting these rules wouldn't be found in the +# encodings keys and would raise an exception with the unicode typecaster +for k, v in encodings.items(): + k = k.replace('_', '').replace('-', '').upper() + encodings[k] = v __all__ = filter(lambda k: not k.startswith('_'), locals().keys()) diff --git a/lib/extras.py b/lib/extras.py index 2a36c6b0..69da5264 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -26,6 +26,7 @@ and classes untill a better place in the distribution is found. # License for more details. import os +import sys import time import codecs import warnings @@ -41,13 +42,14 @@ from psycopg2 import extensions as _ext from psycopg2.extensions import cursor as _cursor from psycopg2.extensions import connection as _connection from psycopg2.extensions import adapt as _A +from psycopg2.extensions import b class DictCursorBase(_cursor): """Base class for all dict-like cursors.""" def __init__(self, *args, **kwargs): - if kwargs.has_key('row_factory'): + if 'row_factory' in kwargs: row_factory = kwargs['row_factory'] del kwargs['row_factory'] else: @@ -140,20 +142,17 @@ class DictRow(list): self[:] = [None] * len(cursor.description) def __getitem__(self, x): - if type(x) != int: + if not isinstance(x, (int, slice)): x = self._index[x] return list.__getitem__(self, x) def __setitem__(self, x, v): - if type(x) != int: + if not isinstance(x, (int, slice)): x = self._index[x] list.__setitem__(self, x, v) def items(self): - res = [] - for n, v in self._index.items(): - res.append((n, list.__getitem__(self, v))) - return res + return list(self.iteritems()) def keys(self): return self._index.keys() @@ -162,7 +161,7 @@ class DictRow(list): return tuple(self[:]) def has_key(self, x): - return self._index.has_key(x) + return x in self._index def get(self, x, default=None): try: @@ -171,7 +170,7 @@ class DictRow(list): return default def iteritems(self): - for n, v in self._index.items(): + for n, v in self._index.iteritems(): yield n, list.__getitem__(self, v) def iterkeys(self): @@ -181,10 +180,18 @@ class DictRow(list): return list.__iter__(self) def copy(self): - return dict(self.items()) + return dict(self.iteritems()) def __contains__(self, x): - return self._index.__contains__(x) + return x in self._index + + # grop the crusty Py2 methods + if sys.version_info[0] > 2: + items = iteritems; del iteritems + keys = iterkeys; del iterkeys + values = itervalues; del itervalues + del has_key + class RealDictConnection(_connection): """A connection that uses `RealDictCursor` automatically.""" @@ -501,7 +508,7 @@ class Inet(object): obj = _A(self.addr) if hasattr(obj, 'prepare'): obj.prepare(self._conn) - return obj.getquoted()+"::inet" + return obj.getquoted() + b("::inet") def __conform__(self, foo): if foo is _ext.ISQLQuote: @@ -568,7 +575,7 @@ class HstoreAdapter(object): def _getquoted_8(self): """Use the operators available in PG pre-9.0.""" if not self.wrapped: - return "''::hstore" + return b("''::hstore") adapt = _ext.adapt rv = [] @@ -582,22 +589,23 @@ class HstoreAdapter(object): v.prepare(self.conn) v = v.getquoted() else: - v = 'NULL' + v = b('NULL') - rv.append("(%s => %s)" % (k, v)) + # XXX this b'ing is painfully inefficient! + rv.append(b("(") + k + b(" => ") + v + b(")")) - return "(" + '||'.join(rv) + ")" + return b("(") + b('||').join(rv) + b(")") def _getquoted_9(self): """Use the hstore(text[], text[]) function.""" if not self.wrapped: - return "''::hstore" + return b("''::hstore") k = _ext.adapt(self.wrapped.keys()) k.prepare(self.conn) v = _ext.adapt(self.wrapped.values()) v.prepare(self.conn) - return "hstore(%s, %s)" % (k.getquoted(), v.getquoted()) + return b("hstore(") + k.getquoted() + b(", ") + v.getquoted() + b(")") getquoted = _getquoted_9 @@ -614,10 +622,8 @@ class HstoreAdapter(object): (?:\s*,\s*|$) # pairs separated by comma or end of string. """, regex.VERBOSE) - # backslash decoder - _bsdec = codecs.getdecoder("string_escape") - - def parse(self, s, cur, _decoder=_bsdec): + @classmethod + def parse(self, s, cur, _bsdec=regex.compile(r"\\(.)")): """Parse an hstore representation in a Python string. The hstore is represented as something like:: @@ -635,10 +641,10 @@ class HstoreAdapter(object): if m is None or m.start() != start: raise psycopg2.InterfaceError( "error parsing hstore pair at char %d" % start) - k = _decoder(m.group(1))[0] + k = _bsdec.sub(r'\1', m.group(1)) v = m.group(2) if v is not None: - v = _decoder(v)[0] + v = _bsdec.sub(r'\1', v) rv[k] = v start = m.end() @@ -649,16 +655,14 @@ class HstoreAdapter(object): return rv - parse = classmethod(parse) - + @classmethod def parse_unicode(self, s, cur): """Parse an hstore returning unicode keys and values.""" - codec = codecs.getdecoder(_ext.encodings[cur.connection.encoding]) - bsdec = self._bsdec - decoder = lambda s: codec(bsdec(s)[0]) - return self.parse(s, cur, _decoder=decoder) + if s is None: + return None - parse_unicode = classmethod(parse_unicode) + s = s.decode(_ext.encodings[cur.connection.encoding]) + return self.parse(s, cur) @classmethod def get_oids(self, conn_or_curs): @@ -704,11 +708,11 @@ def register_hstore(conn_or_curs, globally=False, unicode=False): uses a single database you can pass *globally*\=True to have the typecaster registered on all the connections. - By default the returned dicts will have `str` objects as keys and values: + On Python 2, by default the returned dicts will have `str` objects as keys and values: use *unicode*\=True to return `unicode` objects instead. When adapting a dictionary both `str` and `unicode` keys and values are handled (the `unicode` values will be converted according to the current - `~connection.encoding`). + `~connection.encoding`). The option is not available on Python 3. The |hstore| contrib module must be already installed in the database (executing the ``hstore.sql`` script in your ``contrib`` directory). @@ -721,7 +725,7 @@ def register_hstore(conn_or_curs, globally=False, unicode=False): "please install it from your 'contrib/hstore.sql' file") # create and register the typecaster - if unicode: + if sys.version_info[0] < 3 and unicode: cast = HstoreAdapter.parse_unicode else: cast = HstoreAdapter.parse diff --git a/lib/pool.py b/lib/pool.py index ab517b6a..8a8fa539 100644 --- a/lib/pool.py +++ b/lib/pool.py @@ -100,7 +100,7 @@ class AbstractConnectionPool(object): if self.closed: raise PoolError("connection pool is closed") if key is None: key = self._getkey() - if self._used.has_key(key): + if key in self._used: return self._used[key] if self._pool: diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index 31cfabbe..bc64aa74 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -22,37 +22,44 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public * License for more details. */ - -#define PY_SSIZE_T_CLEAN -#include -#include -#include -#include #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/adapter_asis.h" #include "psycopg/microprotocols_proto.h" +#include + + /** the AsIs object **/ static PyObject * -asis_str(asisObject *self) +asis_getquoted(asisObject *self, PyObject *args) { + PyObject *rv; if (self->wrapped == Py_None) { - return PyString_FromString("NULL"); + rv = Bytes_FromString("NULL"); } else { - return PyObject_Str(self->wrapped); + rv = PyObject_Str(self->wrapped); +#if PY_MAJOR_VERSION > 2 + /* unicode to bytes in Py3 */ + if (rv) { + PyObject *tmp = PyUnicode_AsUTF8String(rv); + Py_DECREF(rv); + rv = tmp; + } +#endif } + + return rv; } static PyObject * -asis_getquoted(asisObject *self, PyObject *args) +asis_str(asisObject *self) { - return asis_str(self); + return psycopg_ensure_text(asis_getquoted(self, NULL)); } static PyObject * @@ -76,7 +83,7 @@ asis_conform(asisObject *self, PyObject *args) /* object member list */ static struct PyMemberDef asisObject_members[] = { - {"adapted", T_OBJECT, offsetof(asisObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(asisObject, wrapped), READONLY}, {NULL} }; @@ -96,7 +103,7 @@ asis_setup(asisObject *self, PyObject *obj) { Dprintf("asis_setup: init asis object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -104,7 +111,7 @@ asis_setup(asisObject *self, PyObject *obj) Dprintf("asis_setup: good asis object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -125,10 +132,10 @@ asis_dealloc(PyObject* obj) Dprintf("asis_dealloc: deleted asis object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -167,8 +174,7 @@ asis_repr(asisObject *self) "AsIs(str) -> new AsIs adapter object" PyTypeObject asisType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.AsIs", sizeof(asisObject), 0, diff --git a/psycopg/adapter_asis.h b/psycopg/adapter_asis.h index 28992a6e..04ba08bc 100644 --- a/psycopg/adapter_asis.h +++ b/psycopg/adapter_asis.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_ASIS_H #define PSYCOPG_ASIS_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 9b4a1c38..ccfaf240 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -23,21 +23,15 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" -#include "psycopg/connection.h" + #include "psycopg/adapter_binary.h" #include "psycopg/microprotocols_proto.h" +#include "psycopg/connection.h" + +#include + /** the quoting code */ @@ -64,7 +58,14 @@ binary_quote(binaryObject *self) size_t len = 0; /* if we got a plain string or a buffer we escape it and save the buffer */ - if (PyString_Check(self->wrapped) || PyBuffer_Check(self->wrapped)) { + if (Bytes_Check(self->wrapped) +#if PY_MAJOR_VERSION < 3 + || PyBuffer_Check(self->wrapped) +#else + || PyByteArray_Check(self->wrapped) + || PyMemoryView_Check(self->wrapped) +#endif + ) { /* escape and build quoted buffer */ if (PyObject_AsReadBuffer(self->wrapped, (const void **)&buffer, &buffer_len) < 0) @@ -78,11 +79,11 @@ binary_quote(binaryObject *self) } if (len > 0) - self->buffer = PyString_FromFormat( + self->buffer = Bytes_FromFormat( (self->conn && ((connectionObject*)self->conn)->equote) ? "E'%s'::bytea" : "'%s'::bytea" , to); else - self->buffer = PyString_FromString("''::bytea"); + self->buffer = Bytes_FromString("''::bytea"); PQfreemem(to); } @@ -99,19 +100,21 @@ binary_quote(binaryObject *self) /* binary_str, binary_getquoted - return result of quoting */ static PyObject * -binary_str(binaryObject *self) +binary_getquoted(binaryObject *self, PyObject *args) { if (self->buffer == NULL) { - binary_quote(self); + if (!(binary_quote(self))) { + return NULL; + } } - Py_XINCREF(self->buffer); + Py_INCREF(self->buffer); return self->buffer; } static PyObject * -binary_getquoted(binaryObject *self, PyObject *args) +binary_str(binaryObject *self) { - return binary_str(self); + return psycopg_ensure_text(binary_getquoted(self, NULL)); } static PyObject * @@ -153,8 +156,8 @@ binary_conform(binaryObject *self, PyObject *args) /* object member list */ static struct PyMemberDef binaryObject_members[] = { - {"adapted", T_OBJECT, offsetof(binaryObject, wrapped), RO}, - {"buffer", T_OBJECT, offsetof(binaryObject, buffer), RO}, + {"adapted", T_OBJECT, offsetof(binaryObject, wrapped), READONLY}, + {"buffer", T_OBJECT, offsetof(binaryObject, buffer), READONLY}, {NULL} }; @@ -176,7 +179,7 @@ binary_setup(binaryObject *self, PyObject *str) { Dprintf("binary_setup: init binary object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); self->buffer = NULL; @@ -186,7 +189,7 @@ binary_setup(binaryObject *self, PyObject *str) Dprintf("binary_setup: good binary object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt); + self, Py_REFCNT(self)); return 0; } @@ -212,10 +215,10 @@ binary_dealloc(PyObject* obj) Dprintf("binary_dealloc: deleted binary object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -253,8 +256,7 @@ binary_repr(binaryObject *self) "Binary(buffer) -> new binary object" PyTypeObject binaryType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Binary", sizeof(binaryObject), 0, diff --git a/psycopg/adapter_binary.h b/psycopg/adapter_binary.h index 37aec702..899f5ce5 100644 --- a/psycopg/adapter_binary.h +++ b/psycopg/adapter_binary.h @@ -26,12 +26,6 @@ #ifndef PSYCOPG_BINARY_H #define PSYCOPG_BINARY_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index fe7acc8e..5850be8f 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -23,21 +23,17 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/adapter_datetime.h" +#include "psycopg/microprotocols_proto.h" + #include #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_datetime.h" -#include "psycopg/microprotocols_proto.h" extern HIDDEN PyObject *pyPsycopgTzModule; extern HIDDEN PyObject *pyPsycopgTzLOCAL; @@ -59,59 +55,78 @@ psyco_adapter_datetime_init(void) /* datetime_str, datetime_getquoted - return result of quoting */ static PyObject * -pydatetime_str(pydatetimeObject *self) +_pydatetime_string_date_time(pydatetimeObject *self) { - PyObject *res = NULL; - PyObject *iso; - if (self->type <= PSYCO_DATETIME_TIMESTAMP) { - PyObject *tz; + PyObject *rv = NULL; + PyObject *iso = NULL; + PyObject *tz; - /* Select the right PG type to cast into. */ - char *fmt = NULL; - switch (self->type) { - case PSYCO_DATETIME_TIME: - fmt = "'%s'::time"; - break; - case PSYCO_DATETIME_DATE: - fmt = "'%s'::date"; - break; - case PSYCO_DATETIME_TIMESTAMP: - tz = PyObject_GetAttrString(self->wrapped, "tzinfo"); - if (!tz) { return NULL; } - fmt = (tz == Py_None) ? "'%s'::timestamp" : "'%s'::timestamptz"; - Py_DECREF(tz); - break; - } - - iso = PyObject_CallMethod(self->wrapped, "isoformat", NULL); - if (iso) { - res = PyString_FromFormat(fmt, PyString_AsString(iso)); - Py_DECREF(iso); - } - return res; + /* Select the right PG type to cast into. */ + char *fmt = NULL; + switch (self->type) { + case PSYCO_DATETIME_TIME: + fmt = "'%s'::time"; + break; + case PSYCO_DATETIME_DATE: + fmt = "'%s'::date"; + break; + case PSYCO_DATETIME_TIMESTAMP: + tz = PyObject_GetAttrString(self->wrapped, "tzinfo"); + if (!tz) { goto error; } + fmt = (tz == Py_None) ? "'%s'::timestamp" : "'%s'::timestamptz"; + Py_DECREF(tz); + break; } - else { - PyDateTime_Delta *obj = (PyDateTime_Delta*)self->wrapped; - char buffer[8]; - int i; - int a = obj->microseconds; - - for (i=0; i < 6 ; i++) { - buffer[5-i] = '0' + (a % 10); - a /= 10; - } - buffer[6] = '\0'; - - return PyString_FromFormat("'%d days %d.%s seconds'::interval", - obj->days, obj->seconds, buffer); + if (!(iso = psycopg_ensure_bytes( + PyObject_CallMethod(self->wrapped, "isoformat", NULL)))) { + goto error; } + + rv = Bytes_FromFormat(fmt, Bytes_AsString(iso)); + + Py_DECREF(iso); + return rv; + +error: + Py_XDECREF(iso); + return rv; +} + +static PyObject * +_pydatetime_string_delta(pydatetimeObject *self) +{ + PyDateTime_Delta *obj = (PyDateTime_Delta*)self->wrapped; + + char buffer[8]; + int i; + int a = obj->microseconds; + + for (i=0; i < 6 ; i++) { + buffer[5-i] = '0' + (a % 10); + a /= 10; + } + buffer[6] = '\0'; + + return Bytes_FromFormat("'%d days %d.%s seconds'::interval", + obj->days, obj->seconds, buffer); } static PyObject * pydatetime_getquoted(pydatetimeObject *self, PyObject *args) { - return pydatetime_str(self); + if (self->type <= PSYCO_DATETIME_TIMESTAMP) { + return _pydatetime_string_date_time(self); + } + else { + return _pydatetime_string_delta(self); + } +} + +static PyObject * +pydatetime_str(pydatetimeObject *self) +{ + return psycopg_ensure_text(pydatetime_getquoted(self, NULL)); } static PyObject * @@ -135,8 +150,8 @@ pydatetime_conform(pydatetimeObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pydatetimeObject_members[] = { - {"adapted", T_OBJECT, offsetof(pydatetimeObject, wrapped), RO}, - {"type", T_INT, offsetof(pydatetimeObject, type), RO}, + {"adapted", T_OBJECT, offsetof(pydatetimeObject, wrapped), READONLY}, + {"type", T_INT, offsetof(pydatetimeObject, type), READONLY}, {NULL} }; @@ -156,7 +171,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type) { Dprintf("pydatetime_setup: init datetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt); + self, Py_REFCNT(self)); self->type = type; Py_INCREF(obj); @@ -164,7 +179,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type) Dprintf("pydatetime_setup: good pydatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt); + self, Py_REFCNT(self)); return 0; } @@ -185,9 +200,9 @@ pydatetime_dealloc(PyObject* obj) Py_CLEAR(self->wrapped); Dprintf("mpydatetime_dealloc: deleted pydatetime object at %p, " - "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, obj->ob_refcnt); + "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -227,8 +242,7 @@ pydatetime_repr(pydatetimeObject *self) "datetime(datetime, type) -> new datetime wrapper object" PyTypeObject pydatetimeType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.datetime", sizeof(pydatetimeObject), 0, diff --git a/psycopg/adapter_datetime.h b/psycopg/adapter_datetime.h index a2c36fb9..dd59e9b5 100644 --- a/psycopg/adapter_datetime.h +++ b/psycopg/adapter_datetime.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_DATETIME_H #define PSYCOPG_DATETIME_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index f9ab509a..b7e1f967 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -23,15 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/adapter_list.h" #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" @@ -51,7 +45,7 @@ list_quote(listObject *self) /* empty arrays are converted to NULLs (still searching for a way to insert an empty array in postgresql */ - if (len == 0) return PyString_FromString("'{}'"); + if (len == 0) return Bytes_FromString("'{}'::text[]"); tmp = PyTuple_New(len); @@ -59,7 +53,7 @@ list_quote(listObject *self) PyObject *quoted; PyObject *wrapped = PyList_GET_ITEM(self->wrapped, i); if (wrapped == Py_None) - quoted = PyString_FromString("NULL"); + quoted = Bytes_FromString("NULL"); else quoted = microprotocol_getquoted(wrapped, (connectionObject*)self->connection); @@ -73,11 +67,11 @@ list_quote(listObject *self) /* now that we have a tuple of adapted objects we just need to join them and put "ARRAY[] around the result */ - str = PyString_FromString(", "); + str = Bytes_FromString(", "); joined = PyObject_CallMethod(str, "join", "(O)", tmp); if (joined == NULL) goto error; - res = PyString_FromFormat("ARRAY[%s]", PyString_AsString(joined)); + res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined)); error: Py_XDECREF(tmp); @@ -89,7 +83,7 @@ list_quote(listObject *self) static PyObject * list_str(listObject *self) { - return list_quote(self); + return psycopg_ensure_text(list_quote(self)); } static PyObject * @@ -139,7 +133,7 @@ list_conform(listObject *self, PyObject *args) /* object member list */ static struct PyMemberDef listObject_members[] = { - {"adapted", T_OBJECT, offsetof(listObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(listObject, wrapped), READONLY}, {NULL} }; @@ -161,7 +155,7 @@ list_setup(listObject *self, PyObject *obj, const char *enc) { Dprintf("list_setup: init list object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); if (!PyList_Check(obj)) @@ -176,7 +170,7 @@ list_setup(listObject *self, PyObject *obj, const char *enc) Dprintf("list_setup: good list object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -201,9 +195,9 @@ list_dealloc(PyObject* obj) if (self->encoding) free(self->encoding); Dprintf("list_dealloc: deleted list object at %p, " - "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, obj->ob_refcnt); + "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -242,8 +236,7 @@ list_repr(listObject *self) "List(list) -> new list wrapper object" PyTypeObject listType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.List", sizeof(listObject), 0, diff --git a/psycopg/adapter_list.h b/psycopg/adapter_list.h index a368e1a0..d7136103 100644 --- a/psycopg/adapter_list.h +++ b/psycopg/adapter_list.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_LIST_H #define PSYCOPG_LIST_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index c87d8a66..793dfba2 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -23,19 +23,16 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +/* TODO: check if still compiles ok: I have no mx on this box */ +#include "psycopg/adapter_mxdatetime.h" +#include "psycopg/microprotocols_proto.h" + #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_mxdatetime.h" -#include "psycopg/microprotocols_proto.h" int psyco_adapter_mxdatetime_init(void) @@ -138,8 +135,8 @@ mxdatetime_conform(mxdatetimeObject *self, PyObject *args) /* object member list */ static struct PyMemberDef mxdatetimeObject_members[] = { - {"adapted", T_OBJECT, offsetof(mxdatetimeObject, wrapped), RO}, - {"type", T_INT, offsetof(mxdatetimeObject, type), RO}, + {"adapted", T_OBJECT, offsetof(mxdatetimeObject, wrapped), READONLY}, + {"type", T_INT, offsetof(mxdatetimeObject, type), READONLY}, {NULL} }; @@ -159,7 +156,7 @@ mxdatetime_setup(mxdatetimeObject *self, PyObject *obj, int type) { Dprintf("mxdatetime_setup: init mxdatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); self->type = type; @@ -168,7 +165,7 @@ mxdatetime_setup(mxdatetimeObject *self, PyObject *obj, int type) Dprintf("mxdatetime_setup: good mxdatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -191,10 +188,10 @@ mxdatetime_dealloc(PyObject* obj) Dprintf("mxdatetime_dealloc: deleted mxdatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -234,8 +231,7 @@ mxdatetime_repr(mxdatetimeObject *self) "MxDateTime(mx, type) -> new mx.DateTime wrapper object" PyTypeObject mxdatetimeType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.MxDateTime", sizeof(mxdatetimeObject), 0, diff --git a/psycopg/adapter_mxdatetime.h b/psycopg/adapter_mxdatetime.h index 5fdeb960..bd9d4c9d 100644 --- a/psycopg/adapter_mxdatetime.h +++ b/psycopg/adapter_mxdatetime.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_MXDATETIME_H #define PSYCOPG_MXDATETIME_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index edffe8f5..4e2c4464 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -23,46 +23,41 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/adapter_pboolean.h" #include "psycopg/microprotocols_proto.h" +#include + /** the Boolean object **/ static PyObject * -pboolean_str(pbooleanObject *self) +pboolean_getquoted(pbooleanObject *self, PyObject *args) { #ifdef PSYCOPG_NEW_BOOLEAN if (PyObject_IsTrue(self->wrapped)) { - return PyString_FromString("true"); + return Bytes_FromString("true"); } else { - return PyString_FromString("false"); + return Bytes_FromString("false"); } #else if (PyObject_IsTrue(self->wrapped)) { - return PyString_FromString("'t'"); + return Bytes_FromString("'t'"); } else { - return PyString_FromString("'f'"); + return Bytes_FromString("'f'"); } #endif } static PyObject * -pboolean_getquoted(pbooleanObject *self, PyObject *args) +pboolean_str(pbooleanObject *self) { - return pboolean_str(self); + return psycopg_ensure_text(pboolean_getquoted(self, NULL)); } static PyObject * @@ -86,7 +81,7 @@ pboolean_conform(pbooleanObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pbooleanObject_members[] = { - {"adapted", T_OBJECT, offsetof(pbooleanObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(pbooleanObject, wrapped), READONLY}, {NULL} }; @@ -106,7 +101,7 @@ pboolean_setup(pbooleanObject *self, PyObject *obj) { Dprintf("pboolean_setup: init pboolean object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -114,7 +109,7 @@ pboolean_setup(pbooleanObject *self, PyObject *obj) Dprintf("pboolean_setup: good pboolean object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -137,10 +132,10 @@ pboolean_dealloc(PyObject* obj) Dprintf("pboolean_dealloc: deleted pboolean object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -180,8 +175,7 @@ pboolean_repr(pbooleanObject *self) "Boolean(str) -> new Boolean adapter object" PyTypeObject pbooleanType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Boolean", sizeof(pbooleanObject), 0, diff --git a/psycopg/adapter_pboolean.h b/psycopg/adapter_pboolean.h index de5cd9f3..b1bb18ae 100644 --- a/psycopg/adapter_pboolean.h +++ b/psycopg/adapter_pboolean.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_PBOOLEAN_H #define PSYCOPG_PBOOLEAN_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index 347461af..9b573467 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -23,24 +23,20 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/adapter_pdecimal.h" #include "psycopg/microprotocols_proto.h" +#include +#include + /** the Decimal object **/ static PyObject * -pdecimal_str(pdecimalObject *self) +pdecimal_getquoted(pdecimalObject *self, PyObject *args) { PyObject *check, *res = NULL; check = PyObject_CallMethod(self->wrapped, "is_finite", NULL); @@ -49,7 +45,7 @@ pdecimal_str(pdecimalObject *self) goto end; } else if (check) { - res = PyString_FromString("'NaN'::numeric"); + res = Bytes_FromString("'NaN'::numeric"); goto end; } @@ -61,7 +57,7 @@ pdecimal_str(pdecimalObject *self) goto end; } if (PyObject_IsTrue(check)) { - res = PyString_FromString("'NaN'::numeric"); + res = Bytes_FromString("'NaN'::numeric"); goto end; } @@ -70,11 +66,19 @@ pdecimal_str(pdecimalObject *self) goto end; } if (PyObject_IsTrue(check)) { - res = PyString_FromString("'NaN'::numeric"); + res = Bytes_FromString("'NaN'::numeric"); goto end; } res = PyObject_Str(self->wrapped); +#if PY_MAJOR_VERSION > 2 + /* unicode to bytes in Py3 */ + if (res) { + PyObject *tmp = PyUnicode_AsUTF8String(res); + Py_DECREF(res); + res = tmp; + } +#endif end: Py_XDECREF(check); @@ -82,9 +86,9 @@ end: } static PyObject * -pdecimal_getquoted(pdecimalObject *self, PyObject *args) +pdecimal_str(pdecimalObject *self) { - return pdecimal_str(self); + return psycopg_ensure_text(pdecimal_getquoted(self, NULL)); } static PyObject * @@ -108,7 +112,7 @@ pdecimal_conform(pdecimalObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pdecimalObject_members[] = { - {"adapted", T_OBJECT, offsetof(pdecimalObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(pdecimalObject, wrapped), READONLY}, {NULL} }; @@ -128,7 +132,7 @@ pdecimal_setup(pdecimalObject *self, PyObject *obj) { Dprintf("pdecimal_setup: init pdecimal object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -136,7 +140,7 @@ pdecimal_setup(pdecimalObject *self, PyObject *obj) Dprintf("pdecimal_setup: good pdecimal object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -159,10 +163,10 @@ pdecimal_dealloc(PyObject* obj) Dprintf("pdecimal_dealloc: deleted pdecimal object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -202,8 +206,7 @@ pdecimal_repr(pdecimalObject *self) "Decimal(str) -> new Decimal adapter object" PyTypeObject pdecimalType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Decimal", sizeof(pdecimalObject), 0, diff --git a/psycopg/adapter_pdecimal.h b/psycopg/adapter_pdecimal.h index 82825a23..4f89fad5 100644 --- a/psycopg/adapter_pdecimal.h +++ b/psycopg/adapter_pdecimal.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_PDECIMAL_H #define PSYCOPG_PDECIMAL_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 3e9f341b..2753737a 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -23,38 +23,47 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/adapter_pfloat.h" #include "psycopg/microprotocols_proto.h" +#include +#include + /** the Float object **/ static PyObject * -pfloat_str(pfloatObject *self) +pfloat_getquoted(pfloatObject *self, PyObject *args) { + PyObject *rv; double n = PyFloat_AsDouble(self->wrapped); if (isnan(n)) - return PyString_FromString("'NaN'::float"); + rv = Bytes_FromString("'NaN'::float"); else if (isinf(n)) - return PyString_FromString("'Infinity'::float"); - else - return PyObject_Repr(self->wrapped); + rv = Bytes_FromString("'Infinity'::float"); + else { + rv = PyObject_Repr(self->wrapped); + +#if PY_MAJOR_VERSION > 2 + /* unicode to bytes in Py3 */ + if (rv) { + PyObject *tmp = PyUnicode_AsUTF8String(rv); + Py_DECREF(rv); + rv = tmp; + } +#endif + } + + return rv; } static PyObject * -pfloat_getquoted(pfloatObject *self, PyObject *args) +pfloat_str(pfloatObject *self) { - return pfloat_str(self); + return psycopg_ensure_text(pfloat_getquoted(self, NULL)); } static PyObject * @@ -78,7 +87,7 @@ pfloat_conform(pfloatObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pfloatObject_members[] = { - {"adapted", T_OBJECT, offsetof(pfloatObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(pfloatObject, wrapped), READONLY}, {NULL} }; @@ -98,7 +107,7 @@ pfloat_setup(pfloatObject *self, PyObject *obj) { Dprintf("pfloat_setup: init pfloat object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -106,7 +115,7 @@ pfloat_setup(pfloatObject *self, PyObject *obj) Dprintf("pfloat_setup: good pfloat object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -129,10 +138,10 @@ pfloat_dealloc(PyObject* obj) Dprintf("pfloat_dealloc: deleted pfloat object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -172,8 +181,7 @@ pfloat_repr(pfloatObject *self) "Float(str) -> new Float adapter object" PyTypeObject pfloatType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Float", sizeof(pfloatObject), 0, diff --git a/psycopg/adapter_pfloat.h b/psycopg/adapter_pfloat.h index 12b40e04..7439c04f 100644 --- a/psycopg/adapter_pfloat.h +++ b/psycopg/adapter_pfloat.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_PFLOAT_H #define PSYCOPG_PFLOAT_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 59c1d7b2..a9d4ec6e 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -23,22 +23,15 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/adapter_qstring.h" #include "psycopg/microprotocols_proto.h" +#include + /* qstring_quote - do the quote process on plain and unicode strings */ @@ -56,24 +49,12 @@ qstring_quote(qstringObject *self) Dprintf("qstring_quote: encoding to %s", self->encoding); if (PyUnicode_Check(self->wrapped) && self->encoding) { - PyObject *enc = PyDict_GetItemString(psycoEncodings, self->encoding); - /* note that enc is a borrowed reference */ - - if (enc) { - const char *s = PyString_AsString(enc); - Dprintf("qstring_quote: encoding unicode object to %s", s); - str = PyUnicode_AsEncodedString(self->wrapped, s, NULL); - Dprintf("qstring_quote: got encoded object at %p", str); - if (str == NULL) return NULL; - } - else { - /* can't find the right encoder, raise exception */ - PyErr_Format(InterfaceError, - "can't encode unicode string to %s", self->encoding); - return NULL; - } + str = PyUnicode_AsEncodedString(self->wrapped, self->encoding, NULL); + Dprintf("qstring_quote: got encoded object at %p", str); + if (str == NULL) return NULL; } +#if PY_MAJOR_VERSION < 3 /* if the wrapped object is a simple string, we don't know how to (re)encode it, so we pass it as-is */ else if (PyString_Check(self->wrapped)) { @@ -81,6 +62,7 @@ qstring_quote(qstringObject *self) /* INCREF to make it ref-wise identical to unicode one */ Py_INCREF(str); } +#endif /* if the wrapped object is not a string, this is an error */ else { @@ -90,7 +72,7 @@ qstring_quote(qstringObject *self) } /* encode the string into buffer */ - PyString_AsStringAndSize(str, &s, &len); + Bytes_AsStringAndSize(str, &s, &len); /* Call qstring_escape with the GIL released, then reacquire the GIL before verifying that the results can fit into a Python string; raise @@ -113,8 +95,8 @@ qstring_quote(qstringObject *self) Py_DECREF(str); return NULL; } - - self->buffer = PyString_FromStringAndSize(buffer, qlen); + + self->buffer = Bytes_FromStringAndSize(buffer, qlen); PyMem_Free(buffer); Py_DECREF(str); @@ -124,7 +106,7 @@ qstring_quote(qstringObject *self) /* qstring_str, qstring_getquoted - return result of quoting */ static PyObject * -qstring_str(qstringObject *self) +qstring_getquoted(qstringObject *self, PyObject *args) { if (self->buffer == NULL) { qstring_quote(self); @@ -134,9 +116,9 @@ qstring_str(qstringObject *self) } static PyObject * -qstring_getquoted(qstringObject *self, PyObject *args) +qstring_str(qstringObject *self) { - return qstring_str(self); + return psycopg_ensure_text(qstring_getquoted(self, NULL)); } static PyObject * @@ -151,8 +133,8 @@ qstring_prepare(qstringObject *self, PyObject *args) we don't need the encoding if that's not the case */ if (PyUnicode_Check(self->wrapped)) { if (self->encoding) free(self->encoding); - self->encoding = strdup(conn->encoding); - Dprintf("qstring_prepare: set encoding to %s", conn->encoding); + self->encoding = strdup(conn->codec); + Dprintf("qstring_prepare: set encoding to %s", conn->codec); } Py_CLEAR(self->conn); @@ -186,9 +168,9 @@ qstring_conform(qstringObject *self, PyObject *args) /* object member list */ static struct PyMemberDef qstringObject_members[] = { - {"adapted", T_OBJECT, offsetof(qstringObject, wrapped), RO}, - {"buffer", T_OBJECT, offsetof(qstringObject, buffer), RO}, - {"encoding", T_STRING, offsetof(qstringObject, encoding), RO}, + {"adapted", T_OBJECT, offsetof(qstringObject, wrapped), READONLY}, + {"buffer", T_OBJECT, offsetof(qstringObject, buffer), READONLY}, + {"encoding", T_STRING, offsetof(qstringObject, encoding), READONLY}, {NULL} }; @@ -210,7 +192,7 @@ qstring_setup(qstringObject *self, PyObject *str, const char *enc) { Dprintf("qstring_setup: init qstring object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); self->buffer = NULL; @@ -224,7 +206,7 @@ qstring_setup(qstringObject *self, PyObject *str, const char *enc) Dprintf("qstring_setup: good qstring object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -253,10 +235,10 @@ qstring_dealloc(PyObject* obj) Dprintf("qstring_dealloc: deleted qstring object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -296,8 +278,7 @@ qstring_repr(qstringObject *self) "QuotedString(str, enc) -> new quoted object with 'enc' encoding" PyTypeObject qstringType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.QuotedString", sizeof(qstringObject), 0, diff --git a/psycopg/adapter_qstring.h b/psycopg/adapter_qstring.h index 83a8a7d4..d825fe0e 100644 --- a/psycopg/adapter_qstring.h +++ b/psycopg/adapter_qstring.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_QSTRING_H #define PSYCOPG_QSTRING_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif @@ -42,6 +37,10 @@ typedef struct { PyObject *wrapped; PyObject *buffer; + /* NOTE: this used to be a PostgreSQL encoding: changed in 2.3.2 to be a + * Python codec name. I don't expect there has been any user for this + * object other than adapting str/unicode, so I don't expect client code + * broken for this reason. */ char *encoding; PyObject *conn; diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c new file mode 100644 index 00000000..206e870d --- /dev/null +++ b/psycopg/bytes_format.c @@ -0,0 +1,299 @@ +/* bytes_format.c - bytes-oriented version of PyString_Format + * + * Copyright (C) 2010 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +/* This implementation is based on the PyString_Format function available in + * Python 2.7.1. The function is altered to be used with both Python 2 strings + * and Python 3 bytes and is stripped of the support of formats different than + * 's'. Original license follows. + * + * PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 + * -------------------------------------------- + * + * 1. This LICENSE AGREEMENT is between the Python Software Foundation + * ("PSF"), and the Individual or Organization ("Licensee") accessing and + * otherwise using this software ("Python") in source or binary form and + * its associated documentation. + * + * 2. Subject to the terms and conditions of this License Agreement, PSF hereby + * grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, + * analyze, test, perform and/or display publicly, prepare derivative works, + * distribute, and otherwise use Python alone or in any derivative version, + * provided, however, that PSF's License Agreement and PSF's notice of copyright, + * i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 + * Python Software Foundation; All Rights Reserved" are retained in Python alone or + * in any derivative version prepared by Licensee. + * + * 3. In the event Licensee prepares a derivative work that is based on + * or incorporates Python or any part thereof, and wants to make + * the derivative work available to others as provided herein, then + * Licensee hereby agrees to include in any such work a brief summary of + * the changes made to Python. + * + * 4. PSF is making Python available to Licensee on an "AS IS" + * basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR + * IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND + * DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS + * FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT + * INFRINGE ANY THIRD PARTY RIGHTS. + * + * 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON + * FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS + * A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, + * OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + * + * 6. This License Agreement will automatically terminate upon a material + * breach of its terms and conditions. + * + * 7. Nothing in this License Agreement shall be deemed to create any + * relationship of agency, partnership, or joint venture between PSF and + * Licensee. This License Agreement does not grant permission to use PSF + * trademarks or trade name in a trademark sense to endorse or promote + * products or services of Licensee, or any third party. + * + * 8. By copying, installing or otherwise using Python, Licensee + * agrees to be bound by the terms and conditions of this License + * Agreement. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#ifndef Py_LOCAL_INLINE +#define Py_LOCAL_INLINE(type) static type +#endif + +/* Helpers for formatstring */ + +Py_LOCAL_INLINE(PyObject *) +getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) +{ + Py_ssize_t argidx = *p_argidx; + if (argidx < arglen) { + (*p_argidx)++; + if (arglen < 0) + return args; + else + return PyTuple_GetItem(args, argidx); + } + PyErr_SetString(PyExc_TypeError, + "not enough arguments for format string"); + return NULL; +} + +/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */ + +PyObject * +Bytes_Format(PyObject *format, PyObject *args) +{ + char *fmt, *res; + Py_ssize_t arglen, argidx; + Py_ssize_t reslen, rescnt, fmtcnt; + int args_owned = 0; + PyObject *result, *orig_args; + PyObject *dict = NULL; + if (format == NULL || !Bytes_Check(format) || args == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + orig_args = args; + fmt = Bytes_AS_STRING(format); + fmtcnt = Bytes_GET_SIZE(format); + reslen = rescnt = fmtcnt + 100; + result = Bytes_FromStringAndSize((char *)NULL, reslen); + if (result == NULL) + return NULL; + res = Bytes_AsString(result); + if (PyTuple_Check(args)) { + arglen = PyTuple_GET_SIZE(args); + argidx = 0; + } + else { + arglen = -1; + argidx = -2; + } + if (Py_TYPE(args)->tp_as_mapping && !PyTuple_Check(args) && + !PyObject_TypeCheck(args, &Bytes_Type)) + dict = args; + while (--fmtcnt >= 0) { + if (*fmt != '%') { + if (--rescnt < 0) { + rescnt = fmtcnt + 100; + reslen += rescnt; + if (_Bytes_Resize(&result, reslen)) + return NULL; + res = Bytes_AS_STRING(result) + + reslen - rescnt; + --rescnt; + } + *res++ = *fmt++; + } + else { + /* Got a format specifier */ + Py_ssize_t width = -1; + int c = '\0'; + PyObject *v = NULL; + PyObject *temp = NULL; + char *pbuf; + Py_ssize_t len; + fmt++; + if (*fmt == '(') { + char *keystart; + Py_ssize_t keylen; + PyObject *key; + int pcount = 1; + + if (dict == NULL) { + PyErr_SetString(PyExc_TypeError, + "format requires a mapping"); + goto error; + } + ++fmt; + --fmtcnt; + keystart = fmt; + /* Skip over balanced parentheses */ + while (pcount > 0 && --fmtcnt >= 0) { + if (*fmt == ')') + --pcount; + else if (*fmt == '(') + ++pcount; + fmt++; + } + keylen = fmt - keystart - 1; + if (fmtcnt < 0 || pcount > 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format key"); + goto error; + } + key = Text_FromUTF8AndSize(keystart, keylen); + if (key == NULL) + goto error; + if (args_owned) { + Py_DECREF(args); + args_owned = 0; + } + args = PyObject_GetItem(dict, key); + Py_DECREF(key); + if (args == NULL) { + goto error; + } + args_owned = 1; + arglen = -1; + argidx = -2; + } + while (--fmtcnt >= 0) { + c = *fmt++; + break; + } + if (fmtcnt < 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format"); + goto error; + } + if (c != '%') { + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto error; + } + switch (c) { + case '%': + pbuf = "%"; + len = 1; + break; + case 's': + /* only bytes! */ + if (!Bytes_CheckExact(v)) { + PyErr_Format(PyExc_ValueError, + "only bytes values expected, got %s", + Py_TYPE(v)->tp_name); + goto error; + } + temp = v; + Py_INCREF(v); + pbuf = Bytes_AS_STRING(temp); + len = Bytes_GET_SIZE(temp); + break; + default: + PyErr_Format(PyExc_ValueError, + "unsupported format character '%c' (0x%x) " + "at index " FORMAT_CODE_PY_SSIZE_T, + c, c, + (Py_ssize_t)(fmt - 1 - + Bytes_AsString(format))); + goto error; + } + if (width < len) + width = len; + if (rescnt < width) { + reslen -= rescnt; + rescnt = width + fmtcnt + 100; + reslen += rescnt; + if (reslen < 0) { + Py_DECREF(result); + Py_XDECREF(temp); + return PyErr_NoMemory(); + } + if (_Bytes_Resize(&result, reslen)) { + Py_XDECREF(temp); + return NULL; + } + res = Bytes_AS_STRING(result) + + reslen - rescnt; + } + Py_MEMCPY(res, pbuf, len); + res += len; + rescnt -= len; + while (--width >= len) { + --rescnt; + *res++ = ' '; + } + if (dict && (argidx < arglen) && c != '%') { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + Py_XDECREF(temp); + goto error; + } + Py_XDECREF(temp); + } /* '%' */ + } /* until end */ + if (argidx < arglen && !dict) { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + goto error; + } + if (args_owned) { + Py_DECREF(args); + } + if (_Bytes_Resize(&result, reslen - rescnt)) + return NULL; + return result; + + error: + Py_DECREF(result); + if (args_owned) { + Py_DECREF(args); + } + return NULL; +} + diff --git a/psycopg/connection.h b/psycopg/connection.h index 529a211e..af2470da 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_CONNECTION_H #define PSYCOPG_CONNECTION_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" #include "psycopg/xid.h" #ifdef __cplusplus @@ -88,6 +83,7 @@ typedef struct { char *dsn; /* data source name */ char *critical; /* critical error on this connection */ char *encoding; /* current backend encoding */ + char *codec; /* python codec name for encoding */ long int closed; /* 1 means connection has been closed; 2 that something horrible happened */ @@ -124,6 +120,7 @@ typedef struct { } connectionObject; /* C-callable functions in connection_int.c and connection_ext.c */ +HIDDEN PyObject *conn_text_from_chars(connectionObject *pgconn, const char *str); HIDDEN int conn_get_standard_conforming_strings(PGconn *pgconn); HIDDEN int conn_get_isolation_level(PGresult *pgres); HIDDEN int conn_get_protocol_version(PGconn *pgconn); diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 9ccc11d5..882b0eff 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -23,19 +23,35 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/cursor.h" #include "psycopg/pqpath.h" #include "psycopg/green.h" #include "psycopg/notify.h" +#include + + +/* Return a new "string" from a char* from the database. + * + * On Py2 just get a string, on Py3 decode it in the connection codec. + * + * Use a fallback if the connection is NULL. + */ +PyObject * +conn_text_from_chars(connectionObject *self, const char *str) +{ +#if PY_MAJOR_VERSION < 3 + return PyString_FromString(str); +#else + const char *codec = self ? self->codec : "ascii"; + return PyUnicode_Decode(str, strlen(str), codec, "replace"); +#endif +} + /* conn_notice_callback - process notices */ static void @@ -77,8 +93,7 @@ conn_notice_process(connectionObject *self) while (notice != NULL) { PyObject *msg; - msg = PyString_FromString(notice->message); - + msg = conn_text_from_chars(self, notice->message); Dprintf("conn_notice_process: %s", notice->message); /* Respect the order in which notices were produced, @@ -146,8 +161,8 @@ conn_notifies_process(connectionObject *self) (int) pgn->be_pid, pgn->relname); if (!(pid = PyInt_FromLong((long)pgn->be_pid))) { goto error; } - if (!(channel = PyString_FromString(pgn->relname))) { goto error; } - if (!(payload = PyString_FromString(pgn->extra))) { goto error; } + if (!(channel = conn_text_from_chars(self, pgn->relname))) { goto error; } + if (!(payload = conn_text_from_chars(self, pgn->extra))) { goto error; } if (!(notify = PyObject_CallFunctionObjArgs((PyObject *)&NotifyType, pid, channel, payload, NULL))) { @@ -213,38 +228,97 @@ conn_get_standard_conforming_strings(PGconn *pgconn) return equote; } -/* Return a string containing the client_encoding setting. +/* Convert a PostgreSQL encoding to a Python codec. * - * Return a new string allocated by malloc(): use free() to free it. - * Return NULL in case of failure. + * Return a new copy of the codec name allocated on the Python heap, + * NULL with exception in case of error. */ static char * -conn_get_encoding(PGconn *pgconn) +conn_encoding_to_codec(const char *enc) { - const char *tmp, *i; - char *encoding, *j; + char *tmp; + Py_ssize_t size; + PyObject *pyenc = NULL; + char *rv = NULL; + + /* Find the Py codec name from the PG encoding */ + if (!(pyenc = PyDict_GetItemString(psycoEncodings, enc))) { + PyErr_Format(OperationalError, + "no Python codec for client encoding '%s'", enc); + goto exit; + } + + /* Convert the codec in a bytes string to extract the c string. */ + Py_INCREF(pyenc); + if (!(pyenc = psycopg_ensure_bytes(pyenc))) { + goto exit; + } + + if (-1 == Bytes_AsStringAndSize(pyenc, &tmp, &size)) { + goto exit; + } + + /* have our own copy of the python codec name */ + rv = psycopg_strdup(tmp, size); + +exit: + Py_XDECREF(pyenc); + return rv; +} + +/* Read the client encoding from the connection. + * + * Store the encoding in the pgconn->encoding field and the name of the + * matching python codec in codec. The buffers are allocated on the Python + * heap. + * + * Return 0 on success, else nonzero. + */ +static int +conn_read_encoding(connectionObject *self, PGconn *pgconn) +{ + char *enc = NULL, *codec = NULL, *j; + const char *tmp; + int rv = -1; tmp = PQparameterStatus(pgconn, "client_encoding"); Dprintf("conn_connect: client encoding: %s", tmp ? tmp : "(none)"); if (!tmp) { PyErr_SetString(OperationalError, "server didn't return client encoding"); - return NULL; + goto exit; } - encoding = malloc(strlen(tmp)+1); - if (encoding == NULL) { + if (!(enc = PyMem_Malloc(strlen(tmp)+1))) { PyErr_NoMemory(); - return NULL; + goto exit; } - /* return in uppercase */ - i = tmp; - j = encoding; - while (*i) { *j++ = toupper(*i++); } + /* turn encoding in uppercase */ + j = enc; + while (*tmp) { *j++ = toupper(*tmp++); } *j = '\0'; - return encoding; + /* Look for this encoding in Python codecs. */ + if (!(codec = conn_encoding_to_codec(enc))) { + goto exit; + } + + /* Good, success: store the encoding/codec in the connection. */ + PyMem_Free(self->encoding); + self->encoding = enc; + enc = NULL; + + PyMem_Free(self->codec); + self->codec = codec; + codec = NULL; + + rv = 0; + +exit: + PyMem_Free(enc); + PyMem_Free(codec); + return rv; } int @@ -324,9 +398,8 @@ conn_setup(connectionObject *self, PGconn *pgconn) PyErr_SetString(InterfaceError, "only protocol 3 supported"); return -1; } - /* conn_get_encoding returns a malloc'd string */ - self->encoding = conn_get_encoding(pgconn); - if (self->encoding == NULL) { + + if (conn_read_encoding(self, pgconn)) { return -1; } @@ -656,9 +729,7 @@ _conn_poll_setup_async(connectionObject *self) PyErr_SetString(InterfaceError, "only protocol 3 supported"); break; } - /* conn_get_encoding returns a malloc'd string */ - self->encoding = conn_get_encoding(self->pgconn); - if (self->encoding == NULL) { + if (conn_read_encoding(self, self->pgconn)) { break; } self->cancel = conn_get_cancel(self->pgconn); @@ -887,11 +958,15 @@ conn_set_client_encoding(connectionObject *self, const char *enc) char *error = NULL; char query[48]; int res = 0; + char *codec; /* If the current encoding is equal to the requested one we don't issue any query to the backend */ if (strcmp(self->encoding, enc) == 0) return 0; + /* We must know what python codec this encoding is. */ + if (!(codec = conn_encoding_to_codec(enc))) { return -1; } + Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); @@ -900,19 +975,29 @@ conn_set_client_encoding(connectionObject *self, const char *enc) /* abort the current transaction, to set the encoding ouside of transactions */ - res = pq_abort_locked(self, &pgres, &error, &_save); - - if (res == 0) { - res = pq_execute_command_locked(self, query, &pgres, &error, &_save); - if (res == 0) { - /* no error, we can proceeed and store the new encoding */ - if (self->encoding) free(self->encoding); - self->encoding = strdup(enc); - Dprintf("conn_set_client_encoding: set encoding to %s", - self->encoding); - } + if ((res = pq_abort_locked(self, &pgres, &error, &_save))) { + goto endlock; } + if ((res = pq_execute_command_locked(self, query, &pgres, &error, &_save))) { + goto endlock; + } + + /* no error, we can proceeed and store the new encoding */ + PyMem_Free(self->encoding); + if (!(self->encoding = psycopg_strdup(enc, 0))) { + res = 1; /* don't call pq_complete_error below */ + goto endlock; + } + + /* Store the python codec too. */ + PyMem_Free(self->codec); + self->codec = codec; + + Dprintf("conn_set_client_encoding: set encoding to %s (codec: %s)", + self->encoding, self->codec); + +endlock: pthread_mutex_unlock(&self->lock); Py_END_ALLOW_THREADS; @@ -978,8 +1063,8 @@ conn_tpc_command(connectionObject *self, const char *cmd, XidObject *xid) Dprintf("conn_tpc_command: %s", cmd); /* convert the xid into PostgreSQL transaction id while keeping the GIL */ - if (!(tid = xid_get_tid(xid))) { goto exit; } - if (!(ctid = PyString_AsString(tid))) { goto exit; } + if (!(tid = psycopg_ensure_bytes(xid_get_tid(xid)))) { goto exit; } + if (!(ctid = Bytes_AsString(tid))) { goto exit; } Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 97ca4e3c..ae92d68b 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -23,18 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/cursor.h" #include "psycopg/pqpath.h" @@ -42,6 +33,10 @@ #include "psycopg/green.h" #include "psycopg/xid.h" +#include +#include + + /** DBAPI methods **/ /* cursor method - allocate a new cursor */ @@ -103,7 +98,7 @@ psyco_conn_cursor(connectionObject *self, PyObject *args, PyObject *keywds) Dprintf("psyco_conn_cursor: new cursor at %p: refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); return obj; } @@ -427,35 +422,38 @@ psyco_conn_set_isolation_level(connectionObject *self, PyObject *args) static PyObject * psyco_conn_set_client_encoding(connectionObject *self, PyObject *args) { - const char *enc = NULL; - char *buffer; - size_t i, j; + const char *enc; + char *buffer, *dest; + PyObject *rv = NULL; + Py_ssize_t len; EXC_IF_CONN_CLOSED(self); EXC_IF_CONN_ASYNC(self, set_client_encoding); EXC_IF_TPC_PREPARED(self, set_client_encoding); - if (!PyArg_ParseTuple(args, "s", &enc)) return NULL; + if (!PyArg_ParseTuple(args, "s#", &enc, &len)) return NULL; /* convert to upper case and remove '-' and '_' from string */ - buffer = PyMem_Malloc(strlen(enc)+1); - for (i=j=0 ; i < strlen(enc) ; i++) { - if (enc[i] == '_' || enc[i] == '-') - continue; - else - buffer[j++] = toupper(enc[i]); + if (!(dest = buffer = PyMem_Malloc(len+1))) { + return PyErr_NoMemory(); } - buffer[j] = '\0'; + + while (*enc) { + if (*enc == '_' || *enc == '-') { + ++enc; + } + else { + *dest++ = toupper(*enc++); + } + } + *dest = '\0'; if (conn_set_client_encoding(self, buffer) == 0) { - PyMem_Free(buffer); Py_INCREF(Py_None); - return Py_None; - } - else { - PyMem_Free(buffer); - return NULL; + rv = Py_None; } + PyMem_Free(buffer); + return rv; } /* get_transaction_status method - Get backend transaction status */ @@ -497,7 +495,7 @@ psyco_conn_get_parameter_status(connectionObject *self, PyObject *args) Py_INCREF(Py_None); return Py_None; } - return PyString_FromString(val); + return conn_text_from_chars(self, val); } @@ -516,15 +514,16 @@ static PyObject * psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) { Oid oid=InvalidOid, new_oid=InvalidOid; - char *smode = NULL, *new_file = NULL; - int mode=0; - PyObject *obj, *factory = NULL; + char *new_file = NULL; + const char *smode = ""; + PyObject *factory = (PyObject *)&lobjectType; + PyObject *obj; static char *kwlist[] = {"oid", "mode", "new_oid", "new_file", "cursor_factory", NULL}; - + if (!PyArg_ParseTupleAndKeywords(args, keywds, "|izizO", kwlist, - &oid, &smode, &new_oid, &new_file, + &oid, &smode, &new_oid, &new_file, &factory)) { return NULL; } @@ -539,33 +538,13 @@ psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) oid, smode); Dprintf("psyco_conn_lobject: parameters: new_oid = %d, new_file = %s", new_oid, new_file); - - /* build a mode number out of the mode string: right now we only accept - 'r', 'w' and 'rw' (but note that 'w' implies 'rw' because PostgreSQL - backend does that. */ - if (smode) { - if (strncmp("rw", smode, 2) == 0) - mode = INV_READ+INV_WRITE; - else if (smode[0] == 'r') - mode = INV_READ; - else if (smode[0] == 'w') - mode = INV_WRITE; - else if (smode[0] == 'n') - mode = -1; - else { - PyErr_SetString(PyExc_TypeError, - "mode should be one of 'r', 'w' or 'rw'"); - return NULL; - } - } - if (factory == NULL) factory = (PyObject *)&lobjectType; if (new_file) - obj = PyObject_CallFunction(factory, "Oiiis", - self, oid, mode, new_oid, new_file); + obj = PyObject_CallFunction(factory, "Oisis", + self, oid, smode, new_oid, new_file); else - obj = PyObject_CallFunction(factory, "Oiii", - self, oid, mode, new_oid); + obj = PyObject_CallFunction(factory, "Oisi", + self, oid, smode, new_oid); if (obj == NULL) return NULL; if (PyObject_IsInstance(obj, (PyObject *)&lobjectType) == 0) { @@ -574,10 +553,10 @@ psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) Py_DECREF(obj); return NULL; } - + Dprintf("psyco_conn_lobject: new lobject at %p: refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt); + obj, Py_REFCNT(obj)); return obj; } @@ -786,31 +765,31 @@ static struct PyMethodDef connectionObject_methods[] = { static struct PyMemberDef connectionObject_members[] = { #ifdef PSYCOPG_EXTENSIONS - {"closed", T_LONG, offsetof(connectionObject, closed), RO, + {"closed", T_LONG, offsetof(connectionObject, closed), READONLY, "True if the connection is closed."}, {"isolation_level", T_LONG, - offsetof(connectionObject, isolation_level), RO, + offsetof(connectionObject, isolation_level), READONLY, "The current isolation level."}, - {"encoding", T_STRING, offsetof(connectionObject, encoding), RO, + {"encoding", T_STRING, offsetof(connectionObject, encoding), READONLY, "The current client encoding."}, - {"notices", T_OBJECT, offsetof(connectionObject, notice_list), RO}, - {"notifies", T_OBJECT, offsetof(connectionObject, notifies), RO}, - {"dsn", T_STRING, offsetof(connectionObject, dsn), RO, + {"notices", T_OBJECT, offsetof(connectionObject, notice_list), READONLY}, + {"notifies", T_OBJECT, offsetof(connectionObject, notifies), READONLY}, + {"dsn", T_STRING, offsetof(connectionObject, dsn), READONLY, "The current connection string."}, - {"async", T_LONG, offsetof(connectionObject, async), RO, + {"async", T_LONG, offsetof(connectionObject, async), READONLY, "True if the connection is asynchronous."}, {"status", T_INT, - offsetof(connectionObject, status), RO, + offsetof(connectionObject, status), READONLY, "The current transaction status."}, - {"string_types", T_OBJECT, offsetof(connectionObject, string_types), RO, + {"string_types", T_OBJECT, offsetof(connectionObject, string_types), READONLY, "A set of typecasters to convert textual values."}, - {"binary_types", T_OBJECT, offsetof(connectionObject, binary_types), RO, + {"binary_types", T_OBJECT, offsetof(connectionObject, binary_types), READONLY, "A set of typecasters to convert binary values."}, {"protocol_version", T_INT, - offsetof(connectionObject, protocol), RO, + offsetof(connectionObject, protocol), READONLY, "Protocol version used for this connection. Currently always 3."}, {"server_version", T_INT, - offsetof(connectionObject, server_version), RO, + offsetof(connectionObject, server_version), READONLY, "Server version."}, #endif {NULL} @@ -845,26 +824,18 @@ connection_setup(connectionObject *self, const char *dsn, long int async) Dprintf("connection_setup: init connection object at %p, " "async %ld, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, async, ((PyObject *)self)->ob_refcnt + self, async, Py_REFCNT(self) ); self->dsn = strdup(dsn); self->notice_list = PyList_New(0); self->notifies = PyList_New(0); - self->closed = 0; self->async = async; self->status = CONN_STATUS_SETUP; - self->critical = NULL; - self->async_cursor = NULL; self->async_status = ASYNC_DONE; - self->pgconn = NULL; - self->cancel = NULL; - self->mark = 0; self->string_types = PyDict_New(); self->binary_types = PyDict_New(); - self->notice_pending = NULL; - self->encoding = NULL; - self->weakreflist = NULL; + /* other fields have been zeroed by tp_alloc */ pthread_mutex_init(&(self->lock), NULL); @@ -875,7 +846,7 @@ connection_setup(connectionObject *self, const char *dsn, long int async) else { Dprintf("connection_setup: good connection object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); res = 0; } @@ -906,7 +877,8 @@ connection_dealloc(PyObject* obj) conn_notice_clean(self); if (self->dsn) free(self->dsn); - if (self->encoding) free(self->encoding); + PyMem_Free(self->encoding); + PyMem_Free(self->codec); if (self->critical) free(self->critical); Py_CLEAR(self->tpc_xid); @@ -921,10 +893,10 @@ connection_dealloc(PyObject* obj) Dprintf("connection_dealloc: deleted connection object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -984,8 +956,7 @@ connection_traverse(connectionObject *self, visitproc visit, void *arg) " ProgrammingError, IntegrityError, DataError, NotSupportedError" PyTypeObject connectionType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.connection", sizeof(connectionObject), 0, diff --git a/psycopg/cursor.h b/psycopg/cursor.h index 96a133fa..92a122b9 100644 --- a/psycopg/cursor.h +++ b/psycopg/cursor.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_CURSOR_H #define PSYCOPG_CURSOR_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" #include "psycopg/connection.h" #ifdef __cplusplus diff --git a/psycopg/cursor_int.c b/psycopg/cursor_int.c index 75d1c687..40e0bae3 100644 --- a/psycopg/cursor_int.c +++ b/psycopg/cursor_int.c @@ -23,13 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" + #include "psycopg/cursor.h" #include "psycopg/pqpath.h" #include "psycopg/typecast.h" @@ -68,6 +64,9 @@ curs_get_cast(cursorObject *self, PyObject *oid) return psyco_default_cast; } +#include + + /* curs_reset - reset the cursor to a clean state */ void diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 5416269e..ba22176b 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -23,15 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/cursor.h" #include "psycopg/connection.h" #include "psycopg/green.h" @@ -39,9 +33,12 @@ #include "psycopg/typecast.h" #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" -#include "pgversion.h" + +#include + #include + extern PyObject *pyPsycopgTzFixedOffsetTimezone; @@ -90,7 +87,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) just before returning. we also init *new to NULL to exit with an error if we can't complete the mogrification */ n = *new = NULL; - c = PyString_AsString(fmt); + c = Bytes_AsString(fmt); while(*c) { /* handle plain percent symbol in format string */ @@ -119,7 +116,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) for (d = c + 2; *d && *d != ')'; d++); if (*d == ')') { - key = PyString_FromStringAndSize(c+2, (Py_ssize_t) (d-c-2)); + key = Text_FromUTF8AndSize(c+2, (Py_ssize_t) (d-c-2)); value = PyObject_GetItem(var, key); /* key has refcnt 1, value the original value + 1 */ @@ -132,7 +129,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) } Dprintf("_mogrify: value refcnt: " - FORMAT_CODE_PY_SSIZE_T " (+1)", value->ob_refcnt); + FORMAT_CODE_PY_SSIZE_T " (+1)", Py_REFCNT(value)); if (n == NULL) { n = PyDict_New(); @@ -147,7 +144,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) optimization over the adapting code and can go away in the future if somebody finds a None adapter usefull. */ if (value == Py_None) { - t = PyString_FromString("NULL"); + t = Bytes_FromString("NULL"); PyDict_SetItem(n, key, t); /* t is a new object, refcnt = 1, key is at 2 */ @@ -185,7 +182,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) Py_DECREF(key); /* key has the original refcnt now */ Dprintf("_mogrify: after value refcnt: " FORMAT_CODE_PY_SSIZE_T, - value->ob_refcnt + Py_REFCNT(value) ); } c = d; @@ -223,7 +220,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) d = c+1; if (value == Py_None) { - PyTuple_SET_ITEM(n, index, PyString_FromString("NULL")); + PyTuple_SET_ITEM(n, index, Bytes_FromString("NULL")); while (*d && !isalpha(*d)) d++; if (*d) *d = 's'; Py_DECREF(value); @@ -270,26 +267,16 @@ static PyObject *_psyco_curs_validate_sql_basic( goto fail; } - if (PyString_Check(sql)) { + if (Bytes_Check(sql)) { /* Necessary for ref-count symmetry with the unicode case: */ Py_INCREF(sql); } else if (PyUnicode_Check(sql)) { - PyObject *enc = PyDict_GetItemString(psycoEncodings, - self->conn->encoding); - /* enc is a borrowed reference; we won't decref it */ - - if (enc) { - sql = PyUnicode_AsEncodedString(sql, PyString_AsString(enc), NULL); - /* if there was an error during the encoding from unicode to the - target encoding, we just let the exception propagate */ - if (sql == NULL) { goto fail; } - } else { - PyErr_Format(InterfaceError, - "can't encode unicode SQL statement to %s", - self->conn->encoding); - goto fail; - } + char *enc = self->conn->codec; + sql = PyUnicode_AsEncodedString(sql, enc, NULL); + /* if there was an error during the encoding from unicode to the + target encoding, we just let the exception propagate */ + if (sql == NULL) { goto fail; } } else { /* the is not unicode or string, raise an error */ @@ -327,7 +314,7 @@ _psyco_curs_merge_query_args(cursorObject *self, the curren exception (we will later restore it if the type or the strings do not match.) */ - if (!(fquery = PyString_Format(query, args))) { + if (!(fquery = Bytes_Format(query, args))) { PyObject *err, *arg, *trace; int pe = 0; @@ -340,7 +327,7 @@ _psyco_curs_merge_query_args(cursorObject *self, if (PyObject_HasAttrString(arg, "args")) { PyObject *args = PyObject_GetAttrString(arg, "args"); PyObject *str = PySequence_GetItem(args, 0); - const char *s = PyString_AS_STRING(str); + const char *s = Bytes_AS_STRING(str); Dprintf("psyco_curs_execute: -> %s", s); @@ -410,9 +397,9 @@ _psyco_curs_execute(cursorObject *self, } if (self->name != NULL) { - self->query = PyString_FromFormat( + self->query = Bytes_FromFormat( "DECLARE %s CURSOR WITHOUT HOLD FOR %s", - self->name, PyString_AS_STRING(fquery)); + self->name, Bytes_AS_STRING(fquery)); Py_DECREF(fquery); } else { @@ -421,9 +408,9 @@ _psyco_curs_execute(cursorObject *self, } else { if (self->name != NULL) { - self->query = PyString_FromFormat( + self->query = Bytes_FromFormat( "DECLARE %s CURSOR WITHOUT HOLD FOR %s", - self->name, PyString_AS_STRING(operation)); + self->name, Bytes_AS_STRING(operation)); } else { /* Transfer reference ownership of the str in operation to @@ -436,7 +423,7 @@ _psyco_curs_execute(cursorObject *self, /* At this point, the SQL statement must be str, not unicode */ - res = pq_execute(self, PyString_AS_STRING(self->query), async); + res = pq_execute(self, Bytes_AS_STRING(self->query), async); Dprintf("psyco_curs_execute: res = %d, pgres = %p", res, self->pgres); if (res == -1) { goto fail; } @@ -596,7 +583,7 @@ _psyco_curs_mogrify(cursorObject *self, Dprintf("psyco_curs_mogrify: cvt->refcnt = " FORMAT_CODE_PY_SSIZE_T ", fquery->refcnt = " FORMAT_CODE_PY_SSIZE_T, - cvt->ob_refcnt, fquery->ob_refcnt); + Py_REFCNT(cvt), Py_REFCNT(fquery)); } else { fquery = operation; @@ -704,7 +691,7 @@ _psyco_curs_buildrow_fill(cursorObject *self, PyObject *res, if (val) { Dprintf("_psyco_curs_buildrow: val->refcnt = " FORMAT_CODE_PY_SSIZE_T, - val->ob_refcnt + Py_REFCNT(val) ); if (istuple) { PyTuple_SET_ITEM(res, i, val); @@ -1044,7 +1031,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args, PyObject *kwargs) sql[sl-2] = ')'; sql[sl-1] = '\0'; - operation = PyString_FromString(sql); + operation = Bytes_FromString(sql); PyMem_Free((void*)sql); if (_psyco_curs_execute(self, operation, parameters, self->conn->async)) { @@ -1199,14 +1186,11 @@ static int _psyco_curs_copy_columns(PyObject *columns, char *columnlist) columnlist[0] = '('; while ((col = PyIter_Next(coliter)) != NULL) { - if (!PyString_Check(col)) { - Py_DECREF(col); + if (!(col = psycopg_ensure_bytes(col))) { Py_DECREF(coliter); - PyErr_SetString(PyExc_ValueError, - "elements in column list must be strings"); return -1; } - PyString_AsStringAndSize(col, &colname, &collen); + Bytes_AsStringAndSize(col, &colname, &collen); if (offset + collen > DEFAULT_COPYBUFF - 2) { Py_DECREF(col); Py_DECREF(coliter); @@ -1511,7 +1495,7 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs) self->copyfile = file; /* At this point, the SQL statement must be str, not unicode */ - if (pq_execute(self, PyString_AS_STRING(sql), 0) != 1) { goto fail; } + if (pq_execute(self, Bytes_AS_STRING(sql), 0) != 1) { goto fail; } res = Py_None; Py_INCREF(res); @@ -1631,29 +1615,29 @@ static struct PyMethodDef cursorObject_methods[] = { static struct PyMemberDef cursorObject_members[] = { /* DBAPI-2.0 basics */ - {"rowcount", T_LONG, OFFSETOF(rowcount), RO, + {"rowcount", T_LONG, OFFSETOF(rowcount), READONLY, "Number of rows read from the backend in the last command."}, {"arraysize", T_LONG, OFFSETOF(arraysize), 0, "Number of records `fetchmany()` must fetch if not explicitly " \ "specified."}, - {"description", T_OBJECT, OFFSETOF(description), RO, + {"description", T_OBJECT, OFFSETOF(description), READONLY, "Cursor description as defined in DBAPI-2.0."}, - {"lastrowid", T_LONG, OFFSETOF(lastoid), RO, + {"lastrowid", T_LONG, OFFSETOF(lastoid), READONLY, "The ``oid`` of the last row inserted by the cursor."}, /* DBAPI-2.0 extensions */ - {"rownumber", T_LONG, OFFSETOF(row), RO, + {"rownumber", T_LONG, OFFSETOF(row), READONLY, "The current row position."}, - {"connection", T_OBJECT, OFFSETOF(conn), RO, + {"connection", T_OBJECT, OFFSETOF(conn), READONLY, "The connection where the cursor comes from."}, #ifdef PSYCOPG_EXTENSIONS - {"name", T_STRING, OFFSETOF(name), RO}, - {"statusmessage", T_OBJECT, OFFSETOF(pgstatus), RO, + {"name", T_STRING, OFFSETOF(name), READONLY}, + {"statusmessage", T_OBJECT, OFFSETOF(pgstatus), READONLY, "The return message of the last command."}, - {"query", T_OBJECT, OFFSETOF(query), RO, + {"query", T_OBJECT, OFFSETOF(query), READONLY, "The last query text sent to the backend."}, {"row_factory", T_OBJECT, OFFSETOF(tuple_factory), 0}, {"tzinfo_factory", T_OBJECT, OFFSETOF(tzinfo_factory), 0}, - {"typecaster", T_OBJECT, OFFSETOF(caster), RO}, + {"typecaster", T_OBJECT, OFFSETOF(caster), READONLY}, {"string_types", T_OBJECT, OFFSETOF(string_types), 0}, {"binary_types", T_OBJECT, OFFSETOF(binary_types), 0}, #endif @@ -1723,7 +1707,7 @@ cursor_setup(cursorObject *self, connectionObject *conn, const char *name) Dprintf("cursor_setup: good cursor object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -1755,9 +1739,9 @@ cursor_dealloc(PyObject* obj) Dprintf("cursor_dealloc: deleted cursor object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt); + obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -1815,8 +1799,7 @@ cursor_traverse(cursorObject *self, visitproc visit, void *arg) "A database cursor." PyTypeObject cursorType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.cursor", sizeof(cursorObject), 0, diff --git a/psycopg/green.c b/psycopg/green.c index 4c7cc409..c9b6e07f 100644 --- a/psycopg/green.c +++ b/psycopg/green.c @@ -24,13 +24,13 @@ */ #define PSYCOPG_MODULE -#include "psycopg/python.h" #include "psycopg/psycopg.h" -#include "psycopg/config.h" + #include "psycopg/green.h" #include "psycopg/connection.h" #include "psycopg/pqpath.h" + HIDDEN PyObject *wait_callback = NULL; static PyObject *have_wait_callback(void); diff --git a/psycopg/lobject.h b/psycopg/lobject.h index 2d6715cc..293f608f 100644 --- a/psycopg/lobject.h +++ b/psycopg/lobject.h @@ -26,11 +26,8 @@ #ifndef PSYCOPG_LOBJECT_H #define PSYCOPG_LOBJECT_H 1 -#include -#include #include -#include "psycopg/config.h" #include "psycopg/connection.h" #ifdef __cplusplus @@ -45,7 +42,8 @@ typedef struct { connectionObject *conn; /* connection owning the lobject */ long int mark; /* copied from conn->mark */ - const char *smode; /* string mode if lobject was opened */ + char *smode; /* string mode if lobject was opened */ + int mode; /* numeric version of smode */ int fd; /* the file descriptor for file-like ops */ Oid oid; /* the oid for this lobject */ @@ -54,7 +52,7 @@ typedef struct { /* functions exported from lobject_int.c */ HIDDEN int lobject_open(lobjectObject *self, connectionObject *conn, - Oid oid, int mode, Oid new_oid, + Oid oid, const char *smode, Oid new_oid, const char *new_file); HIDDEN int lobject_unlink(lobjectObject *self); HIDDEN int lobject_export(lobjectObject *self, const char *filename); @@ -90,6 +88,12 @@ if (self->conn->mark != self->mark) { \ return NULL; \ } +/* Values for the lobject mode */ +#define LOBJECT_READ 1 +#define LOBJECT_WRITE 2 +#define LOBJECT_BINARY 4 +#define LOBJECT_TEXT 8 + #ifdef __cplusplus } #endif diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index c3ae662f..252d1c93 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -23,17 +23,15 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" -#include "psycopg/connection.h" + #include "psycopg/lobject.h" +#include "psycopg/connection.h" #include "psycopg/pqpath.h" +#include + #ifdef PSYCOPG_EXTENSIONS static void @@ -45,15 +43,118 @@ collect_error(connectionObject *conn, char **error) *error = strdup(msg); } + +/* Check if the mode passed to the large object is valid. + * In case of success return a value >= 0 + * On error return a value < 0 and set an exception. + * + * Valid mode are [r|w|rw|n][t|b] + */ +static int +_lobject_parse_mode(const char *mode) +{ + int rv = 0; + size_t pos = 0; + + if (0 == strncmp("rw", mode, 2)) { + rv |= LOBJECT_READ | LOBJECT_WRITE; + pos += 2; + } + else { + switch (mode[0]) { + case 'r': + rv |= LOBJECT_READ; + pos += 1; + break; + case 'w': + rv |= LOBJECT_WRITE; + pos += 1; + break; + case 'n': + pos += 1; + break; + default: + rv |= LOBJECT_READ; + break; + } + } + + switch (mode[pos]) { + case 't': + rv |= LOBJECT_TEXT; + pos += 1; + break; + case 'b': + rv |= LOBJECT_BINARY; + pos += 1; + break; + default: +#if PY_MAJOR_VERSION < 3 + rv |= LOBJECT_BINARY; +#else + rv |= LOBJECT_TEXT; +#endif + break; + } + + if (pos != strlen(mode)) { + PyErr_Format(PyExc_ValueError, + "bad mode for lobject: '%s'", mode); + rv = -1; + } + + return rv; +} + + +/* Return a string representing the lobject mode. + * + * The return value is a new string allocated on the Python heap. + */ +static char * +_lobject_unparse_mode(int mode) +{ + char *buf; + char *c; + + /* the longest is 'rwt' */ + c = buf = PyMem_Malloc(4); + + if (mode & LOBJECT_READ) { *c++ = 'r'; } + if (mode & LOBJECT_WRITE) { *c++ = 'w'; } + + if (buf == c) { + /* neither read nor write */ + *c++ = 'n'; + } + else { + if (mode & LOBJECT_TEXT) { + *c++ = 't'; + } + else { + *c++ = 'b'; + } + } + *c = '\0'; + + return buf; +} + /* lobject_open - create a new/open an existing lo */ int lobject_open(lobjectObject *self, connectionObject *conn, - Oid oid, int mode, Oid new_oid, const char *new_file) + Oid oid, const char *smode, Oid new_oid, const char *new_file) { int retvalue = -1; PGresult *pgres = NULL; char *error = NULL; + int pgmode = 0; + int mode; + + if (0 > (mode = _lobject_parse_mode(smode))) { + return -1; + } Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); @@ -80,19 +181,19 @@ lobject_open(lobjectObject *self, connectionObject *conn, goto end; } - mode = INV_WRITE; + mode = (mode & ~LOBJECT_READ) | LOBJECT_WRITE; } else { self->oid = oid; - if (mode == 0) mode = INV_READ; } - /* if the oid is a real one we try to open with the given mode, - unless the mode is -1, meaning "don't open!" */ - if (mode != -1) { - self->fd = lo_open(self->conn->pgconn, self->oid, mode); - Dprintf("lobject_open: large object opened with fd = %d", - self->fd); + /* if the oid is a real one we try to open with the given mode */ + if (mode & LOBJECT_READ) { pgmode |= INV_READ; } + if (mode & LOBJECT_WRITE) { pgmode |= INV_WRITE; } + if (pgmode) { + self->fd = lo_open(self->conn->pgconn, self->oid, pgmode); + Dprintf("lobject_open: large object opened with mode = %i fd = %d", + pgmode, self->fd); if (self->fd == -1) { collect_error(self->conn, &error); @@ -100,17 +201,10 @@ lobject_open(lobjectObject *self, connectionObject *conn, goto end; } } + /* set the mode for future reference */ - switch (mode) { - case -1: - self->smode = "n"; break; - case INV_READ: - self->smode = "r"; break; - case INV_WRITE: - self->smode = "w"; break; - case INV_READ+INV_WRITE: - self->smode = "rw"; break; - } + self->mode = mode; + self->smode = _lobject_unparse_mode(mode); retvalue = 0; end: @@ -220,7 +314,7 @@ lobject_write(lobjectObject *self, const char *buf, size_t len) PGresult *pgres = NULL; char *error = NULL; - Dprintf("lobject_writing: fd = %d, len = " FORMAT_CODE_PY_SSIZE_T, + Dprintf("lobject_writing: fd = %d, len = " FORMAT_CODE_SIZE_T, self->fd, len); Py_BEGIN_ALLOW_THREADS; @@ -355,7 +449,7 @@ lobject_truncate(lobjectObject *self, size_t len) PGresult *pgres = NULL; char *error = NULL; - Dprintf("lobject_truncate: fd = %d, len = " FORMAT_CODE_PY_SSIZE_T, + Dprintf("lobject_truncate: fd = %d, len = " FORMAT_CODE_SIZE_T, self->fd, len); Py_BEGIN_ALLOW_THREADS; diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 8aefa74e..49f64bab 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -23,20 +23,16 @@ * License for more details. */ -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/lobject.h" #include "psycopg/connection.h" #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" #include "psycopg/pqpath.h" -#include "pgversion.h" + +#include #ifdef PSYCOPG_EXTENSIONS @@ -75,18 +71,48 @@ psyco_lobj_close(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_write(lobjectObject *self, PyObject *args) { - int len, res=0; - const char *buffer; + char *buffer; + Py_ssize_t len; + Py_ssize_t res; + PyObject *obj; + PyObject *data = NULL; + PyObject *rv = NULL; - if (!PyArg_ParseTuple(args, "s#", &buffer, &len)) return NULL; + if (!PyArg_ParseTuple(args, "O", &obj)) return NULL; EXC_IF_LOBJ_CLOSED(self); EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); - if ((res = lobject_write(self, buffer, len)) < 0) return NULL; + if (Bytes_Check(obj)) { + Py_INCREF(obj); + data = obj; + } + else if (PyUnicode_Check(obj)) { + if (!(data = PyUnicode_AsEncodedString(obj, self->conn->codec, NULL))) { + goto exit; + } + } + else { + PyErr_Format(PyExc_TypeError, + "lobject.write requires a string; got %s instead", + Py_TYPE(obj)->tp_name); + goto exit; + } - return PyInt_FromLong((long)res); + if (-1 == Bytes_AsStringAndSize(data, &buffer, &len)) { + goto exit; + } + + if (0 > (res = lobject_write(self, buffer, (size_t)len))) { + goto exit; + } + + rv = PyInt_FromLong((long)res); + +exit: + Py_XDECREF(data); + return rv; } /* read method - read data from the lobject */ @@ -123,9 +149,13 @@ psyco_lobj_read(lobjectObject *self, PyObject *args) return NULL; } - res = PyString_FromStringAndSize(buffer, size); + if (self->mode & LOBJECT_BINARY) { + res = Bytes_FromStringAndSize(buffer, size); + } else { + res = PyUnicode_Decode(buffer, size, self->conn->codec, NULL); + } PyMem_Free(buffer); - + return res; } @@ -277,10 +307,10 @@ static struct PyMethodDef lobjectObject_methods[] = { /* object member list */ static struct PyMemberDef lobjectObject_members[] = { - {"oid", T_UINT, offsetof(lobjectObject, oid), RO, + {"oid", T_UINT, offsetof(lobjectObject, oid), READONLY, "The backend OID associated to this lobject."}, - {"mode", T_STRING, offsetof(lobjectObject, smode), RO, - "Open mode ('r', 'w', 'rw' or 'n')."}, + {"mode", T_STRING, offsetof(lobjectObject, smode), READONLY, + "Open mode."}, {NULL} }; @@ -296,7 +326,7 @@ static struct PyGetSetDef lobjectObject_getsets[] = { static int lobject_setup(lobjectObject *self, connectionObject *conn, - Oid oid, int mode, Oid new_oid, const char *new_file) + Oid oid, const char *smode, Oid new_oid, const char *new_file) { Dprintf("lobject_setup: init lobject object at %p", self); @@ -314,11 +344,11 @@ lobject_setup(lobjectObject *self, connectionObject *conn, self->fd = -1; self->oid = InvalidOid; - if (lobject_open(self, conn, oid, mode, new_oid, new_file) == -1) + if (lobject_open(self, conn, oid, smode, new_oid, new_file) == -1) return -1; Dprintf("lobject_setup: good lobject object at %p, refcnt = " - FORMAT_CODE_PY_SSIZE_T, self, ((PyObject *)self)->ob_refcnt); + FORMAT_CODE_PY_SSIZE_T, self, Py_REFCNT(self)); Dprintf("lobject_setup: oid = %d, fd = %d", self->oid, self->fd); return 0; } @@ -331,27 +361,28 @@ lobject_dealloc(PyObject* obj) if (lobject_close(self) < 0) PyErr_Print(); Py_XDECREF((PyObject*)self->conn); + PyMem_Free(self->smode); Dprintf("lobject_dealloc: deleted lobject object at %p, refcnt = " - FORMAT_CODE_PY_SSIZE_T, obj, obj->ob_refcnt); + FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int lobject_init(PyObject *obj, PyObject *args, PyObject *kwds) { Oid oid=InvalidOid, new_oid=InvalidOid; - int mode=0; + const char *smode = ""; const char *new_file = NULL; PyObject *conn; - if (!PyArg_ParseTuple(args, "O|iiis", - &conn, &oid, &mode, &new_oid, &new_file)) + if (!PyArg_ParseTuple(args, "O|iziz", + &conn, &oid, &smode, &new_oid, &new_file)) return -1; return lobject_setup((lobjectObject *)obj, - (connectionObject *)conn, oid, mode, new_oid, new_file); + (connectionObject *)conn, oid, smode, new_oid, new_file); } static PyObject * @@ -380,8 +411,7 @@ lobject_repr(lobjectObject *self) "A database large object." PyTypeObject lobjectType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.lobject", sizeof(lobjectObject), 0, diff --git a/psycopg/microprotocols.c b/psycopg/microprotocols.c index a00f20d2..f48c0fbf 100644 --- a/psycopg/microprotocols.c +++ b/psycopg/microprotocols.c @@ -23,18 +23,13 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" -#include "psycopg/cursor.h" -#include "psycopg/connection.h" + #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" +#include "psycopg/cursor.h" +#include "psycopg/connection.h" /** the adapters registry **/ @@ -87,8 +82,12 @@ _get_superclass_adapter(PyObject *obj, PyObject *proto) PyObject *key, *adapter; Py_ssize_t i, ii; - type = (PyTypeObject *)Py_TYPE(obj); - if (!((Py_TPFLAGS_HAVE_CLASS & type->tp_flags) && type->tp_mro)) { + type = Py_TYPE(obj); + if (!( +#if PY_MAJOR_VERSION < 3 + (Py_TPFLAGS_HAVE_CLASS & type->tp_flags) && +#endif + type->tp_mro)) { /* has no mro */ return NULL; } @@ -136,7 +135,8 @@ microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) because the ISQLQuote type is abstract and there is no way to get a quotable object to be its instance */ - Dprintf("microprotocols_adapt: trying to adapt %s", obj->ob_type->tp_name); + Dprintf("microprotocols_adapt: trying to adapt %s", + Py_TYPE(obj)->tp_name); /* look for an adapter in the registry */ key = PyTuple_Pack(2, Py_TYPE(obj), proto); @@ -192,12 +192,16 @@ microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) } /* else set the right exception and return NULL */ - PyOS_snprintf(buffer, 255, "can't adapt type '%s'", obj->ob_type->tp_name); + PyOS_snprintf(buffer, 255, "can't adapt type '%s'", + Py_TYPE(obj)->tp_name); psyco_set_error(ProgrammingError, NULL, buffer, NULL, NULL); return NULL; } -/* microprotocol_getquoted - utility function that adapt and call getquoted */ +/* microprotocol_getquoted - utility function that adapt and call getquoted. + * + * Return a bytes string, NULL on error. + */ PyObject * microprotocol_getquoted(PyObject *obj, connectionObject *conn) @@ -211,7 +215,7 @@ microprotocol_getquoted(PyObject *obj, connectionObject *conn) } Dprintf("microprotocol_getquoted: adapted to %s", - adapted->ob_type->tp_name); + Py_TYPE(adapted)->tp_name); /* if requested prepare the object passing it the connection */ if (conn) { @@ -235,6 +239,16 @@ microprotocol_getquoted(PyObject *obj, connectionObject *conn) adapted to the right protocol) */ res = PyObject_CallMethod(adapted, "getquoted", NULL); + /* Convert to bytes. */ + if (res && PyUnicode_CheckExact(res)) { + PyObject *b; + const char *codec; + codec = (conn && conn->codec) ? conn->codec : "utf8"; + b = PyUnicode_AsEncodedString(res, codec, NULL); + Py_DECREF(res); + res = b; + } + exit: Py_XDECREF(adapted); Py_XDECREF(prepare); diff --git a/psycopg/microprotocols.h b/psycopg/microprotocols.h index 08e86104..df084c58 100644 --- a/psycopg/microprotocols.h +++ b/psycopg/microprotocols.h @@ -26,9 +26,6 @@ #ifndef PSYCOPG_MICROPROTOCOLS_H #define PSYCOPG_MICROPROTOCOLS_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include "psycopg/config.h" #include "psycopg/connection.h" #include "psycopg/cursor.h" diff --git a/psycopg/microprotocols_proto.c b/psycopg/microprotocols_proto.c index 1c17a85a..775889d9 100644 --- a/psycopg/microprotocols_proto.c +++ b/psycopg/microprotocols_proto.c @@ -23,19 +23,13 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/microprotocols_proto.h" #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/microprotocols_proto.h" - /** void protocol implementation **/ @@ -99,7 +93,7 @@ static struct PyMethodDef isqlquoteObject_methods[] = { static struct PyMemberDef isqlquoteObject_members[] = { /* DBAPI-2.0 extensions (exception objects) */ - {"_wrapped", T_OBJECT, offsetof(isqlquoteObject, wrapped), RO}, + {"_wrapped", T_OBJECT, offsetof(isqlquoteObject, wrapped), READONLY}, {NULL} }; @@ -121,7 +115,7 @@ isqlquote_dealloc(PyObject* obj) Py_XDECREF(self->wrapped); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -156,8 +150,7 @@ isqlquote_del(PyObject* self) "returning the SQL representation of the object.\n\n" PyTypeObject isqlquoteType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.ISQLQuote", sizeof(isqlquoteObject), 0, diff --git a/psycopg/microprotocols_proto.h b/psycopg/microprotocols_proto.h index d7a67f24..614a2637 100644 --- a/psycopg/microprotocols_proto.h +++ b/psycopg/microprotocols_proto.h @@ -26,12 +26,6 @@ #ifndef PSYCOPG_ISQLQUOTE_H #define PSYCOPG_ISQLQUOTE_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/notify.h b/psycopg/notify.h index 1490fb41..645fdd73 100644 --- a/psycopg/notify.h +++ b/psycopg/notify.h @@ -26,10 +26,6 @@ #ifndef PSYCOPG_NOTIFY_H #define PSYCOPG_NOTIFY_H 1 -#include - -#include "psycopg/config.h" - extern HIDDEN PyTypeObject NotifyType; typedef struct { diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index 53969709..ec2ec36f 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -23,16 +23,12 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/notify.h" + static const char notify_doc[] = "A notification received from the backend.\n\n" "`!Notify` instances are made available upon reception on the\n" @@ -56,9 +52,9 @@ static const char payload_doc[] = "of the server this member is always the empty string."; static PyMemberDef notify_members[] = { - { "pid", T_OBJECT, offsetof(NotifyObject, pid), RO, (char *)pid_doc }, - { "channel", T_OBJECT, offsetof(NotifyObject, channel), RO, (char *)channel_doc }, - { "payload", T_OBJECT, offsetof(NotifyObject, payload), RO, (char *)payload_doc }, + { "pid", T_OBJECT, offsetof(NotifyObject, pid), READONLY, (char *)pid_doc }, + { "channel", T_OBJECT, offsetof(NotifyObject, channel), READONLY, (char *)channel_doc }, + { "payload", T_OBJECT, offsetof(NotifyObject, payload), READONLY, (char *)payload_doc }, { NULL } }; @@ -82,7 +78,7 @@ notify_init(NotifyObject *self, PyObject *args, PyObject *kwargs) } if (!payload) { - payload = PyString_FromStringAndSize("", 0); + payload = Text_FromUTF8(""); } Py_CLEAR(self->pid); @@ -116,7 +112,7 @@ notify_dealloc(NotifyObject *self) Py_CLEAR(self->channel); Py_CLEAR(self->payload); - self->ob_type->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free((PyObject *)self); } static void @@ -217,7 +213,7 @@ notify_repr(NotifyObject *self) PyObject *format = NULL; PyObject *args = NULL; - if (!(format = PyString_FromString("Notify(%r, %r, %r)"))) { + if (!(format = Text_FromUTF8("Notify(%r, %r, %r)"))) { goto exit; } @@ -229,7 +225,7 @@ notify_repr(NotifyObject *self) Py_INCREF(self->payload); PyTuple_SET_ITEM(args, 2, self->payload); - rv = PyString_Format(format, args); + rv = Text_Format(format, args); exit: Py_XDECREF(args); @@ -280,8 +276,7 @@ static PySequenceMethods notify_sequence = { PyTypeObject NotifyType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2.extensions.Notify", sizeof(NotifyObject), 0, diff --git a/psycopg/pgversion.h b/psycopg/pgversion.h deleted file mode 100644 index f874a9ae..00000000 --- a/psycopg/pgversion.h +++ /dev/null @@ -1,2 +0,0 @@ -#define PG_VERSION_MAJOR 7 -#define PG_VERSION_MINOR 4 diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 6c7fabf4..7f9a1013 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -29,21 +29,17 @@ connection. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/pqpath.h" #include "psycopg/connection.h" #include "psycopg/cursor.h" #include "psycopg/green.h" #include "psycopg/typecast.h" #include "psycopg/pgtypes.h" -#include "psycopg/pgversion.h" + +#include /* Strip off the severity from a Postgres error message. */ @@ -976,14 +972,14 @@ _pq_fetch_tuples(cursorObject *curs) } Dprintf("_pq_fetch_tuples: using cast at %p (%s) for type %d", - cast, PyString_AS_STRING(((typecastObject*)cast)->name), + cast, Bytes_AS_STRING(((typecastObject*)cast)->name), PQftype(curs->pgres,i)); Py_INCREF(cast); PyTuple_SET_ITEM(curs->casts, i, cast); /* 1/ fill the other fields */ PyTuple_SET_ITEM(dtitem, 0, - PyString_FromString(PQfname(curs->pgres, i))); + conn_text_from_chars(curs->conn, PQfname(curs->pgres, i))); PyTuple_SET_ITEM(dtitem, 1, type); /* 2/ display size is the maximum size of this field result tuples. */ @@ -1061,19 +1057,50 @@ _pq_copy_in_v3(cursorObject *curs) } while (1) { - o = PyObject_CallFunctionObjArgs(func, size, NULL); - if (!(o && PyString_Check(o) && (length = PyString_GET_SIZE(o)) != -1)) { + if (!(o = PyObject_CallFunctionObjArgs(func, size, NULL))) { + Dprintf("_pq_copy_in_v3: read() failed"); error = 1; + break; + } + + /* a file may return unicode in Py3: encode in client encoding. */ +#if PY_MAJOR_VERSION > 2 + if (PyUnicode_Check(o)) { + PyObject *tmp; + if (!(tmp = PyUnicode_AsEncodedString(o, curs->conn->codec, NULL))) { + Dprintf("_pq_copy_in_v3: encoding() failed"); + error = 1; + break; + } + Py_DECREF(o); + o = tmp; + } +#endif + + if (!Bytes_Check(o)) { + Dprintf("_pq_copy_in_v3: got %s instead of bytes", + Py_TYPE(o)->tp_name); + error = 1; + break; + } + + if (0 == (length = Bytes_GET_SIZE(o))) { + break; + } + if (length > INT_MAX) { + Dprintf("_pq_copy_in_v3: bad length: " FORMAT_CODE_PY_SSIZE_T, + length); + error = 1; + break; } - if (length == 0 || length > INT_MAX || error == 1) break; Py_BEGIN_ALLOW_THREADS; - res = PQputCopyData(curs->conn->pgconn, PyString_AS_STRING(o), + res = PQputCopyData(curs->conn->pgconn, Bytes_AS_STRING(o), /* Py_ssize_t->int cast was validated above */ (int) length); - Dprintf("_pq_copy_in_v3: sent %d bytes of data; res = %d", - (int) length, res); - + Dprintf("_pq_copy_in_v3: sent " FORMAT_CODE_PY_SSIZE_T " bytes of data; res = %d", + length, res); + if (res == 0) { /* FIXME: in theory this should not happen but adding a check here would be a nice idea */ @@ -1101,6 +1128,7 @@ _pq_copy_in_v3(cursorObject *curs) else if (error == 2) res = PQputCopyEnd(curs->conn->pgconn, "error in PQputCopyData() call"); else + /* XXX would be nice to propagate the exeption */ res = PQputCopyEnd(curs->conn->pgconn, "error in .read() call"); IFCLEARPGRES(curs->pgres); @@ -1135,7 +1163,9 @@ static int _pq_copy_out_v3(cursorObject *curs) { PyObject *tmp = NULL, *func; + PyObject *obj = NULL; int ret = -1; + int is_text; char *buffer; Py_ssize_t len; @@ -1145,14 +1175,28 @@ _pq_copy_out_v3(cursorObject *curs) goto exit; } + /* if the file is text we must pass it unicode. */ + if (-1 == (is_text = psycopg_is_text_file(curs->copyfile))) { + goto exit; + } + while (1) { Py_BEGIN_ALLOW_THREADS; len = PQgetCopyData(curs->conn->pgconn, &buffer, 0); Py_END_ALLOW_THREADS; if (len > 0 && buffer) { - tmp = PyObject_CallFunction(func, "s#", buffer, len); + if (is_text) { + obj = PyUnicode_Decode(buffer, len, curs->conn->codec, NULL); + } else { + obj = Bytes_FromStringAndSize(buffer, len); + } + PQfreemem(buffer); + if (!obj) { goto exit; } + tmp = PyObject_CallFunctionObjArgs(func, obj, NULL); + Py_DECREF(obj); + if (tmp == NULL) { goto exit; } else { @@ -1215,7 +1259,7 @@ pq_fetch(cursorObject *curs) /* backend status message */ Py_XDECREF(curs->pgstatus); - curs->pgstatus = PyString_FromString(PQcmdStatus(curs->pgres)); + curs->pgstatus = conn_text_from_chars(curs->conn, PQcmdStatus(curs->pgres)); switch(pgstatus) { diff --git a/psycopg/pqpath.h b/psycopg/pqpath.h index 51a640dc..8e9e20a2 100644 --- a/psycopg/pqpath.h +++ b/psycopg/pqpath.h @@ -26,7 +26,6 @@ #ifndef PSYCOPG_PQPATH_H #define PSYCOPG_PQPATH_H 1 -#include "psycopg/config.h" #include "psycopg/cursor.h" #include "psycopg/connection.h" diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index d023e47e..8edc42ed 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -120,6 +120,11 @@ HIDDEN void psyco_set_error(PyObject *exc, PyObject *curs, const char *msg, HIDDEN char *psycopg_escape_string(PyObject *conn, const char *from, Py_ssize_t len, char *to, Py_ssize_t *tolen); +HIDDEN char *psycopg_strdup(const char *from, Py_ssize_t len); +HIDDEN PyObject * psycopg_ensure_bytes(PyObject *obj); +HIDDEN PyObject * psycopg_ensure_text(PyObject *obj); +HIDDEN int psycopg_is_text_file(PyObject *f); + /* Exceptions docstrings */ #define Error_doc \ "Base class for error exceptions." diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index db44807f..f91483df 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -23,13 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/cursor.h" #include "psycopg/green.h" @@ -130,17 +126,38 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) return NULL; } +#if PY_MAJOR_VERSION < 3 if (pyport && PyString_Check(pyport)) { - PyObject *pyint = PyInt_FromString(PyString_AsString(pyport), NULL, 10); - if (!pyint) goto fail; - /* Must use PyInt_AsLong rather than PyInt_AS_LONG, because - * PyInt_FromString can return a PyLongObject: */ - iport = PyInt_AsLong(pyint); - Py_DECREF(pyint); + PyObject *pyint = PyInt_FromString(PyString_AsString(pyport), NULL, 10); + if (!pyint) goto fail; + /* Must use PyInt_AsLong rather than PyInt_AS_LONG, because + * PyInt_FromString can return a PyLongObject: */ + iport = PyInt_AsLong(pyint); + Py_DECREF(pyint); + if (iport == -1 && PyErr_Occurred()) + goto fail; } else if (pyport && PyInt_Check(pyport)) { - iport = PyInt_AsLong(pyport); + iport = PyInt_AsLong(pyport); + if (iport == -1 && PyErr_Occurred()) + goto fail; } +#else + if (pyport && PyUnicode_Check(pyport)) { + PyObject *pyint = PyObject_CallFunction((PyObject*)&PyLong_Type, + "Oi", pyport, 10); + if (!pyint) goto fail; + iport = PyLong_AsLong(pyint); + Py_DECREF(pyint); + if (iport == -1 && PyErr_Occurred()) + goto fail; + } + else if (pyport && PyLong_Check(pyport)) { + iport = PyLong_AsLong(pyport); + if (iport == -1 && PyErr_Occurred()) + goto fail; + } +#endif else if (pyport != NULL) { PyErr_SetString(PyExc_TypeError, "port must be a string or int"); goto fail; @@ -292,13 +309,23 @@ psyco_adapters_init(PyObject *mod) PyTypeObject *type; microprotocols_add(&PyFloat_Type, NULL, (PyObject*)&pfloatType); +#if PY_MAJOR_VERSION < 3 microprotocols_add(&PyInt_Type, NULL, (PyObject*)&asisType); +#endif microprotocols_add(&PyLong_Type, NULL, (PyObject*)&asisType); microprotocols_add(&PyBool_Type, NULL, (PyObject*)&pbooleanType); +#if PY_MAJOR_VERSION < 3 microprotocols_add(&PyString_Type, NULL, (PyObject*)&qstringType); +#endif microprotocols_add(&PyUnicode_Type, NULL, (PyObject*)&qstringType); +#if PY_MAJOR_VERSION < 3 microprotocols_add(&PyBuffer_Type, NULL, (PyObject*)&binaryType); +#else + microprotocols_add(&PyBytes_Type, NULL, (PyObject*)&binaryType); + microprotocols_add(&PyByteArray_Type, NULL, (PyObject*)&binaryType); + microprotocols_add(&PyMemoryView_Type, NULL, (PyObject*)&binaryType); +#endif microprotocols_add(&PyList_Type, NULL, (PyObject*)&listType); if ((type = (PyTypeObject*)psyco_GetDecimalType()) != NULL) @@ -329,7 +356,32 @@ psyco_adapters_init(PyObject *mod) Fill the module's postgresql<->python encoding table */ static encodingPair encodings[] = { - {"SQL_ASCII", "ascii"}, + {"ABC", "cp1258"}, + {"ALT", "cp866"}, + {"BIG5", "big5"}, + {"EUC_CN", "euccn"}, + {"EUC_JIS_2004", "euc_jis_2004"}, + {"EUC_JP", "euc_jp"}, + {"EUC_KR", "euc_kr"}, + {"GB18030", "gb18030"}, + {"GBK", "gbk"}, + {"ISO_8859_1", "iso8859_1"}, + {"ISO_8859_2", "iso8859_2"}, + {"ISO_8859_3", "iso8859_3"}, + {"ISO_8859_5", "iso8859_5"}, + {"ISO_8859_6", "iso8859_6"}, + {"ISO_8859_7", "iso8859_7"}, + {"ISO_8859_8", "iso8859_8"}, + {"ISO_8859_9", "iso8859_9"}, + {"ISO_8859_10", "iso8859_10"}, + {"ISO_8859_13", "iso8859_13"}, + {"ISO_8859_14", "iso8859_14"}, + {"ISO_8859_15", "iso8859_15"}, + {"ISO_8859_16", "iso8859_16"}, + {"JOHAB", "johab"}, + {"KOI8", "koi8_r"}, + {"KOI8R", "koi8_r"}, + {"KOI8U", "koi8_u"}, {"LATIN1", "iso8859_1"}, {"LATIN2", "iso8859_2"}, {"LATIN3", "iso8859_3"}, @@ -339,46 +391,30 @@ static encodingPair encodings[] = { {"LATIN7", "iso8859_13"}, {"LATIN8", "iso8859_14"}, {"LATIN9", "iso8859_15"}, - {"ISO88591", "iso8859_1"}, - {"ISO88592", "iso8859_2"}, - {"ISO88593", "iso8859_3"}, - {"ISO88595", "iso8859_5"}, - {"ISO88596", "iso8859_6"}, - {"ISO88597", "iso8859_7"}, - {"ISO885913", "iso8859_13"}, - {"ISO88598", "iso8859_8"}, - {"ISO88599", "iso8859_9"}, - {"ISO885914", "iso8859_14"}, - {"ISO885915", "iso8859_15"}, - {"UNICODE", "utf_8"}, /* Not valid in 8.2, backward compatibility */ - {"UTF8", "utf_8"}, - {"WIN950", "cp950"}, - {"Windows950", "cp950"}, - {"BIG5", "big5"}, - {"EUC_JP", "euc_jp"}, - {"EUC_KR", "euc_kr"}, - {"GB18030", "gb18030"}, - {"GBK", "gbk"}, - {"WIN936", "gbk"}, - {"Windows936", "gbk"}, - {"JOHAB", "johab"}, - {"KOI8", "koi8_r"}, /* in PG: KOI8 == KOI8R == KOI8-R == KOI8-U - but in Python there is koi8_r AND koi8_u */ - {"KOI8R", "koi8_r"}, - {"SJIS", "cp932"}, + {"LATIN10", "iso8859_16"}, {"Mskanji", "cp932"}, {"ShiftJIS", "cp932"}, - {"WIN932", "cp932"}, - {"Windows932", "cp932"}, + {"SHIFT_JIS_2004", "shift_jis_2004"}, + {"SJIS", "cp932"}, + {"SQL_ASCII", "ascii"}, /* XXX this is wrong: SQL_ASCII means "no + * encoding" we should fix the unicode + * typecaster to return a str or bytes in Py3 + */ + {"TCVN", "cp1258"}, + {"TCVN5712", "cp1258"}, {"UHC", "cp949"}, - {"WIN949", "cp949"}, - {"Windows949", "cp949"}, + {"UNICODE", "utf_8"}, /* Not valid in 8.2, backward compatibility */ + {"UTF8", "utf_8"}, + {"VSCII", "cp1258"}, + {"WIN", "cp1251"}, {"WIN866", "cp866"}, - {"ALT", "cp866"}, {"WIN874", "cp874"}, + {"WIN932", "cp932"}, + {"WIN936", "gbk"}, + {"WIN949", "cp949"}, + {"WIN950", "cp950"}, {"WIN1250", "cp1250"}, {"WIN1251", "cp1251"}, - {"WIN", "cp1251"}, {"WIN1252", "cp1252"}, {"WIN1253", "cp1253"}, {"WIN1254", "cp1254"}, @@ -386,16 +422,13 @@ static encodingPair encodings[] = { {"WIN1256", "cp1256"}, {"WIN1257", "cp1257"}, {"WIN1258", "cp1258"}, - {"ABC", "cp1258"}, - {"TCVN", "cp1258"}, - {"TCVN5712", "cp1258"}, - {"VSCII", "cp1258"}, + {"Windows932", "cp932"}, + {"Windows936", "gbk"}, + {"Windows949", "cp949"}, + {"Windows950", "cp950"}, /* those are missing from Python: */ -/* {"EUC_CN", "?"}, */ /* {"EUC_TW", "?"}, */ -/* {"LATIN10", "?"}, */ -/* {"ISO885916", "?"}, */ /* {"MULE_INTERNAL", "?"}, */ {NULL, NULL} }; @@ -405,7 +438,7 @@ static void psyco_encodings_fill(PyObject *dict) encodingPair *enc; for (enc = encodings; enc->pgenc != NULL; enc++) { - PyObject *value = PyString_FromString(enc->pyenc); + PyObject *value = Text_FromUTF8(enc->pyenc); PyDict_SetItemString(dict, enc->pgenc, value); Py_DECREF(value); } @@ -470,12 +503,18 @@ psyco_errors_init(void) dict = PyDict_New(); if (exctable[i].docstr) { - str = PyString_FromString(exctable[i].docstr); + str = Text_FromUTF8(exctable[i].docstr); PyDict_SetItemString(dict, "__doc__", str); } - if (exctable[i].base == 0) + if (exctable[i].base == 0) { + #if PY_MAJOR_VERSION < 3 base = PyExc_StandardError; + #else + /* StandardError is gone in 3.0 */ + base = NULL; + #endif + } else base = *exctable[i].base; @@ -543,23 +582,26 @@ psyco_set_error(PyObject *exc, PyObject *curs, const char *msg, PyObject *err = PyObject_CallFunction(exc, "s", msg); if (err) { + connectionObject *conn = NULL; + if (curs) { + PyObject_SetAttrString(err, "cursor", curs); + conn = ((cursorObject *)curs)->conn; + } + if (pgerror) { - t = PyString_FromString(pgerror); + t = conn_text_from_chars(conn, pgerror); PyObject_SetAttrString(err, "pgerror", t); Py_DECREF(t); } if (pgcode) { - t = PyString_FromString(pgcode); + t = conn_text_from_chars(conn, pgcode); PyObject_SetAttrString(err, "pgcode", t); Py_DECREF(t); } - if (curs) - PyObject_SetAttrString(err, "cursor", curs); - PyErr_SetObject(exc, err); - Py_DECREF(err); + Py_DECREF(err); } } @@ -701,13 +743,29 @@ static PyMethodDef psycopgMethods[] = { {NULL, NULL, 0, NULL} /* Sentinel */ }; -PyMODINIT_FUNC -init_psycopg(void) -{ - static void *PSYCOPG_API[PSYCOPG_API_pointers]; +#if PY_MAJOR_VERSION > 2 +static struct PyModuleDef psycopgmodule = { + PyModuleDef_HEAD_INIT, + "_psycopg", + NULL, + -1, + psycopgMethods, + NULL, + NULL, + NULL, + NULL +}; +#endif - PyObject *module, *dict; +PyMODINIT_FUNC +INIT_MODULE(_psycopg)(void) +{ +#if PY_VERSION_HEX < 0x03020000 + static void *PSYCOPG_API[PSYCOPG_API_pointers]; PyObject *c_api_object; +#endif + + PyObject *module = NULL, *dict; #ifdef PSYCOPG_DEBUG if (getenv("PSYCOPG_DEBUG")) @@ -717,51 +775,51 @@ init_psycopg(void) Dprintf("initpsycopg: initializing psycopg %s", PSYCOPG_VERSION); /* initialize all the new types and then the module */ - connectionType.ob_type = &PyType_Type; - cursorType.ob_type = &PyType_Type; - typecastType.ob_type = &PyType_Type; - qstringType.ob_type = &PyType_Type; - binaryType.ob_type = &PyType_Type; - isqlquoteType.ob_type = &PyType_Type; - pbooleanType.ob_type = &PyType_Type; - pfloatType.ob_type = &PyType_Type; - pdecimalType.ob_type = &PyType_Type; - asisType.ob_type = &PyType_Type; - listType.ob_type = &PyType_Type; - chunkType.ob_type = &PyType_Type; - NotifyType.ob_type = &PyType_Type; - XidType.ob_type = &PyType_Type; + Py_TYPE(&connectionType) = &PyType_Type; + Py_TYPE(&cursorType) = &PyType_Type; + Py_TYPE(&typecastType) = &PyType_Type; + Py_TYPE(&qstringType) = &PyType_Type; + Py_TYPE(&binaryType) = &PyType_Type; + Py_TYPE(&isqlquoteType) = &PyType_Type; + Py_TYPE(&pbooleanType) = &PyType_Type; + Py_TYPE(&pfloatType) = &PyType_Type; + Py_TYPE(&pdecimalType) = &PyType_Type; + Py_TYPE(&asisType) = &PyType_Type; + Py_TYPE(&listType) = &PyType_Type; + Py_TYPE(&chunkType) = &PyType_Type; + Py_TYPE(&NotifyType) = &PyType_Type; + Py_TYPE(&XidType) = &PyType_Type; - if (PyType_Ready(&connectionType) == -1) return; - if (PyType_Ready(&cursorType) == -1) return; - if (PyType_Ready(&typecastType) == -1) return; - if (PyType_Ready(&qstringType) == -1) return; - if (PyType_Ready(&binaryType) == -1) return; - if (PyType_Ready(&isqlquoteType) == -1) return; - if (PyType_Ready(&pbooleanType) == -1) return; - if (PyType_Ready(&pfloatType) == -1) return; - if (PyType_Ready(&pdecimalType) == -1) return; - if (PyType_Ready(&asisType) == -1) return; - if (PyType_Ready(&listType) == -1) return; - if (PyType_Ready(&chunkType) == -1) return; - if (PyType_Ready(&NotifyType) == -1) return; - if (PyType_Ready(&XidType) == -1) return; + if (PyType_Ready(&connectionType) == -1) goto exit; + if (PyType_Ready(&cursorType) == -1) goto exit; + if (PyType_Ready(&typecastType) == -1) goto exit; + if (PyType_Ready(&qstringType) == -1) goto exit; + if (PyType_Ready(&binaryType) == -1) goto exit; + if (PyType_Ready(&isqlquoteType) == -1) goto exit; + if (PyType_Ready(&pbooleanType) == -1) goto exit; + if (PyType_Ready(&pfloatType) == -1) goto exit; + if (PyType_Ready(&pdecimalType) == -1) goto exit; + if (PyType_Ready(&asisType) == -1) goto exit; + if (PyType_Ready(&listType) == -1) goto exit; + if (PyType_Ready(&chunkType) == -1) goto exit; + if (PyType_Ready(&NotifyType) == -1) goto exit; + if (PyType_Ready(&XidType) == -1) goto exit; #ifdef PSYCOPG_EXTENSIONS - lobjectType.ob_type = &PyType_Type; - if (PyType_Ready(&lobjectType) == -1) return; + Py_TYPE(&lobjectType) = &PyType_Type; + if (PyType_Ready(&lobjectType) == -1) goto exit; #endif /* import mx.DateTime module, if necessary */ #ifdef HAVE_MXDATETIME - mxdatetimeType.ob_type = &PyType_Type; - if (PyType_Ready(&mxdatetimeType) == -1) return; + Py_TYPE(&mxdatetimeType) = &PyType_Type; + if (PyType_Ready(&mxdatetimeType) == -1) goto exit; if (mxDateTime_ImportModuleAndAPI() != 0) { Dprintf("initpsycopg: why marc hide mx.DateTime again?!"); PyErr_SetString(PyExc_ImportError, "can't import mx.DateTime module"); - return; + goto exit; } - if (psyco_adapter_mxdatetime_init()) { return; } + if (psyco_adapter_mxdatetime_init()) { goto exit; } #endif /* import python builtin datetime module, if available */ @@ -769,22 +827,22 @@ init_psycopg(void) if (pyDateTimeModuleP == NULL) { Dprintf("initpsycopg: can't import datetime module"); PyErr_SetString(PyExc_ImportError, "can't import datetime module"); - return; + goto exit; } /* Initialize the PyDateTimeAPI everywhere is used */ PyDateTime_IMPORT; - if (psyco_adapter_datetime_init()) { return; } + if (psyco_adapter_datetime_init()) { goto exit; } - pydatetimeType.ob_type = &PyType_Type; - if (PyType_Ready(&pydatetimeType) == -1) return; + Py_TYPE(&pydatetimeType) = &PyType_Type; + if (PyType_Ready(&pydatetimeType) == -1) goto exit; /* import psycopg2.tz anyway (TODO: replace with C-level module?) */ pyPsycopgTzModule = PyImport_ImportModule("psycopg2.tz"); if (pyPsycopgTzModule == NULL) { Dprintf("initpsycopg: can't import psycopg2.tz module"); PyErr_SetString(PyExc_ImportError, "can't import psycopg2.tz module"); - return; + goto exit; } pyPsycopgTzLOCAL = PyObject_GetAttrString(pyPsycopgTzModule, "LOCAL"); @@ -792,16 +850,25 @@ init_psycopg(void) PyObject_GetAttrString(pyPsycopgTzModule, "FixedOffsetTimezone"); /* initialize the module and grab module's dictionary */ +#if PY_MAJOR_VERSION < 3 module = Py_InitModule("_psycopg", psycopgMethods); +#else + module = PyModule_Create(&psycopgmodule); +#endif + if (!module) { goto exit; } + dict = PyModule_GetDict(module); /* initialize all the module's exported functions */ /* PyBoxer_API[PyBoxer_Fake_NUM] = (void *)PyBoxer_Fake; */ /* Create a CObject containing the API pointer array's address */ + /* If anybody asks for a PyCapsule we'll deal with it. */ +#if PY_VERSION_HEX < 0x03020000 c_api_object = PyCObject_FromVoidPtr((void *)PSYCOPG_API, NULL); if (c_api_object != NULL) PyModule_AddObject(module, "_C_API", c_api_object); +#endif /* other mixed initializations of module-level variables */ psycoEncodings = PyDict_New(); @@ -810,9 +877,9 @@ init_psycopg(void) /* set some module's parameters */ PyModule_AddStringConstant(module, "__version__", PSYCOPG_VERSION); PyModule_AddStringConstant(module, "__doc__", "psycopg PostgreSQL driver"); - PyModule_AddObject(module, "apilevel", PyString_FromString(APILEVEL)); + PyModule_AddObject(module, "apilevel", Text_FromUTF8(APILEVEL)); PyModule_AddObject(module, "threadsafety", PyInt_FromLong(THREADSAFETY)); - PyModule_AddObject(module, "paramstyle", PyString_FromString(PARAMSTYLE)); + PyModule_AddObject(module, "paramstyle", Text_FromUTF8(PARAMSTYLE)); /* put new types in module dictionary */ PyModule_AddObject(module, "connection", (PyObject*)&connectionType); @@ -864,4 +931,11 @@ init_psycopg(void) #endif Dprintf("initpsycopg: module initialization complete"); + +exit: +#if PY_MAJOR_VERSION > 2 + return module; +#else + return; +#endif } diff --git a/psycopg/python.h b/psycopg/python.h index f4256d8d..c3f9a0d1 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -26,9 +26,10 @@ #ifndef PSYCOPG_PYTHON_H #define PSYCOPG_PYTHON_H 1 -#define PY_SSIZE_T_CLEAN -#include #include +#if PY_MAJOR_VERSION < 3 +#include +#endif #if PY_VERSION_HEX < 0x02040000 # error "psycopg requires Python >= 2.4" @@ -53,8 +54,17 @@ #define CONV_CODE_PY_SSIZE_T "n" #endif -#ifndef Py_TYPE - #define Py_TYPE(o) (((PyObject*)(o))->ob_type) +/* Macros defined in Python 2.6 */ +#ifndef Py_REFCNT +#define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#define PyVarObject_HEAD_INIT(x,n) PyObject_HEAD_INIT(x) n, +#endif + +/* Missing at least in Python 2.4 */ +#ifndef Py_MEMCPY +#define Py_MEMCPY memcpy #endif /* FORMAT_CODE_PY_SSIZE_T is for Py_ssize_t: */ @@ -69,4 +79,73 @@ #define FORMAT_CODE_SIZE_T "%zu" #endif +/* Abstract from text type. Only supported for ASCII and UTF-8 */ +#if PY_MAJOR_VERSION < 3 +#define Text_Type PyString_Type +#define Text_Check(s) PyString_Check(s) +#define Text_Format(f,a) PyString_Format(f,a) +#define Text_FromUTF8(s) PyString_FromString(s) +#define Text_FromUTF8AndSize(s,n) PyString_FromStringAndSize(s,n) +#else +#define Text_Type PyUnicode_Type +#define Text_Check(s) PyUnicode_Check(s) +#define Text_Format(f,a) PyUnicode_Format(f,a) +#define Text_FromUTF8(s) PyUnicode_FromString(s) +#define Text_FromUTF8AndSize(s,n) PyUnicode_FromStringAndSize(s,n) +#endif + +#if PY_MAJOR_VERSION > 2 +#define PyInt_Type PyLong_Type +#define PyInt_AsLong PyLong_AsLong +#define PyInt_FromLong PyLong_FromLong +#define PyInt_FromSsize_t PyLong_FromSsize_t +#define PyString_FromFormat PyUnicode_FromFormat +#define Py_TPFLAGS_HAVE_ITER 0L +#define Py_TPFLAGS_HAVE_RICHCOMPARE 0L +#define Py_TPFLAGS_HAVE_WEAKREFS 0L +#ifndef PyNumber_Int +#define PyNumber_Int PyNumber_Long +#endif +#endif /* PY_MAJOR_VERSION > 2 */ + +#if PY_MAJOR_VERSION < 3 +#define Bytes_Type PyString_Type +#define Bytes_Check PyString_Check +#define Bytes_CheckExact PyString_CheckExact +#define Bytes_AS_STRING PyString_AS_STRING +#define Bytes_GET_SIZE PyString_GET_SIZE +#define Bytes_Size PyString_Size +#define Bytes_AsString PyString_AsString +#define Bytes_AsStringAndSize PyString_AsStringAndSize +#define Bytes_FromString PyString_FromString +#define Bytes_FromStringAndSize PyString_FromStringAndSize +#define Bytes_FromFormat PyString_FromFormat +#define _Bytes_Resize _PyString_Resize + +#else + +#define Bytes_Type PyBytes_Type +#define Bytes_Check PyBytes_Check +#define Bytes_CheckExact PyBytes_CheckExact +#define Bytes_AS_STRING PyBytes_AS_STRING +#define Bytes_GET_SIZE PyBytes_GET_SIZE +#define Bytes_Size PyBytes_Size +#define Bytes_AsString PyBytes_AsString +#define Bytes_AsStringAndSize PyBytes_AsStringAndSize +#define Bytes_FromString PyBytes_FromString +#define Bytes_FromStringAndSize PyBytes_FromStringAndSize +#define Bytes_FromFormat PyBytes_FromFormat +#define _Bytes_Resize _PyBytes_Resize + +#endif + +HIDDEN PyObject *Bytes_Format(PyObject *format, PyObject *args); + +/* Mangle the module name into the name of the module init function */ +#if PY_MAJOR_VERSION > 2 +#define INIT_MODULE(m) PyInit_ ## m +#else +#define INIT_MODULE(m) init ## m +#endif + #endif /* !defined(PSYCOPG_PYTHON_H) */ diff --git a/psycopg/typecast.c b/psycopg/typecast.c index 6da1effc..484ce45e 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -23,14 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" -#include "psycopg/python.h" + #include "psycopg/typecast.h" #include "psycopg/cursor.h" @@ -312,7 +307,7 @@ typecast_add(PyObject *obj, PyObject *dict, int binary) Dprintf("typecast_add: object at %p, values refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, type->values->ob_refcnt + obj, Py_REFCNT(type->values) ); if (dict == NULL) @@ -393,8 +388,8 @@ typecast_richcompare(PyObject *obj1, PyObject* obj2, int opid) } static struct PyMemberDef typecastObject_members[] = { - {"name", T_OBJECT, OFFSETOF(name), RO}, - {"values", T_OBJECT, OFFSETOF(values), RO}, + {"name", T_OBJECT, OFFSETOF(name), READONLY}, + {"values", T_OBJECT, OFFSETOF(values), READONLY}, {NULL} }; @@ -410,7 +405,7 @@ typecast_dealloc(PyObject *obj) Py_CLEAR(self->pcast); Py_CLEAR(self->bcast); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -431,32 +426,47 @@ typecast_del(void *self) PyObject_GC_Del(self); } +static PyObject * +typecast_repr(PyObject *self) +{ + PyObject *name = ((typecastObject *)self)->name; + PyObject *rv; + + Py_INCREF(name); + if (!(name = psycopg_ensure_bytes(name))) { + return NULL; + } + + rv = PyString_FromFormat("<%s '%s' at %p>", + Py_TYPE(self)->tp_name, Bytes_AS_STRING(name), self); + + Py_DECREF(name); + return rv; +} + static PyObject * typecast_call(PyObject *obj, PyObject *args, PyObject *kwargs) { - PyObject *string, *cursor; + char *string; + Py_ssize_t length; + PyObject *cursor; - if (!PyArg_ParseTuple(args, "OO", &string, &cursor)) { + if (!PyArg_ParseTuple(args, "z#O", &string, &length, &cursor)) { return NULL; } // If the string is not a string but a None value we're being called - // from a Python-defined caster. There is no need to convert, just - // return it. - - if (string == Py_None) { - Py_INCREF(string); - return string; + // from a Python-defined caster. + if (!string) { + Py_INCREF(Py_None); + return Py_None; } - return typecast_cast(obj, - PyString_AsString(string), PyString_Size(string), - cursor); + return typecast_cast(obj, string, length, cursor); } PyTypeObject typecastType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.type", sizeof(typecastObject), 0, @@ -466,7 +476,7 @@ PyTypeObject typecastType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ typecast_cmp, /*tp_compare*/ - 0, /*tp_repr*/ + typecast_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ @@ -524,7 +534,7 @@ typecast_new(PyObject *name, PyObject *values, PyObject *cast, PyObject *base) if (obj == NULL) return NULL; Dprintf("typecast_new: new type at = %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt); + obj, Py_REFCNT(obj)); Py_INCREF(values); obj->values = values; @@ -566,7 +576,7 @@ typecast_from_python(PyObject *self, PyObject *args, PyObject *keywds) if (!PyArg_ParseTupleAndKeywords(args, keywds, "O!|O!OO", kwlist, &PyTuple_Type, &v, - &PyString_Type, &name, + &Text_Type, &name, &cast, &base)) { return NULL; } @@ -591,7 +601,7 @@ typecast_from_c(typecastObject_initlist *type, PyObject *dict) } } - name = PyString_FromString(type->name); + name = Text_FromUTF8(type->name); if (!name) goto end; while (type->values[len] != 0) len++; @@ -630,7 +640,27 @@ typecast_cast(PyObject *obj, const char *str, Py_ssize_t len, PyObject *curs) res = self->ccast(str, len, curs); } else if (self->pcast) { - res = PyObject_CallFunction(self->pcast, "s#O", str, len, curs); + PyObject *s; + /* XXX we have bytes in the adapters and strings in the typecasters. + * are you sure this is ok? + * Notice that this way it is about impossible to create a python + * typecaster on a binary type. */ + if (str) { +#if PY_MAJOR_VERSION < 3 + s = PyString_FromStringAndSize(str, len); +#else + s = PyUnicode_Decode(str, len, + ((cursorObject *)curs)->conn->codec, NULL); +#endif + } + else { + Py_INCREF(Py_None); + s = Py_None; + } + if (s) { + res = PyObject_CallFunctionObjArgs(self->pcast, s, curs, NULL); + Py_DECREF(s); + } } else { PyErr_SetString(Error, "internal error: no casting function found"); diff --git a/psycopg/typecast.h b/psycopg/typecast.h index 06232207..cbae10a7 100644 --- a/psycopg/typecast.h +++ b/psycopg/typecast.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_TYPECAST_H #define PSYCOPG_TYPECAST_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/typecast_basic.c b/psycopg/typecast_basic.c index e9ad527a..20c7b81f 100644 --- a/psycopg/typecast_basic.c +++ b/psycopg/typecast_basic.c @@ -25,6 +25,7 @@ /** INTEGER - cast normal integers (4 bytes) to python int **/ +#if PY_MAJOR_VERSION < 3 static PyObject * typecast_INTEGER_cast(const char *s, Py_ssize_t len, PyObject *curs) { @@ -37,6 +38,9 @@ typecast_INTEGER_cast(const char *s, Py_ssize_t len, PyObject *curs) } return PyInt_FromString((char *)s, NULL, 0); } +#else +#define typecast_INTEGER_cast typecast_LONGINTEGER_cast +#endif /** LONGINTEGER - cast long integers (8 bytes) to python long **/ @@ -59,44 +63,42 @@ static PyObject * typecast_FLOAT_cast(const char *s, Py_ssize_t len, PyObject *curs) { PyObject *str = NULL, *flo = NULL; - char *pend; if (s == NULL) {Py_INCREF(Py_None); return Py_None;} - str = PyString_FromStringAndSize(s, len); - flo = PyFloat_FromString(str, &pend); + str = Text_FromUTF8AndSize(s, len); +#if PY_MAJOR_VERSION < 3 + flo = PyFloat_FromString(str, NULL); +#else + flo = PyFloat_FromString(str); +#endif Py_DECREF(str); return flo; } /** STRING - cast strings of any type to python string **/ +#if PY_MAJOR_VERSION < 3 static PyObject * typecast_STRING_cast(const char *s, Py_ssize_t len, PyObject *curs) { if (s == NULL) {Py_INCREF(Py_None); return Py_None;} return PyString_FromStringAndSize(s, len); } +#else +#define typecast_STRING_cast typecast_UNICODE_cast +#endif /** UNICODE - cast strings of any type to a python unicode object **/ static PyObject * typecast_UNICODE_cast(const char *s, Py_ssize_t len, PyObject *curs) { - PyObject *enc; + char *enc; if (s == NULL) {Py_INCREF(Py_None); return Py_None;} - enc = PyDict_GetItemString(psycoEncodings, - ((cursorObject*)curs)->conn->encoding); - if (enc) { - return PyUnicode_Decode(s, len, PyString_AsString(enc), NULL); - } - else { - PyErr_Format(InterfaceError, - "can't decode into unicode string from %s", - ((cursorObject*)curs)->conn->encoding); - return NULL; - } + enc = ((cursorObject*)curs)->conn->codec; + return PyUnicode_Decode(s, len, enc, NULL); } /** BOOLEAN - cast boolean value into right python object **/ diff --git a/psycopg/typecast_binary.c b/psycopg/typecast_binary.c index 6a707eae..e93aad57 100644 --- a/psycopg/typecast_binary.c +++ b/psycopg/typecast_binary.c @@ -25,7 +25,6 @@ #include "typecast_binary.h" -#include #include @@ -42,7 +41,7 @@ chunk_dealloc(chunkObject *self) self->base, self->len ); PQfreemem(self->base); - self->ob_type->tp_free((PyObject *) self); + Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject * @@ -54,6 +53,9 @@ chunk_repr(chunkObject *self) ); } +#if PY_MAJOR_VERSION < 3 + +/* XXX support 3.0 buffer protocol */ static Py_ssize_t chunk_getreadbuffer(chunkObject *self, Py_ssize_t segment, void **ptr) { @@ -83,11 +85,26 @@ static PyBufferProcs chunk_as_buffer = (charbufferproc) NULL }; +#else + +/* 3.0 buffer interface */ +int chunk_getbuffer(PyObject *_self, Py_buffer *view, int flags) +{ + chunkObject *self = (chunkObject*)_self; + return PyBuffer_FillInfo(view, _self, self->base, self->len, 1, flags); +} +static PyBufferProcs chunk_as_buffer = +{ + chunk_getbuffer, + NULL, +}; + +#endif + #define chunk_doc "memory chunk" PyTypeObject chunkType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.chunk", /* tp_name */ sizeof(chunkObject), /* tp_basicsize */ 0, /* tp_itemsize */ @@ -158,8 +175,13 @@ typecast_BINARY_cast(const char *s, Py_ssize_t l, PyObject *curs) /* size_t->Py_ssize_t cast was validated above: */ chunk->len = (Py_ssize_t) len; +#if PY_MAJOR_VERSION < 3 if ((res = PyBuffer_FromObject((PyObject *)chunk, 0, chunk->len)) == NULL) goto fail; +#else + if ((res = PyMemoryView_FromObject((PyObject*)chunk)) == NULL) + goto fail; +#endif /* PyBuffer_FromObject() created a new reference. We'll release our * reference held in 'chunk' in the 'cleanup' clause. */ diff --git a/psycopg/typecast_binary.h b/psycopg/typecast_binary.h index 14064568..bc70992a 100644 --- a/psycopg/typecast_binary.h +++ b/psycopg/typecast_binary.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_TYPECAST_BINARY_H #define PSYCOPG_TYPECAST_BINARY_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/typecast_datetime.c b/psycopg/typecast_datetime.c index 2bc3625e..29149c05 100644 --- a/psycopg/typecast_datetime.c +++ b/psycopg/typecast_datetime.c @@ -159,7 +159,7 @@ typecast_PYDATETIME_cast(const char *str, Py_ssize_t len, PyObject *curs) y, m, d, hh, mm, ss, us, tzinfo); Dprintf("typecast_PYDATETIME_cast: tzinfo: %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - tzinfo, tzinfo->ob_refcnt + tzinfo, Py_REFCNT(tzinfo) ); Py_DECREF(tzinfo); } diff --git a/psycopg/utils.c b/psycopg/utils.c index c6028052..4e5c83ed 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -23,15 +23,13 @@ * License for more details. */ -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/pgtypes.h" -#include "psycopg/pgversion.h" + +#include #include char * @@ -72,3 +70,122 @@ psycopg_escape_string(PyObject *obj, const char *from, Py_ssize_t len, return to; } + +/* Duplicate a string. + * + * Allocate a new buffer on the Python heap containing the new string. + * 'len' is optional: if 0 the length is calculated. + * + * Return NULL and set an exception in case of error. + */ +char * +psycopg_strdup(const char *from, Py_ssize_t len) +{ + char *rv; + + if (!len) { len = strlen(from); } + if (!(rv = PyMem_Malloc(len + 1))) { + PyErr_NoMemory(); + return NULL; + } + strcpy(rv, from); + return rv; +} + +/* Ensure a Python object is a bytes string. + * + * Useful when a char * is required out of it. + * + * The function is ref neutral: steals a ref from obj and adds one to the + * return value. This also means that you shouldn't call the function on a + * borrowed ref, if having the object unallocated is not what you want. + * + * It is safe to call the function on NULL. + */ +PyObject * +psycopg_ensure_bytes(PyObject *obj) +{ + PyObject *rv = NULL; + if (!obj) { return NULL; } + + if (PyUnicode_CheckExact(obj)) { + rv = PyUnicode_AsUTF8String(obj); + Py_DECREF(obj); + } + else if (Bytes_CheckExact(obj)) { + rv = obj; + } + else { + PyErr_Format(PyExc_TypeError, + "Expected bytes or unicode string, got %s instead", + Py_TYPE(obj)->tp_name); + Py_DECREF(obj); /* steal the ref anyway */ + } + + return rv; +} + +/* Take a Python object and return text from it. + * + * On Py3 this means converting bytes to unicode. On Py2 bytes are fine. + * + * The function is ref neutral: steals a ref from obj and adds one to the + * return value. It is safe to call it on NULL. + */ +PyObject * +psycopg_ensure_text(PyObject *obj) +{ +#if PY_MAJOR_VERSION < 3 + return obj; +#else + if (obj) { + /* bytes to unicode in Py3 */ + PyObject *rv = PyUnicode_FromEncodedObject(obj, "utf8", "replace"); + Py_DECREF(obj); + return rv; + } + else { + return NULL; + } +#endif +} + +/* Check if a file derives from TextIOBase. + * + * Return 1 if it does, else 0, -1 on errors. + */ +int +psycopg_is_text_file(PyObject *f) +{ + /* NULL before any call. + * then io.TextIOBase if exists, else None. */ + static PyObject *base; + + /* Try to import os.TextIOBase */ + if (NULL == base) { + PyObject *m; + Dprintf("psycopg_is_text_file: importing io.TextIOBase"); + if (!(m = PyImport_ImportModule("io"))) { + Dprintf("psycopg_is_text_file: io module not found"); + PyErr_Clear(); + Py_INCREF(Py_None); + base = Py_None; + } + else { + if (!(base = PyObject_GetAttrString(m, "TextIOBase"))) { + Dprintf("psycopg_is_text_file: io.TextIOBase not found"); + PyErr_Clear(); + Py_INCREF(Py_None); + base = Py_None; + } + } + Py_XDECREF(m); + } + + if (base != Py_None) { + return PyObject_IsInstance(f, base); + } else { + return 0; + } +} + diff --git a/psycopg/xid.h b/psycopg/xid.h index 017f3b4c..e63db619 100644 --- a/psycopg/xid.h +++ b/psycopg/xid.h @@ -27,10 +27,6 @@ #ifndef PSYCOPG_XID_H #define PSYCOPG_XID_H 1 -#include - -#include "psycopg/config.h" - extern HIDDEN PyTypeObject XidType; typedef struct { diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index 606cb64e..29577492 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -24,16 +24,12 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/xid.h" + static const char xid_doc[] = "A transaction identifier used for two-phase commit.\n\n" "Usually returned by the connection methods `~connection.xid()` and\n" @@ -70,12 +66,12 @@ static const char database_doc[] = "Database the recovered transaction belongs to."; static PyMemberDef xid_members[] = { - { "format_id", T_OBJECT, offsetof(XidObject, format_id), RO, (char *)format_id_doc }, - { "gtrid", T_OBJECT, offsetof(XidObject, gtrid), RO, (char *)gtrid_doc }, - { "bqual", T_OBJECT, offsetof(XidObject, bqual), RO, (char *)bqual_doc }, - { "prepared", T_OBJECT, offsetof(XidObject, prepared), RO, (char *)prepared_doc }, - { "owner", T_OBJECT, offsetof(XidObject, owner), RO, (char *)owner_doc }, - { "database", T_OBJECT, offsetof(XidObject, database), RO, (char *)database_doc }, + { "format_id", T_OBJECT, offsetof(XidObject, format_id), READONLY, (char *)format_id_doc }, + { "gtrid", T_OBJECT, offsetof(XidObject, gtrid), READONLY, (char *)gtrid_doc }, + { "bqual", T_OBJECT, offsetof(XidObject, bqual), READONLY, (char *)bqual_doc }, + { "prepared", T_OBJECT, offsetof(XidObject, prepared), READONLY, (char *)prepared_doc }, + { "owner", T_OBJECT, offsetof(XidObject, owner), READONLY, (char *)owner_doc }, + { "database", T_OBJECT, offsetof(XidObject, database), READONLY, (char *)database_doc }, { NULL } }; @@ -154,11 +150,11 @@ xid_init(XidObject *self, PyObject *args, PyObject *kwargs) Py_XDECREF(tmp); tmp = self->gtrid; - self->gtrid = PyString_FromString(gtrid); + self->gtrid = Text_FromUTF8(gtrid); Py_XDECREF(tmp); tmp = self->bqual; - self->bqual = PyString_FromString(bqual); + self->bqual = Text_FromUTF8(bqual); Py_XDECREF(tmp); return 0; @@ -186,7 +182,7 @@ xid_dealloc(XidObject *self) Py_CLEAR(self->owner); Py_CLEAR(self->database); - self->ob_type->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free((PyObject *)self); } static void @@ -237,7 +233,7 @@ xid_repr(XidObject *self) PyObject *args = NULL; if (Py_None == self->format_id) { - if (!(format = PyString_FromString(""))) { + if (!(format = Text_FromUTF8(""))) { goto exit; } if (!(args = PyTuple_New(1))) { goto exit; } @@ -245,7 +241,7 @@ xid_repr(XidObject *self) PyTuple_SET_ITEM(args, 0, self->gtrid); } else { - if (!(format = PyString_FromString(""))) { + if (!(format = Text_FromUTF8(""))) { goto exit; } if (!(args = PyTuple_New(3))) { goto exit; } @@ -257,7 +253,7 @@ xid_repr(XidObject *self) PyTuple_SET_ITEM(args, 2, self->bqual); } - rv = PyString_Format(format, args); + rv = Text_Format(format, args); exit: Py_XDECREF(args); @@ -306,8 +302,7 @@ static struct PyMethodDef xid_methods[] = { }; PyTypeObject XidType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2.extensions.Xid", sizeof(XidObject), 0, @@ -404,7 +399,11 @@ _xid_base64_enc_dec(const char *funcname, PyObject *s) if (!(base64 = PyImport_ImportModule("base64"))) { goto exit; } if (!(func = PyObject_GetAttrString(base64, funcname))) { goto exit; } - rv = PyObject_CallFunctionObjArgs(func, s, NULL); + + Py_INCREF(s); + if (!(s = psycopg_ensure_bytes(s))) { goto exit; } + rv = psycopg_ensure_text(PyObject_CallFunctionObjArgs(func, s, NULL)); + Py_DECREF(s); exit: Py_XDECREF(func); @@ -462,7 +461,7 @@ xid_get_tid(XidObject *self) if (!(ebqual = _xid_encode64(self->bqual))) { goto exit; } /* rv = "%d_%s_%s" % (format_id, egtrid, ebqual) */ - if (!(format = PyString_FromString("%d_%s_%s"))) { goto exit; } + if (!(format = Text_FromUTF8("%d_%s_%s"))) { goto exit; } if (!(args = PyTuple_New(3))) { goto exit; } Py_INCREF(self->format_id); @@ -470,7 +469,7 @@ xid_get_tid(XidObject *self) PyTuple_SET_ITEM(args, 1, egtrid); egtrid = NULL; PyTuple_SET_ITEM(args, 2, ebqual); ebqual = NULL; - if (!(rv = PyString_Format(format, args))) { goto exit; } + if (!(rv = Text_Format(format, args))) { goto exit; } } exit: @@ -626,7 +625,7 @@ XidObject * xid_from_string(PyObject *str) { XidObject *rv; - if (!(PyString_Check(str) || PyUnicode_Check(str))) { + if (!(Bytes_Check(str) || PyUnicode_Check(str))) { PyErr_SetString(PyExc_TypeError, "not a valid transaction id"); return NULL; } diff --git a/scripts/fix_b.py b/scripts/fix_b.py new file mode 100644 index 00000000..ccc8e1c9 --- /dev/null +++ b/scripts/fix_b.py @@ -0,0 +1,20 @@ +"""Fixer to change b('string') into b'string'.""" +# Author: Daniele Varrazzo + +import token +from lib2to3 import fixer_base +from lib2to3.pytree import Leaf + +class FixB(fixer_base.BaseFix): + + PATTERN = """ + power< wrapper='b' trailer< '(' arg=[any] ')' > rest=any* > + """ + + def transform(self, node, results): + arg = results['arg'] + wrapper = results["wrapper"] + if len(arg) == 1 and arg[0].type == token.STRING: + b = Leaf(token.STRING, 'b' + arg[0].value, prefix=wrapper.prefix) + node.children = [ b ] + results['rest'] + diff --git a/setup.py b/setup.py index 806ffbce..1eb2206c 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ Intended Audience :: Developers License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) License :: OSI Approved :: Zope Public License Programming Language :: Python +Programming Language :: Python :: 3 Programming Language :: C Programming Language :: SQL Topic :: Database @@ -41,17 +42,33 @@ Operating System :: Microsoft :: Windows Operating System :: Unix """ +# Note: The setup.py must be compatible with both Python 2 and 3 + import os import os.path import sys import re import subprocess -import ConfigParser from distutils.core import setup, Extension from distutils.errors import DistutilsFileError from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc from distutils.ccompiler import get_default_compiler +try: + from distutils.command.build_py import build_py_2to3 as build_py +except ImportError: + from distutils.command.build_py import build_py +else: + # Configure distutils to run our custom 2to3 fixers as well + from lib2to3.refactor import get_fixers_from_package + build_py.fixer_names = get_fixers_from_package('lib2to3.fixes') + build_py.fixer_names.append('fix_b') + sys.path.insert(0, 'scripts') + +try: + import configparser +except ImportError: + import ConfigParser as configparser # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. @@ -74,6 +91,8 @@ def get_pg_config(kind, pg_config): r = p.stdout.readline().strip() if not r: raise Warning(p.stderr.readline()) + if not isinstance(r, str): + r = r.decode('ascii') return r class psycopg_build_ext(build_ext): @@ -294,21 +313,24 @@ or with the pg_config option in 'setup.cfg'. for settingName in ('pg_config', 'include_dirs', 'library_dirs'): try: val = parser.get('build_ext', settingName) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass else: if val.strip() != '': return None # end of guard conditions - import _winreg + try: + import winreg + except ImportError: + import _winreg as winreg pg_inst_base_dir = None pg_config_path = None - reg = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE) + reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) try: - pg_inst_list_key = _winreg.OpenKey(reg, + pg_inst_list_key = winreg.OpenKey(reg, 'SOFTWARE\\PostgreSQL\\Installations' ) except EnvironmentError: @@ -318,23 +340,23 @@ or with the pg_config option in 'setup.cfg'. try: # Determine the name of the first subkey, if any: try: - first_sub_key_name = _winreg.EnumKey(pg_inst_list_key, 0) + first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) except EnvironmentError: first_sub_key_name = None if first_sub_key_name is not None: - pg_first_inst_key = _winreg.OpenKey(reg, + pg_first_inst_key = winreg.OpenKey(reg, 'SOFTWARE\\PostgreSQL\\Installations\\' + first_sub_key_name ) try: - pg_inst_base_dir = _winreg.QueryValueEx( + pg_inst_base_dir = winreg.QueryValueEx( pg_first_inst_key, 'Base Directory' )[0] finally: - _winreg.CloseKey(pg_first_inst_key) + winreg.CloseKey(pg_first_inst_key) finally: - _winreg.CloseKey(pg_inst_list_key) + winreg.CloseKey(pg_inst_list_key) if pg_inst_base_dir and os.path.exists(pg_inst_base_dir): pg_config_path = os.path.join(pg_inst_base_dir, 'bin', @@ -359,16 +381,39 @@ ext = [] ; data_files = [] # sources sources = [ - 'psycopgmodule.c', 'pqpath.c', 'typecast.c', - 'microprotocols.c', 'microprotocols_proto.c', - 'connection_type.c', 'connection_int.c', 'cursor_type.c', 'cursor_int.c', - 'lobject_type.c', 'lobject_int.c', 'notify_type.c', 'xid_type.c', - 'adapter_qstring.c', 'adapter_pboolean.c', 'adapter_binary.c', - 'adapter_asis.c', 'adapter_list.c', 'adapter_datetime.c', - 'adapter_pfloat.c', 'adapter_pdecimal.c', - 'green.c', 'utils.c'] + 'psycopgmodule.c', + 'green.c', 'pqpath.c', 'utils.c', 'bytes_format.c', -parser = ConfigParser.ConfigParser() + 'connection_int.c', 'connection_type.c', + 'cursor_int.c', 'cursor_type.c', + 'lobject_int.c', 'lobject_type.c', + 'notify_type.c', 'xid_type.c', + + 'adapter_asis.c', 'adapter_binary.c', 'adapter_datetime.c', + 'adapter_list.c', 'adapter_pboolean.c', 'adapter_pdecimal.c', + 'adapter_pfloat.c', 'adapter_qstring.c', + 'microprotocols.c', 'microprotocols_proto.c', + 'typecast.c', +] + +depends = [ + # headers + 'config.h', 'pgtypes.h', 'psycopg.h', 'python.h', + 'connection.h', 'cursor.h', 'green.h', 'lobject.h', + 'notify.h', 'pqpath.h', 'xid.h', + + 'adapter_asis.h', 'adapter_binary.h', 'adapter_datetime.h', + 'adapter_list.h', 'adapter_pboolean.h', 'adapter_pdecimal.h', + 'adapter_pfloat.h', 'adapter_qstring.h', + 'microprotocols.h', 'microprotocols_proto.h', + 'typecast.h', 'typecast_binary.h', + + # included sources + 'typecast_array.c', 'typecast_basic.c', 'typecast_binary.c', + 'typecast_builtins.c', 'typecast_datetime.c', +] + +parser = configparser.ConfigParser() parser.read('setup.cfg') # Choose a datetime module @@ -385,6 +430,7 @@ if os.path.exists(mxincludedir): include_dirs.append(mxincludedir) define_macros.append(('HAVE_MXDATETIME','1')) sources.append('adapter_mxdatetime.c') + depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) have_mxdatetime = True version_flags.append('mx') @@ -431,11 +477,13 @@ else: # build the extension -sources = map(lambda x: os.path.join('psycopg', x), sources) +sources = [ os.path.join('psycopg', x) for x in sources] +depends = [ os.path.join('psycopg', x) for x in depends] ext.append(Extension("psycopg2._psycopg", sources, define_macros=define_macros, include_dirs=include_dirs, + depends=depends, undef_macros=[])) setup(name="psycopg2", version=PSYCOPG_VERSION, @@ -449,10 +497,12 @@ setup(name="psycopg2", platforms = ["any"], description=__doc__.split("\n")[0], long_description="\n".join(__doc__.split("\n")[2:]), - classifiers=filter(None, classifiers.split("\n")), + classifiers=[x for x in classifiers.split("\n") if x], data_files=data_files, package_dir={'psycopg2':'lib', 'psycopg2.tests': 'tests'}, packages=['psycopg2', 'psycopg2.tests'], - cmdclass={ 'build_ext': psycopg_build_ext }, + cmdclass={ + 'build_ext': psycopg_build_ext, + 'build_py': build_py, }, ext_modules=ext) diff --git a/tests/__init__.py b/tests/__init__.py index 7a7d9c43..2eaf6ce8 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -24,36 +24,9 @@ import os import sys +from testconfig import dsn from testutils import unittest -dbname = os.environ.get('PSYCOPG2_TESTDB', 'psycopg2_test') -dbhost = os.environ.get('PSYCOPG2_TESTDB_HOST', None) -dbport = os.environ.get('PSYCOPG2_TESTDB_PORT', None) -dbuser = os.environ.get('PSYCOPG2_TESTDB_USER', None) - -# Check if we want to test psycopg's green path. -green = os.environ.get('PSYCOPG2_TEST_GREEN', None) -if green: - if green == '1': - from psycopg2.extras import wait_select as wait_callback - elif green == 'eventlet': - from eventlet.support.psycopg2_patcher import eventlet_wait_callback \ - as wait_callback - else: - raise ValueError("please set 'PSYCOPG2_TEST_GREEN' to a valid value") - - import psycopg2.extensions - psycopg2.extensions.set_wait_callback(wait_callback) - -# Construct a DSN to connect to the test database: -dsn = 'dbname=%s' % dbname -if dbhost is not None: - dsn += ' host=%s' % dbhost -if dbport is not None: - dsn += ' port=%s' % dbport -if dbuser is not None: - dsn += ' user=%s' % dbuser - # If connection to test db fails, bail out early. import psycopg2 try: diff --git a/tests/bug_gc.py b/tests/bug_gc.py index b3d80d1d..798b7e77 100755 --- a/tests/bug_gc.py +++ b/tests/bug_gc.py @@ -28,17 +28,13 @@ import time import unittest import gc -import sys -if sys.version_info < (3,): - import tests -else: - import py3tests as tests +from testconfig import dsn from testutils import skip_if_no_uuid class StolenReferenceTestCase(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() diff --git a/tests/extras_dictcursor.py b/tests/extras_dictcursor.py index c2bacb0b..1bb44ad4 100755 --- a/tests/extras_dictcursor.py +++ b/tests/extras_dictcursor.py @@ -17,15 +17,14 @@ import psycopg2 import psycopg2.extras from testutils import unittest - -import tests +from testconfig import dsn class ExtrasDictCursorTests(unittest.TestCase): """Test if DictCursor extension class works.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) curs = self.conn.cursor() curs.execute("CREATE TEMPORARY TABLE ExtrasDictCursorTests (foo text)") curs.execute("INSERT INTO ExtrasDictCursorTests VALUES ('bar')") @@ -135,7 +134,7 @@ class NamedTupleCursorTest(unittest.TestCase): self.conn = None return - self.conn = psycopg2.connect(tests.dsn, + self.conn = psycopg2.connect(dsn, connection_factory=NamedTupleConnection) curs = self.conn.cursor() curs.execute("CREATE TEMPORARY TABLE nttest (i int, s text)") @@ -207,7 +206,7 @@ class NamedTupleCursorTest(unittest.TestCase): try: if self.conn is not None: self.conn.close() - self.conn = psycopg2.connect(tests.dsn, + self.conn = psycopg2.connect(dsn, connection_factory=NamedTupleConnection) curs = self.conn.cursor() curs.execute("select 1") diff --git a/tests/test_async.py b/tests/test_async.py index 9a388ec6..ea5f1f13 100755 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -32,12 +32,7 @@ import time import select import StringIO -import sys -if sys.version_info < (3,): - import tests -else: - import py3tests as tests - +from testconfig import dsn class PollableStub(object): """A 'pollable' wrapper allowing analysis of the `poll()` calls.""" @@ -57,8 +52,8 @@ class PollableStub(object): class AsyncTests(unittest.TestCase): def setUp(self): - self.sync_conn = psycopg2.connect(tests.dsn) - self.conn = psycopg2.connect(tests.dsn, async=True) + self.sync_conn = psycopg2.connect(dsn) + self.conn = psycopg2.connect(dsn, async=True) self.wait(self.conn) @@ -333,7 +328,7 @@ class AsyncTests(unittest.TestCase): def __init__(self, dsn, async=0): psycopg2.extensions.connection.__init__(self, dsn, async=async) - conn = psycopg2.connect(tests.dsn, connection_factory=MyConn, async=True) + conn = psycopg2.connect(dsn, connection_factory=MyConn, async=True) self.assert_(isinstance(conn, MyConn)) self.assert_(conn.async) conn.close() diff --git a/tests/test_cancel.py b/tests/test_cancel.py index dac2b256..dcdbf41b 100755 --- a/tests/test_cancel.py +++ b/tests/test_cancel.py @@ -25,18 +25,18 @@ import time import threading -from testutils import unittest, skip_if_no_pg_sleep -import tests import psycopg2 import psycopg2.extensions from psycopg2 import extras +from testconfig import dsn +from testutils import unittest, skip_if_no_pg_sleep class CancelTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) cur = self.conn.cursor() cur.execute(''' CREATE TEMPORARY TABLE table1 ( @@ -88,7 +88,7 @@ class CancelTests(unittest.TestCase): @skip_if_no_pg_sleep('conn') def test_async_cancel(self): - async_conn = psycopg2.connect(tests.dsn, async=True) + async_conn = psycopg2.connect(dsn, async=True) self.assertRaises(psycopg2.OperationalError, async_conn.cancel) extras.wait_select(async_conn) cur = async_conn.cursor() @@ -102,7 +102,7 @@ class CancelTests(unittest.TestCase): self.assertEqual(cur.fetchall(), [(1, )]) def test_async_connection_cancel(self): - async_conn = psycopg2.connect(tests.dsn, async=True) + async_conn = psycopg2.connect(dsn, async=True) async_conn.close() self.assertTrue(async_conn.closed) diff --git a/tests/test_connection.py b/tests/test_connection.py index 83661d92..d92391df 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -29,12 +29,12 @@ from operator import attrgetter import psycopg2 import psycopg2.extensions -import tests +from testconfig import dsn, dbname class ConnectionTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): if not self.conn.closed: @@ -117,7 +117,7 @@ class ConnectionTests(unittest.TestCase): @skip_if_no_pg_sleep('conn') def test_concurrent_execution(self): def slave(): - cnn = psycopg2.connect(tests.dsn) + cnn = psycopg2.connect(dsn) cur = cnn.cursor() cur.execute("select pg_sleep(2)") cur.close() @@ -133,6 +133,14 @@ class ConnectionTests(unittest.TestCase): self.assert_(time.time() - t0 < 3, "something broken in concurrency") + def test_encoding_name(self): + self.conn.set_client_encoding("EUC_JP") + # conn.encoding is 'EUCJP' now. + cur = self.conn.cursor() + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, cur) + cur.execute("select 'foo'::text;") + self.assertEqual(cur.fetchone()[0], u'foo') + def test_weakref(self): from weakref import ref conn = psycopg2.connect(self.conn.dsn) @@ -163,7 +171,7 @@ class IsolationLevelsTestCase(unittest.TestCase): conn.close() def connect(self): - conn = psycopg2.connect(tests.dsn) + conn = psycopg2.connect(dsn) self._conns.append(conn) return conn @@ -330,7 +338,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): try: cur.execute( "select gid from pg_prepared_xacts where database = %s", - (tests.dbname,)) + (dbname,)) except psycopg2.ProgrammingError: cnn.rollback() cnn.close() @@ -359,7 +367,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cur.execute(""" select count(*) from pg_prepared_xacts where database = %s;""", - (tests.dbname,)) + (dbname,)) rv = cur.fetchone()[0] cnn.close() return rv @@ -374,7 +382,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): return rv def connect(self): - conn = psycopg2.connect(tests.dsn) + conn = psycopg2.connect(dsn) self._conns.append(conn) return conn @@ -537,13 +545,13 @@ class ConnectionTwoPhaseTests(unittest.TestCase): select gid, prepared, owner, database from pg_prepared_xacts where database = %s;""", - (tests.dbname,)) + (dbname,)) okvals = cur.fetchall() okvals.sort() cnn = self.connect() xids = cnn.tpc_recover() - xids = [ xid for xid in xids if xid.database == tests.dbname ] + xids = [ xid for xid in xids if xid.database == dbname ] xids.sort(key=attrgetter('gtrid')) # check the values returned @@ -563,7 +571,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn = self.connect() cur = cnn.cursor() cur.execute("select gid from pg_prepared_xacts where database = %s;", - (tests.dbname,)) + (dbname,)) self.assertEqual('42_Z3RyaWQ=_YnF1YWw=', cur.fetchone()[0]) def test_xid_roundtrip(self): @@ -580,7 +588,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn = self.connect() xids = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ] + if xid.database == dbname ] self.assertEqual(1, len(xids)) xid = xids[0] self.assertEqual(xid.format_id, fid) @@ -602,7 +610,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn = self.connect() xids = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ] + if xid.database == dbname ] self.assertEqual(1, len(xids)) xid = xids[0] self.assertEqual(xid.format_id, None) @@ -648,7 +656,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn.tpc_prepare() cnn.reset() xid = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ][0] + if xid.database == dbname ][0] self.assertEqual(10, xid.format_id) self.assertEqual('uni', xid.gtrid) self.assertEqual('code', xid.bqual) @@ -664,7 +672,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn.reset() xid = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ][0] + if xid.database == dbname ][0] self.assertEqual(None, xid.format_id) self.assertEqual('transaction-id', xid.gtrid) self.assertEqual(None, xid.bqual) diff --git a/tests/test_copy.py b/tests/test_copy.py index c51ef098..b6da4b1b 100755 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -23,18 +23,19 @@ # License for more details. import os +import sys import string -from testutils import unittest, decorate_all_tests +from testutils import unittest, decorate_all_tests, skip_if_no_iobase from cStringIO import StringIO from itertools import cycle, izip import psycopg2 import psycopg2.extensions -import tests +from testconfig import dsn, green def skip_if_green(f): def skip_if_green_(self): - if tests.green: + if green: return self.skipTest("copy in async mode currently not supported") else: return f(self) @@ -42,7 +43,12 @@ def skip_if_green(f): return skip_if_green_ -class MinimalRead(object): +if sys.version_info[0] < 3: + _base = object +else: + from io import TextIOBase as _base + +class MinimalRead(_base): """A file wrapper exposing the minimal interface to copy from.""" def __init__(self, f): self.f = f @@ -53,7 +59,7 @@ class MinimalRead(object): def readline(self): return self.f.readline() -class MinimalWrite(object): +class MinimalWrite(_base): """A file wrapper exposing the minimal interface to copy to.""" def __init__(self, f): self.f = f @@ -65,7 +71,10 @@ class MinimalWrite(object): class CopyTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) + self._create_temp_table() + + def _create_temp_table(self): curs = self.conn.cursor() curs.execute(''' CREATE TEMPORARY TABLE tcopy ( @@ -126,9 +135,54 @@ class CopyTests(unittest.TestCase): finally: curs.close() + @skip_if_no_iobase + def test_copy_text(self): + self.conn.set_client_encoding('latin1') + self._create_temp_table() # the above call closed the xn + + if sys.version_info[0] < 3: + abin = ''.join(map(chr, range(32, 127) + range(160, 256))) + about = abin.decode('latin1').replace('\\', '\\\\') + + else: + abin = bytes(range(32, 127) + range(160, 256)).decode('latin1') + about = abin.replace('\\', '\\\\') + + curs = self.conn.cursor() + curs.execute('insert into tcopy values (%s, %s)', + (42, abin)) + + import io + f = io.StringIO() + curs.copy_to(f, 'tcopy', columns=('data',)) + f.seek(0) + self.assertEqual(f.readline().rstrip(), about) + + @skip_if_no_iobase + def test_copy_bytes(self): + self.conn.set_client_encoding('latin1') + self._create_temp_table() # the above call closed the xn + + if sys.version_info[0] < 3: + abin = ''.join(map(chr, range(32, 127) + range(160, 255))) + about = abin.replace('\\', '\\\\') + else: + abin = bytes(range(32, 127) + range(160, 255)).decode('latin1') + about = abin.replace('\\', '\\\\').encode('latin1') + + curs = self.conn.cursor() + curs.execute('insert into tcopy values (%s, %s)', + (42, abin)) + + import io + f = io.BytesIO() + curs.copy_to(f, 'tcopy', columns=('data',)) + f.seek(0) + self.assertEqual(f.readline().rstrip(), about) + def _copy_from(self, curs, nrecs, srec, copykw): f = StringIO() - for i, c in izip(xrange(nrecs), cycle(string.letters)): + for i, c in izip(xrange(nrecs), cycle(string.ascii_letters)): l = c * srec f.write("%s\t%s\n" % (i,l)) @@ -139,9 +193,9 @@ class CopyTests(unittest.TestCase): self.assertEqual(nrecs, curs.fetchone()[0]) curs.execute("select data from tcopy where id < %s order by id", - (len(string.letters),)) + (len(string.ascii_letters),)) for i, (l,) in enumerate(curs): - self.assertEqual(l, string.letters[i] * srec) + self.assertEqual(l, string.ascii_letters[i] * srec) def _copy_to(self, curs, srec): f = StringIO() @@ -151,11 +205,11 @@ class CopyTests(unittest.TestCase): ntests = 0 for line in f: n, s = line.split() - if int(n) < len(string.letters): - self.assertEqual(s, string.letters[int(n)] * srec) + if int(n) < len(string.ascii_letters): + self.assertEqual(s, string.ascii_letters[int(n)] * srec) ntests += 1 - self.assertEqual(ntests, len(string.letters)) + self.assertEqual(ntests, len(string.ascii_letters)) decorate_all_tests(CopyTests, skip_if_green) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index a859f42b..292b824d 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -26,12 +26,13 @@ import time import unittest import psycopg2 import psycopg2.extensions -import tests +from psycopg2.extensions import b +from testconfig import dsn class CursorTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -41,7 +42,7 @@ class CursorTests(unittest.TestCase): cur = conn.cursor() cur.execute("create temp table test_exc (data int);") def buggygen(): - yield 1/0 + yield 1//0 self.assertRaises(ZeroDivisionError, cur.executemany, "insert into test_exc values (%s)", buggygen()) cur.close() @@ -55,28 +56,28 @@ class CursorTests(unittest.TestCase): # unicode query containing only ascii data cur.execute(u"SELECT 'foo';") self.assertEqual('foo', cur.fetchone()[0]) - self.assertEqual("SELECT 'foo';", cur.mogrify(u"SELECT 'foo';")) + self.assertEqual(b("SELECT 'foo';"), cur.mogrify(u"SELECT 'foo';")) conn.set_client_encoding('UTF8') snowman = u"\u2603" # unicode query with non-ascii data cur.execute(u"SELECT '%s';" % snowman) - self.assertEqual(snowman.encode('utf8'), cur.fetchone()[0]) - self.assertEqual("SELECT '%s';" % snowman.encode('utf8'), - cur.mogrify(u"SELECT '%s';" % snowman).replace("E'", "'")) + self.assertEqual(snowman.encode('utf8'), b(cur.fetchone()[0])) + self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), + cur.mogrify(u"SELECT '%s';" % snowman).replace(b("E'"), b("'"))) # unicode args cur.execute("SELECT %s;", (snowman,)) - self.assertEqual(snowman.encode("utf-8"), cur.fetchone()[0]) - self.assertEqual("SELECT '%s';" % snowman.encode('utf8'), - cur.mogrify("SELECT %s;", (snowman,)).replace("E'", "'")) + self.assertEqual(snowman.encode("utf-8"), b(cur.fetchone()[0])) + self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), + cur.mogrify("SELECT %s;", (snowman,)).replace(b("E'"), b("'"))) # unicode query and args cur.execute(u"SELECT %s;", (snowman,)) - self.assertEqual(snowman.encode("utf-8"), cur.fetchone()[0]) - self.assertEqual("SELECT '%s';" % snowman.encode('utf8'), - cur.mogrify(u"SELECT %s;", (snowman,)).replace("E'", "'")) + self.assertEqual(snowman.encode("utf-8"), b(cur.fetchone()[0])) + self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), + cur.mogrify(u"SELECT %s;", (snowman,)).replace(b("E'"), b("'"))) def test_mogrify_decimal_explodes(self): # issue #7: explodes on windows with python 2.5 and psycopg 2.2.2 @@ -87,7 +88,7 @@ class CursorTests(unittest.TestCase): conn = self.conn cur = conn.cursor() - self.assertEqual('SELECT 10.3;', + self.assertEqual(b('SELECT 10.3;'), cur.mogrify("SELECT %s;", (Decimal("10.3"),))) def test_cast(self): diff --git a/tests/test_dates.py b/tests/test_dates.py index d393c4f9..db2050ac 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -24,9 +24,9 @@ import math import unittest -import tests import psycopg2 from psycopg2.tz import FixedOffsetTimezone +from testconfig import dsn class CommonDatetimeTestsMixin: @@ -97,7 +97,7 @@ class DatetimeTests(unittest.TestCase, CommonDatetimeTestsMixin): """Tests for the datetime based date handling in psycopg2.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.curs = self.conn.cursor() self.DATE = psycopg2._psycopg.PYDATE self.TIME = psycopg2._psycopg.PYTIME @@ -265,18 +265,18 @@ class DatetimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date(self): from datetime import date - self._test_type_roundtrip(date(2010,05,03)) + self._test_type_roundtrip(date(2010,5,3)) def test_type_roundtrip_datetime(self): from datetime import datetime - dt = self._test_type_roundtrip(datetime(2010,05,03,10,20,30)) + dt = self._test_type_roundtrip(datetime(2010,5,3,10,20,30)) self.assertEqual(None, dt.tzinfo) def test_type_roundtrip_datetimetz(self): from datetime import datetime import psycopg2.tz tz = psycopg2.tz.FixedOffsetTimezone(8*60) - dt1 = datetime(2010,05,03,10,20,30, tzinfo=tz) + dt1 = datetime(2010,5,3,10,20,30, tzinfo=tz) dt2 = self._test_type_roundtrip(dt1) self.assertNotEqual(None, dt2.tzinfo) self.assertEqual(dt1, dt2) @@ -291,11 +291,11 @@ class DatetimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date_array(self): from datetime import date - self._test_type_roundtrip_array(date(2010,05,03)) + self._test_type_roundtrip_array(date(2010,5,3)) def test_type_roundtrip_datetime_array(self): from datetime import datetime - self._test_type_roundtrip_array(datetime(2010,05,03,10,20,30)) + self._test_type_roundtrip_array(datetime(2010,5,3,10,20,30)) def test_type_roundtrip_time_array(self): from datetime import time @@ -315,7 +315,7 @@ class mxDateTimeTests(unittest.TestCase, CommonDatetimeTestsMixin): """Tests for the mx.DateTime based date handling in psycopg2.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.curs = self.conn.cursor() self.DATE = psycopg2._psycopg.MXDATE self.TIME = psycopg2._psycopg.MXTIME @@ -448,11 +448,11 @@ class mxDateTimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date(self): from mx.DateTime import Date - self._test_type_roundtrip(Date(2010,05,03)) + self._test_type_roundtrip(Date(2010,5,3)) def test_type_roundtrip_datetime(self): from mx.DateTime import DateTime - self._test_type_roundtrip(DateTime(2010,05,03,10,20,30)) + self._test_type_roundtrip(DateTime(2010,5,3,10,20,30)) def test_type_roundtrip_time(self): from mx.DateTime import Time @@ -464,11 +464,11 @@ class mxDateTimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date_array(self): from mx.DateTime import Date - self._test_type_roundtrip_array(Date(2010,05,03)) + self._test_type_roundtrip_array(Date(2010,5,3)) def test_type_roundtrip_datetime_array(self): from mx.DateTime import DateTime - self._test_type_roundtrip_array(DateTime(2010,05,03,10,20,30)) + self._test_type_roundtrip_array(DateTime(2010,5,3,10,20,30)) def test_type_roundtrip_time_array(self): from mx.DateTime import Time diff --git a/tests/test_green.py b/tests/test_green.py index f0b1c37e..d641d183 100755 --- a/tests/test_green.py +++ b/tests/test_green.py @@ -26,7 +26,7 @@ import unittest import psycopg2 import psycopg2.extensions import psycopg2.extras -import tests +from testconfig import dsn class ConnectionStub(object): """A `connection` wrapper allowing analysis of the `poll()` calls.""" @@ -46,7 +46,7 @@ class GreenTests(unittest.TestCase): def setUp(self): self._cb = psycopg2.extensions.get_wait_callback() psycopg2.extensions.set_wait_callback(psycopg2.extras.wait_select) - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -85,7 +85,7 @@ class GreenTests(unittest.TestCase): curs.fetchone() # now try to do something that will fail in the callback - psycopg2.extensions.set_wait_callback(lambda conn: 1/0) + psycopg2.extensions.set_wait_callback(lambda conn: 1//0) self.assertRaises(ZeroDivisionError, curs.execute, "select 2") # check that the connection is left in an usable state diff --git a/tests/test_lobject.py b/tests/test_lobject.py index ef27a884..d327bb96 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -29,7 +29,9 @@ from testutils import unittest, decorate_all_tests, skip_if_tpc_disabled import psycopg2 import psycopg2.extensions -import tests +from psycopg2.extensions import b +from testconfig import dsn, green +from testutils import unittest, decorate_all_tests def skip_if_no_lo(f): def skip_if_no_lo_(self): @@ -42,7 +44,7 @@ def skip_if_no_lo(f): def skip_if_green(f): def skip_if_green_(self): - if tests.green: + if green: return self.skipTest("libpq doesn't support LO in async mode") else: return f(self) @@ -75,14 +77,14 @@ class LargeObjectMixin(object): self.conn.close() def connect(self): - return psycopg2.connect(tests.dsn) + return psycopg2.connect(dsn) class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_create(self): lo = self.conn.lobject() self.assertNotEqual(lo, None) - self.assertEqual(lo.mode, "w") + self.assertEqual(lo.mode[0], "w") def test_open_non_existent(self): # By creating then removing a large object, we get an Oid that @@ -96,13 +98,13 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): lo2 = self.conn.lobject(lo.oid) self.assertNotEqual(lo2, None) self.assertEqual(lo2.oid, lo.oid) - self.assertEqual(lo2.mode, "r") + self.assertEqual(lo2.mode[0], "r") def test_open_for_write(self): lo = self.conn.lobject() lo2 = self.conn.lobject(lo.oid, "w") - self.assertEqual(lo2.mode, "w") - lo2.write("some data") + self.assertEqual(lo2.mode[0], "w") + lo2.write(b("some data")) def test_open_mode_n(self): # Openning an object in mode "n" gives us a closed lobject. @@ -138,7 +140,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") fp = open(filename, "wb") - fp.write("some data") + fp.write(b("some data")) fp.close() lo = self.conn.lobject(0, "r", 0, filename) @@ -152,7 +154,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_write(self): lo = self.conn.lobject() - self.assertEqual(lo.write("some data"), len("some data")) + self.assertEqual(lo.write(b("some data")), len("some data")) def test_write_large(self): lo = self.conn.lobject() @@ -161,26 +163,54 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_read(self): lo = self.conn.lobject() - length = lo.write("some data") + length = lo.write(b("some data")) lo.close() lo = self.conn.lobject(lo.oid) - self.assertEqual(lo.read(4), "some") + x = lo.read(4) + self.assertEqual(type(x), type('')) + self.assertEqual(x, "some") self.assertEqual(lo.read(), " data") + def test_read_binary(self): + lo = self.conn.lobject() + length = lo.write(b("some data")) + lo.close() + + lo = self.conn.lobject(lo.oid, "rb") + x = lo.read(4) + self.assertEqual(type(x), type(b(''))) + self.assertEqual(x, b("some")) + self.assertEqual(lo.read(), b(" data")) + + def test_read_text(self): + lo = self.conn.lobject() + snowman = u"\u2603" + length = lo.write(u"some data " + snowman) + lo.close() + + lo = self.conn.lobject(lo.oid, "rt") + x = lo.read(4) + self.assertEqual(type(x), type(u'')) + self.assertEqual(x, u"some") + self.assertEqual(lo.read(), u" data " + snowman) + def test_read_large(self): lo = self.conn.lobject() data = "data" * 1000000 - length = lo.write("some"+data) + length = lo.write("some" + data) lo.close() lo = self.conn.lobject(lo.oid) self.assertEqual(lo.read(4), "some") - self.assertEqual(lo.read(), data) + data1 = lo.read() + # avoid dumping megacraps in the console in case of error + self.assert_(data == data1, + "%r... != %r..." % (data[:100], data1[:100])) def test_seek_tell(self): lo = self.conn.lobject() - length = lo.write("some data") + length = lo.write(b("some data")) self.assertEqual(lo.tell(), length) lo.close() lo = self.conn.lobject(lo.oid) @@ -210,13 +240,17 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_export(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), "some data") + f = open(filename, "rb") + try: + self.assertEqual(f.read(), b("some data")) + finally: + f.close() def test_close_twice(self): lo = self.conn.lobject() @@ -226,7 +260,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_write_after_close(self): lo = self.conn.lobject() lo.close() - self.assertRaises(psycopg2.InterfaceError, lo.write, "some data") + self.assertRaises(psycopg2.InterfaceError, lo.write, b("some data")) def test_read_after_close(self): lo = self.conn.lobject() @@ -251,14 +285,18 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_export_after_close(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) lo.close() self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), "some data") + f = open(filename, "rb") + try: + self.assertEqual(f.read(), b("some data")) + finally: + f.close() def test_close_after_commit(self): lo = self.conn.lobject() @@ -273,7 +311,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): self.lo_oid = lo.oid self.conn.commit() - self.assertRaises(psycopg2.ProgrammingError, lo.write, "some data") + self.assertRaises(psycopg2.ProgrammingError, lo.write, b("some data")) def test_read_after_commit(self): lo = self.conn.lobject() @@ -306,14 +344,18 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_export_after_commit(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) self.conn.commit() self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), "some data") + f = open(filename, "rb") + try: + self.assertEqual(f.read(), b("some data")) + finally: + f.close() @skip_if_tpc_disabled def test_read_after_tpc_commit(self): @@ -357,7 +399,7 @@ def skip_if_no_truncate(f): class LargeObjectTruncateTests(LargeObjectMixin, unittest.TestCase): def test_truncate(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) lo.close() lo = self.conn.lobject(lo.oid, "w") @@ -366,17 +408,17 @@ class LargeObjectTruncateTests(LargeObjectMixin, unittest.TestCase): # seek position unchanged self.assertEqual(lo.tell(), 0) # data truncated - self.assertEqual(lo.read(), "some") + self.assertEqual(lo.read(), b("some")) lo.truncate(6) lo.seek(0) # large object extended with zeroes - self.assertEqual(lo.read(), "some\x00\x00") + self.assertEqual(lo.read(), b("some\x00\x00")) lo.truncate() lo.seek(0) # large object empty - self.assertEqual(lo.read(), "") + self.assertEqual(lo.read(), b("")) def test_truncate_after_close(self): lo = self.conn.lobject() diff --git a/tests/test_notify.py b/tests/test_notify.py index 23ca1761..ea6b7e0f 100755 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -26,23 +26,20 @@ from testutils import unittest import psycopg2 from psycopg2 import extensions +from testconfig import dsn +from testutils import script_to_py3 +import sys import time import select import signal from subprocess import Popen, PIPE -import sys -if sys.version_info < (3,): - import tests -else: - import py3tests as tests - class NotifiesTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -77,9 +74,9 @@ curs.execute("NOTIFY " %(name)r %(payload)r) curs.close() conn.close() """ - % { 'dsn': tests.dsn, 'sec': sec, 'name': name, 'payload': payload}) + % { 'dsn': dsn, 'sec': sec, 'name': name, 'payload': payload}) - return Popen([sys.executable, '-c', script], stdout=PIPE) + return Popen([sys.executable, '-c', script_to_py3(script)], stdout=PIPE) def test_notifies_received_on_poll(self): self.autocommit(self.conn) diff --git a/tests/test_psycopg2_dbapi20.py b/tests/test_psycopg2_dbapi20.py index 222d9341..744d3224 100755 --- a/tests/test_psycopg2_dbapi20.py +++ b/tests/test_psycopg2_dbapi20.py @@ -28,12 +28,12 @@ from testutils import skip_if_tpc_disabled from testutils import unittest, decorate_all_tests import psycopg2 -import tests +from testconfig import dsn class Psycopg2Tests(dbapi20.DatabaseAPI20Test): driver = psycopg2 connect_args = () - connect_kw_args = {'dsn': tests.dsn} + connect_kw_args = {'dsn': dsn} lower_func = 'lower' # For stored procedure test @@ -50,7 +50,7 @@ class Psycopg2TPCTests(dbapi20_tpc.TwoPhaseCommitTests, unittest.TestCase): driver = psycopg2 def connect(self): - return psycopg2.connect(dsn=tests.dsn) + return psycopg2.connect(dsn=dsn) decorate_all_tests(Psycopg2TPCTests, skip_if_tpc_disabled) diff --git a/tests/test_quote.py b/tests/test_quote.py index 7363a3dd..23bc61f0 100755 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -22,11 +22,13 @@ # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. +import sys from testutils import unittest +from testconfig import dsn import psycopg2 import psycopg2.extensions -import tests +from psycopg2.extensions import b class QuotingTestCase(unittest.TestCase): r"""Checks the correct quoting of strings and binary objects. @@ -47,7 +49,7 @@ class QuotingTestCase(unittest.TestCase): http://www.postgresql.org/docs/8.1/static/runtime-config-compatible.html """ def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -66,14 +68,20 @@ class QuotingTestCase(unittest.TestCase): self.assert_(not self.conn.notices) def test_binary(self): - data = """some data with \000\013 binary + data = b("""some data with \000\013 binary stuff into, 'quotes' and \\ a backslash too. - """ - data += "".join(map(chr, range(256))) + """) + if sys.version_info[0] < 3: + data += "".join(map(chr, range(256))) + else: + data += bytes(range(256)) curs = self.conn.cursor() curs.execute("SELECT %s::bytea;", (psycopg2.Binary(data),)) - res = str(curs.fetchone()[0]) + if sys.version_info[0] < 3: + res = str(curs.fetchone()[0]) + else: + res = curs.fetchone()[0].tobytes() self.assertEqual(res, data) self.assert_(not self.conn.notices) @@ -101,6 +109,55 @@ class QuotingTestCase(unittest.TestCase): self.assertEqual(res, data) self.assert_(not self.conn.notices) + def test_latin1(self): + self.conn.set_client_encoding('LATIN1') + curs = self.conn.cursor() + if sys.version_info[0] < 3: + data = ''.join(map(chr, range(32, 127) + range(160, 256))) + else: + data = bytes(range(32, 127) + range(160, 256)).decode('latin1') + + # as string + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + # as unicode + if sys.version_info[0] < 3: + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, self.conn) + data = data.decode('latin1') + + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + def test_koi8(self): + self.conn.set_client_encoding('KOI8') + curs = self.conn.cursor() + if sys.version_info[0] < 3: + data = ''.join(map(chr, range(32, 127) + range(128, 256))) + else: + data = bytes(range(32, 127) + range(128, 256)).decode('koi8_r') + + # as string + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + # as unicode + if sys.version_info[0] < 3: + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, self.conn) + data = data.decode('koi8_r') + + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 0074215b..cab9c450 100755 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -28,13 +28,12 @@ from testutils import unittest, skip_if_no_pg_sleep import psycopg2 from psycopg2.extensions import ( ISOLATION_LEVEL_SERIALIZABLE, STATUS_BEGIN, STATUS_READY) -import tests - +from testconfig import dsn class TransactionTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.conn.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE) curs = self.conn.cursor() curs.execute(''' @@ -98,7 +97,7 @@ class DeadlockSerializationTests(unittest.TestCase): """Test deadlock and serialization failure errors.""" def connect(self): - conn = psycopg2.connect(tests.dsn) + conn = psycopg2.connect(dsn) conn.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE) return conn @@ -231,7 +230,7 @@ class QueryCancellationTests(unittest.TestCase): """Tests for query cancellation.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.conn.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE) def tearDown(self): diff --git a/tests/testconfig.py b/tests/testconfig.py new file mode 100644 index 00000000..b2730e02 --- /dev/null +++ b/tests/testconfig.py @@ -0,0 +1,33 @@ +# Configure the test suite from the env variables. + +import os + +dbname = os.environ.get('PSYCOPG2_TESTDB', 'psycopg2_test') +dbhost = os.environ.get('PSYCOPG2_TESTDB_HOST', None) +dbport = os.environ.get('PSYCOPG2_TESTDB_PORT', None) +dbuser = os.environ.get('PSYCOPG2_TESTDB_USER', None) + +# Check if we want to test psycopg's green path. +green = os.environ.get('PSYCOPG2_TEST_GREEN', None) +if green: + if green == '1': + from psycopg2.extras import wait_select as wait_callback + elif green == 'eventlet': + from eventlet.support.psycopg2_patcher import eventlet_wait_callback \ + as wait_callback + else: + raise ValueError("please set 'PSYCOPG2_TEST_GREEN' to a valid value") + + import psycopg2.extensions + psycopg2.extensions.set_wait_callback(wait_callback) + +# Construct a DSN to connect to the test database: +dsn = 'dbname=%s' % dbname +if dbhost is not None: + dsn += ' host=%s' % dbhost +if dbport is not None: + dsn += ' port=%s' % dbport +if dbuser is not None: + dsn += ' user=%s' % dbuser + + diff --git a/tests/testutils.py b/tests/testutils.py index cfeb0e05..98297fc6 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -23,6 +23,9 @@ # Use unittest2 if available. Otherwise mock a skip facility with warnings. +import os +import sys + try: import unittest2 unittest = unittest2 @@ -57,6 +60,16 @@ else: unittest.TestCase.skipTest = skipTest +# Silence warnings caused by the stubborness of the Python unittest maintainers +# http://bugs.python.org/issue9424 +if not hasattr(unittest.TestCase, 'assert_') \ +or unittest.TestCase.assert_ is not unittest.TestCase.assertTrue: + # mavaff... + unittest.TestCase.assert_ = unittest.TestCase.assertTrue + unittest.TestCase.failUnless = unittest.TestCase.assertTrue + unittest.TestCase.assertEquals = unittest.TestCase.assertEqual + unittest.TestCase.failUnlessEqual = unittest.TestCase.assertEqual + def decorate_all_tests(cls, decorator): """Apply *decorator* to all the tests defined in the TestCase *cls*.""" @@ -138,3 +151,60 @@ def skip_if_tpc_disabled(f): return skip_if_tpc_disabled_ +def skip_if_no_iobase(f): + """Skip a test if io.TextIOBase is not available.""" + def skip_if_no_iobase_(self): + try: + from io import TextIOBase + except ImportError: + return self.skipTest("io.TextIOBase not found.") + else: + return f(self) + + return skip_if_no_iobase_ + + +def skip_on_python2(f): + """Skip a test on Python 3 and following.""" + def skip_on_python2_(self): + if sys.version_info[0] < 3: + return self.skipTest("skipped because Python 2") + else: + return f(self) + + return skip_on_python2_ + +def skip_on_python3(f): + """Skip a test on Python 3 and following.""" + def skip_on_python3_(self): + if sys.version_info[0] >= 3: + return self.skipTest("skipped because Python 3") + else: + return f(self) + + return skip_on_python3_ + +def script_to_py3(script): + """Convert a script to Python3 syntax if required.""" + if sys.version_info[0] < 3: + return script + + import tempfile + f = tempfile.NamedTemporaryFile(suffix=".py") + f.write(script.encode()) + f.flush() + + # 2to3 is way too chatty + import logging + logging.basicConfig(filename=os.devnull) + + from lib2to3.main import main + if main("lib2to3.fixes", ['--no-diffs', '-w', '-n', f.name]): + raise Exception('py3 conversion failed') + + f2 = open(f.name) + try: + return f2.read() + finally: + f2.close() + diff --git a/tests/types_basic.py b/tests/types_basic.py index c7e639e9..c1392b75 100755 --- a/tests/types_basic.py +++ b/tests/types_basic.py @@ -27,17 +27,19 @@ try: except: pass import sys +import testutils from testutils import unittest +from testconfig import dsn import psycopg2 -import tests +from psycopg2.extensions import b class TypesBasicTests(unittest.TestCase): """Test that all type conversions are working.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -62,13 +64,19 @@ class TypesBasicTests(unittest.TestCase): self.failUnless(s == 1971, "wrong integer quoting: " + str(s)) s = self.execute("SELECT %s AS foo", (1971L,)) self.failUnless(s == 1971L, "wrong integer quoting: " + str(s)) - if sys.version_info[0] < 2 or sys.version_info[1] < 4: + if sys.version_info[0:2] < (2, 4): s = self.execute("SELECT %s AS foo", (19.10,)) self.failUnless(abs(s - 19.10) < 0.001, "wrong float quoting: " + str(s)) + def testBoolean(self): + x = self.execute("SELECT %s as foo", (False,)) + self.assert_(x is False) + x = self.execute("SELECT %s as foo", (True,)) + self.assert_(x is True) + def testDecimal(self): - if sys.version_info[0] >= 2 and sys.version_info[1] >= 4: + if sys.version_info[0:2] >= (2, 4): s = self.execute("SELECT %s AS foo", (decimal.Decimal("19.10"),)) self.failUnless(s - decimal.Decimal("19.10") == 0, "wrong decimal quoting: " + str(s)) @@ -101,38 +109,55 @@ class TypesBasicTests(unittest.TestCase): return self.skipTest("inf::float not available on the server") except ValueError: return self.skipTest("inf not available on this platform") - s = self.execute("SELECT %s AS foo", (float("inf"),)) self.failUnless(str(s) == "inf", "wrong float quoting: " + str(s)) self.failUnless(type(s) == float, "wrong float conversion: " + repr(s)) def testBinary(self): - s = ''.join([chr(x) for x in range(256)]) - b = psycopg2.Binary(s) - buf = self.execute("SELECT %s::bytea AS foo", (b,)) - self.failUnless(str(buf) == s, "wrong binary quoting") + if sys.version_info[0] < 3: + s = ''.join([chr(x) for x in range(256)]) + b = psycopg2.Binary(s) + buf = self.execute("SELECT %s::bytea AS foo", (b,)) + self.assertEqual(s, str(buf)) + else: + s = bytes(range(256)) + b = psycopg2.Binary(s) + buf = self.execute("SELECT %s::bytea AS foo", (b,)) + self.assertEqual(s, buf) def testBinaryEmptyString(self): # test to make sure an empty Binary is converted to an empty string - b = psycopg2.Binary('') - self.assertEqual(str(b), "''::bytea") + if sys.version_info[0] < 3: + b = psycopg2.Binary('') + self.assertEqual(str(b), "''::bytea") + else: + b = psycopg2.Binary(bytes([])) + self.assertEqual(str(b), "''::bytea") def testBinaryRoundTrip(self): # test to make sure buffers returned by psycopg2 are # understood by execute: - s = ''.join([chr(x) for x in range(256)]) - buf = self.execute("SELECT %s::bytea AS foo", (psycopg2.Binary(s),)) - buf2 = self.execute("SELECT %s::bytea AS foo", (buf,)) - self.failUnless(str(buf2) == s, "wrong binary quoting") + if sys.version_info[0] < 3: + s = ''.join([chr(x) for x in range(256)]) + buf = self.execute("SELECT %s::bytea AS foo", (psycopg2.Binary(s),)) + buf2 = self.execute("SELECT %s::bytea AS foo", (buf,)) + self.assertEqual(s, str(buf2)) + else: + s = bytes(range(256)) + buf = self.execute("SELECT %s::bytea AS foo", (psycopg2.Binary(s),)) + buf2 = self.execute("SELECT %s::bytea AS foo", (buf,)) + self.assertEqual(s, buf2) def testArray(self): + s = self.execute("SELECT %s AS foo", ([],)) + self.failUnlessEqual(s, []) s = self.execute("SELECT %s AS foo", ([[1,2],[3,4]],)) - self.failUnless(s == [[1,2],[3,4]], "wrong array quoting " + str(s)) + self.failUnlessEqual(s, [[1,2],[3,4]]) s = self.execute("SELECT %s AS foo", (['one', 'two', 'three'],)) - self.failUnless(s == ['one', 'two', 'three'], - "wrong array quoting " + str(s)) + self.failUnlessEqual(s, ['one', 'two', 'three']) - def testTypeRoundtripBinary(self): + @testutils.skip_on_python3 + def testTypeRoundtripBuffer(self): o1 = buffer("".join(map(chr, range(256)))) o2 = self.execute("select %s;", (o1,)) self.assertEqual(type(o1), type(o2)) @@ -142,12 +167,53 @@ class TypesBasicTests(unittest.TestCase): o2 = self.execute("select %s;", (o1,)) self.assertEqual(type(o1), type(o2)) - def testTypeRoundtripBinaryArray(self): + @testutils.skip_on_python3 + def testTypeRoundtripBufferArray(self): o1 = buffer("".join(map(chr, range(256)))) o1 = [o1] o2 = self.execute("select %s;", (o1,)) self.assertEqual(type(o1[0]), type(o2[0])) + @testutils.skip_on_python2 + def testTypeRoundtripBytes(self): + o1 = bytes(range(256)) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + # Test with an empty buffer + o1 = bytes([]) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + @testutils.skip_on_python2 + def testTypeRoundtripBytesArray(self): + o1 = bytes(range(256)) + o1 = [o1] + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2[0])) + + @testutils.skip_on_python2 + def testAdaptBytearray(self): + o1 = bytearray(range(256)) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + # Test with an empty buffer + o1 = bytearray([]) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + @testutils.skip_on_python2 + def testAdaptMemoryview(self): + o1 = memoryview(bytes(range(256))) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + # Test with an empty buffer + o1 = memoryview(bytes([])) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + class AdaptSubclassTest(unittest.TestCase): def test_adapt_subtype(self): @@ -167,11 +233,12 @@ class AdaptSubclassTest(unittest.TestCase): register_adapter(A, lambda a: AsIs("a")) register_adapter(B, lambda b: AsIs("b")) try: - self.assertEqual('b', adapt(C()).getquoted()) + self.assertEqual(b('b'), adapt(C()).getquoted()) finally: del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] del psycopg2.extensions.adapters[B, psycopg2.extensions.ISQLQuote] + @testutils.skip_on_python3 def test_no_mro_no_joy(self): from psycopg2.extensions import adapt, register_adapter, AsIs @@ -185,6 +252,20 @@ class AdaptSubclassTest(unittest.TestCase): del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] + @testutils.skip_on_python2 + def test_adapt_subtype_3(self): + from psycopg2.extensions import adapt, register_adapter, AsIs + + class A: pass + class B(A): pass + + register_adapter(A, lambda a: AsIs("a")) + try: + self.assertEqual(b("a"), adapt(B()).getquoted()) + finally: + del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/tests/types_extras.py b/tests/types_extras.py index 6b258863..bb763384 100755 --- a/tests/types_extras.py +++ b/tests/types_extras.py @@ -26,20 +26,22 @@ from testutils import unittest, skip_if_no_uuid import psycopg2 import psycopg2.extras -import tests +from psycopg2.extensions import b + +from testconfig import dsn def filter_scs(conn, s): if conn.get_parameter_status("standard_conforming_strings") == 'off': return s else: - return s.replace("E'", "'") + return s.replace(b("E'"), b("'")) class TypesExtrasTests(unittest.TestCase): """Test that all type conversions are working.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -93,7 +95,7 @@ class TypesExtrasTests(unittest.TestCase): a = psycopg2.extensions.adapt(i) a.prepare(self.conn) self.assertEqual( - filter_scs(self.conn, "E'192.168.1.0/24'::inet"), + filter_scs(self.conn, b("E'192.168.1.0/24'::inet")), a.getquoted()) # adapts ok with unicode too @@ -101,7 +103,7 @@ class TypesExtrasTests(unittest.TestCase): a = psycopg2.extensions.adapt(i) a.prepare(self.conn) self.assertEqual( - filter_scs(self.conn, "E'192.168.1.0/24'::inet"), + filter_scs(self.conn, b("E'192.168.1.0/24'::inet")), a.getquoted()) def test_adapt_fail(self): @@ -126,7 +128,7 @@ def skip_if_no_hstore(f): class HstoreTestCase(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -145,18 +147,17 @@ class HstoreTestCase(unittest.TestCase): a.prepare(self.conn) q = a.getquoted() - self.assert_(q.startswith("(("), q) - self.assert_(q.endswith("))"), q) - ii = q[1:-1].split("||") + self.assert_(q.startswith(b("((")), q) + ii = q[1:-1].split(b("||")) ii.sort() self.assertEqual(len(ii), len(o)) - self.assertEqual(ii[0], filter_scs(self.conn, "(E'a' => E'1')")) - self.assertEqual(ii[1], filter_scs(self.conn, "(E'b' => E'''')")) - self.assertEqual(ii[2], filter_scs(self.conn, "(E'c' => NULL)")) + self.assertEqual(ii[0], filter_scs(self.conn, b("(E'a' => E'1')"))) + self.assertEqual(ii[1], filter_scs(self.conn, b("(E'b' => E'''')"))) + self.assertEqual(ii[2], filter_scs(self.conn, b("(E'c' => NULL)"))) if 'd' in o: encc = u'\xe0'.encode(psycopg2.extensions.encodings[self.conn.encoding]) - self.assertEqual(ii[3], filter_scs(self.conn, "(E'd' => E'%s')" % encc)) + self.assertEqual(ii[3], filter_scs(self.conn, b("(E'd' => E'") + encc + b("')"))) def test_adapt_9(self): if self.conn.server_version < 90000: @@ -172,11 +173,11 @@ class HstoreTestCase(unittest.TestCase): a.prepare(self.conn) q = a.getquoted() - m = re.match(r'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)', q) + m = re.match(b(r'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)'), q) self.assert_(m, repr(q)) - kk = m.group(1).split(", ") - vv = m.group(2).split(", ") + kk = m.group(1).split(b(", ")) + vv = m.group(2).split(b(", ")) ii = zip(kk, vv) ii.sort() @@ -184,12 +185,12 @@ class HstoreTestCase(unittest.TestCase): return tuple([filter_scs(self.conn, s) for s in args]) self.assertEqual(len(ii), len(o)) - self.assertEqual(ii[0], f("E'a'", "E'1'")) - self.assertEqual(ii[1], f("E'b'", "E''''")) - self.assertEqual(ii[2], f("E'c'", "NULL")) + self.assertEqual(ii[0], f(b("E'a'"), b("E'1'"))) + self.assertEqual(ii[1], f(b("E'b'"), b("E''''"))) + self.assertEqual(ii[2], f(b("E'c'"), b("NULL"))) if 'd' in o: encc = u'\xe0'.encode(psycopg2.extensions.encodings[self.conn.encoding]) - self.assertEqual(ii[3], f("E'd'", "E'%s'" % encc)) + self.assertEqual(ii[3], f(b("E'd'"), b("E'") + encc + b("'"))) def test_parse(self): from psycopg2.extras import HstoreAdapter @@ -305,7 +306,11 @@ class HstoreTestCase(unittest.TestCase): ok({''.join(ab): ''.join(ab)}) self.conn.set_client_encoding('latin1') - ab = map(chr, range(1, 256)) + if sys.version_info[0] < 3: + ab = map(chr, range(32, 127) + range(160, 255)) + else: + ab = bytes(range(32, 127) + range(160, 255)).decode('latin1') + ok({''.join(ab): ''.join(ab)}) ok(dict(zip(ab, ab))) @@ -347,7 +352,7 @@ def skip_if_no_composite(f): class AdaptTypeTestCase(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -356,7 +361,7 @@ class AdaptTypeTestCase(unittest.TestCase): def test_none_in_record(self): curs = self.conn.cursor() s = curs.mogrify("SELECT %s;", [(42, None)]) - self.assertEqual("SELECT (42, NULL);", s) + self.assertEqual(b("SELECT (42, NULL);"), s) curs.execute("SELECT %s;", [(42, None)]) d = curs.fetchone()[0] self.assertEqual("(42,)", d) @@ -377,7 +382,7 @@ class AdaptTypeTestCase(unittest.TestCase): self.assertEqual(ext.adapt(None).getquoted(), "NOPE!") s = curs.mogrify("SELECT %s;", (None,)) - self.assertEqual("SELECT NULL;", s) + self.assertEqual(b("SELECT NULL;"), s) finally: ext.register_adapter(type(None), orig_adapter)