diff --git a/.travis.yml b/.travis.yml index 1aa25416..10411637 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,24 @@ +# Travis CI configuration file for psycopg2 + +dist: trusty +sudo: required language: python python: - - 2.6 - 2.7 - -before_script: - - psql -c 'create database psycopg2_test;' -U postgres + - 3.6-dev + - 2.6 + - 3.5 + - 3.4 + - 3.3 + - 3.2 install: - python setup.py install + - sudo scripts/travis_prepare.sh -script: make check +script: + - scripts/travis_test.sh + +notifications: + email: false diff --git a/MANIFEST.in b/MANIFEST.in index 00e4fc32..0d34fd3d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,10 +2,10 @@ recursive-include psycopg *.c *.h *.manifest recursive-include lib *.py recursive-include tests *.py recursive-include examples *.py somehackers.jpg whereareyou.jpg -recursive-include doc README SUCCESS COPYING.LESSER pep-0249.txt -recursive-include doc Makefile requirements.txt +include doc/README.rst doc/SUCCESS doc/COPYING.LESSER doc/pep-0249.txt +include doc/Makefile doc/requirements.txt recursive-include doc/src *.rst *.py *.css Makefile recursive-include scripts *.py *.sh include scripts/maketypes.sh scripts/buildtypes.py include AUTHORS README.rst INSTALL LICENSE NEWS -include PKG-INFO MANIFEST.in MANIFEST setup.py setup.cfg Makefile +include MANIFEST.in setup.py setup.cfg Makefile diff --git a/Makefile b/Makefile index 232f0d0b..a8f491e4 100644 --- a/Makefile +++ b/Makefile @@ -92,14 +92,9 @@ $(PACKAGE)/tests/%.py: tests/%.py $(PYTHON) setup.py build_py $(BUILD_OPT) touch $@ -$(SDIST): MANIFEST $(SOURCE) +$(SDIST): $(SOURCE) $(PYTHON) setup.py sdist $(SDIST_OPT) -MANIFEST: MANIFEST.in $(SOURCE) - # Run twice as MANIFEST.in includes MANIFEST - $(PYTHON) setup.py sdist --manifest-only - $(PYTHON) setup.py sdist --manifest-only - # docs depend on the build as it partly use introspection. doc/html/genindex.html: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) $(MAKE) -C doc html @@ -111,5 +106,5 @@ doc/docs.zip: doc/html/genindex.html (cd doc/html && zip -r ../docs.zip *) clean: - rm -rf build MANIFEST + rm -rf build $(MAKE) -C doc clean diff --git a/NEWS b/NEWS index 5200c4dd..67883d74 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,12 @@ What's new in psycopg 2.7 New features: -- Added `~psycopg2.extensions.parse_dsn()` function (:ticket:`#321`). +- Added :ref:`replication-support` (:ticket:`#322`). Main authors are + Oleksandr Shulgin and Craig Ringer, who deserve a huge thank you. +- Added `~psycopg2.extensions.parse_dsn()` and + `~psycopg2.extensions.make_dsn()` functions (:tickets:`#321, #363`). + `~psycopg2.connect()` now can take both *dsn* and keyword arguments, merging + them together. - Added `~psycopg2.__libpq_version__` and `~psycopg2.extensions.libpq_version()` to inspect the version of the ``libpq`` library the module was compiled/loaded with @@ -14,19 +19,46 @@ New features: - The attributes `~connection.notices` and `~connection.notifies` can be customized replacing them with any object exposing an `!append()` method (:ticket:`#326`). +- Adapt network types to `ipaddress` objects when available. When not + enabled, convert arrays of network types to lists by default. The old `!Inet` + adapter is deprecated (:tickets:`#317, #343, #387`). - Added `~psycopg2.extensions.quote_ident()` function (:ticket:`#359`). +- Added `~connection.get_dsn_parameters()` connection method (:ticket:`#364`). + +Other changes: + +- Dropped support for Python 2.5. +- Dropped support for client library older than PostgreSQL 9.1 (but older + server versions are still supported). + + +What's new in psycopg 2.6.3 +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- Throw an exception trying to pass ``NULL`` chars as parameters + (:ticket:`#420`). +- Make `~psycopg2.extras.Range` objects picklable (:ticket:`#462`). What's new in psycopg 2.6.2 ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Report the server response status on errors (such as :ticket:`#281`). +- Raise `!NotSupportedError` on unhandled server response status + (:ticket:`#352`). +- Allow overriding string adapter encoding with no connection (:ticket:`#331`). - The `~psycopg2.extras.wait_select` callback allows interrupting a long-running query in an interactive shell using :kbd:`Ctrl-C` (:ticket:`#333`). -- Raise `!NotSupportedError` on unhandled server response status - (:ticket:`#352`). - Fixed `!PersistentConnectionPool` on Python 3 (:ticket:`#348`). +- Fixed segfault on `repr()` of an unitialized connection (:ticket:`#361`). +- Allow adapting bytes using QuotedString on Python 3 too (:ticket:`#365`). +- Added support for setuptools/wheel (:ticket:`#370`). +- Fix build on Windows with Python 3.5, VS 2015 (:ticket:`#380`). +- Fixed `!errorcodes.lookup` initialization thread-safety (:ticket:`#382`). +- Fixed `!read()` exception propagation in copy_from (:ticket:`#412`). +- Fixed possible NULL TZ decref (:ticket:`#424`). +- `~psycopg2.errorcodes` map updated to PostgreSQL 9.5. What's new in psycopg 2.6.1 diff --git a/README.rst b/README.rst index 51d2d6b6..f18be564 100644 --- a/README.rst +++ b/README.rst @@ -44,3 +44,8 @@ For any other resource (source code repository, bug tracker, mailing list) please check the `project homepage`__. .. __: http://initd.org/psycopg/ + + +.. image:: https://travis-ci.org/psycopg/psycopg2.svg?branch=master + :target: https://travis-ci.org/psycopg/psycopg2 + :alt: Build Status diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index 82754ee0..5b5fb354 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -47,7 +47,7 @@ it is the class where query building, execution and result type-casting into Python variables happens. The `~psycopg2.extras` module contains several examples of :ref:`connection -and cursor sublcasses `. +and cursor subclasses `. .. note:: @@ -270,7 +270,7 @@ wasting resources. A simple application could poll the connection from time to time to check if something new has arrived. A better strategy is to use some I/O completion -function such as :py:func:`~select.select` to sleep until awaken from the kernel when there is +function such as :py:func:`~select.select` to sleep until awakened by the kernel when there is some data to read on the connection, thereby using no CPU unless there is something to read:: @@ -423,7 +423,7 @@ this will be probably implemented in a future release. Support for coroutine libraries ------------------------------- -.. versionadded:: 2.2.0 +.. versionadded:: 2.2 Psycopg can be used together with coroutine_\-based libraries and participate in cooperative multithreading. @@ -509,3 +509,90 @@ resources about the topic. conn.commit() cur.close() conn.close() + + + +.. index:: + single: Replication + +.. _replication-support: + +Replication protocol support +---------------------------- + +.. versionadded:: 2.7 + +Modern PostgreSQL servers (version 9.0 and above) support replication. The +replication protocol is built on top of the client-server protocol and can be +operated using ``libpq``, as such it can be also operated by ``psycopg2``. +The replication protocol can be operated on both synchronous and +:ref:`asynchronous ` connections. + +Server version 9.4 adds a new feature called *Logical Replication*. + +.. seealso:: + + - PostgreSQL `Streaming Replication Protocol`__ + + .. __: http://www.postgresql.org/docs/current/static/protocol-replication.html + + +Logical replication Quick-Start +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You must be using PostgreSQL server version 9.4 or above to run this quick +start. + +Make sure that replication connections are permitted for user ``postgres`` in +``pg_hba.conf`` and reload the server configuration. You also need to set +``wal_level=logical`` and ``max_wal_senders``, ``max_replication_slots`` to +value greater than zero in ``postgresql.conf`` (these changes require a server +restart). Create a database ``psycopg2_test``. + +Then run the following code to quickly try the replication support out. This +is not production code -- it has no error handling, it sends feedback too +often, etc. -- and it's only intended as a simple demo of logical +replication:: + + from __future__ import print_function + import sys + import psycopg2 + import psycopg2.extras + + conn = psycopg2.connect('dbname=psycopg2_test user=postgres', + connection_factory=psycopg2.extras.LogicalReplicationConnection) + cur = conn.cursor() + try: + # test_decoding produces textual output + cur.start_replication(slot_name='pytest', decode=True) + except psycopg2.ProgrammingError: + cur.create_replication_slot('pytest', output_plugin='test_decoding') + cur.start_replication(slot_name='pytest', decode=True) + + class DemoConsumer(object): + def __call__(self, msg): + print(msg.payload) + msg.cursor.send_feedback(flush_lsn=msg.data_start) + + democonsumer = DemoConsumer() + + print("Starting streaming, press Control-C to end...", file=sys.stderr) + try: + cur.consume_stream(democonsumer) + except KeyboardInterrupt: + cur.close() + conn.close() + print("The slot 'pytest' still exists. Drop it with " + "SELECT pg_drop_replication_slot('pytest'); if no longer needed.", + file=sys.stderr) + print("WARNING: Transaction logs will accumulate in pg_xlog " + "until the slot is dropped.", file=sys.stderr) + + +You can now make changes to the ``psycopg2_test`` database using a normal +psycopg2 session, ``psql``, etc. and see the logical decoding stream printed +by this demo client. + +This will continue running until terminated with ``Control-C``. + +For the details see :ref:`replication-objects`. diff --git a/doc/src/conf.py b/doc/src/conf.py index 18b81e07..a918c08c 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -42,9 +42,7 @@ master_doc = 'index' # General information about the project. project = u'Psycopg' -from datetime import date -year = date.today().year -copyright = u'2001-%s, Federico Di Gregorio, Daniele Varrazzo' % year +copyright = u'2001-2016, Federico Di Gregorio, Daniele Varrazzo' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -63,8 +61,8 @@ except ImportError: release = version intersphinx_mapping = { - 'py': ('http://docs.python.org/', None), - 'py3': ('http://docs.python.org/3.2', None), + 'py': ('http://docs.python.org/2', None), + 'py3': ('http://docs.python.org/3', None), } # Pattern to generate links to the bug tracker diff --git a/doc/src/connection.rst b/doc/src/connection.rst index cceef1e5..c99c8bd8 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -568,6 +568,29 @@ The ``connection`` class .. versionadded:: 2.0.12 + .. index:: + pair: Connection; Parameters + + .. method:: get_dsn_parameters() + + Get the effective dsn parameters for the connection as a dictionary. + + The *password* parameter is removed from the result. + + Example:: + + >>> conn.get_dsn_parameters() + {'dbname': 'test', 'user': 'postgres', 'port': '5432', 'sslmode': 'prefer'} + + Requires libpq >= 9.3. + + .. seealso:: libpq docs for `PQconninfo()`__ for details. + + .. __: http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PQCONNINFO + + .. versionadded:: 2.7 + + .. index:: pair: Transaction; Status diff --git a/doc/src/cursor.rst b/doc/src/cursor.rst index 9df65865..974e1a2b 100644 --- a/doc/src/cursor.rst +++ b/doc/src/cursor.rst @@ -499,6 +499,9 @@ The ``cursor`` class .. rubric:: COPY-related methods + Efficiently copy data from file-like objects to the database and back. See + :ref:`copy` for an overview. + .. extension:: The :sql:`COPY` command is a PostgreSQL extension to the SQL standard. @@ -507,7 +510,7 @@ The ``cursor`` class .. method:: copy_from(file, table, sep='\\t', null='\\\\N', size=8192, columns=None) Read data *from* the file-like object *file* appending them to - the table named *table*. See :ref:`copy` for an overview. + the table named *table*. :param file: file-like object to read data from. It must have both `!read()` and `!readline()` methods. diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index d96cca4f..b661895d 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -12,17 +12,12 @@ The module contains a few objects and function extending the minimum set of functionalities defined by the |DBAPI|_. -.. function:: parse_dsn(dsn) +Classes definitions +------------------- - Parse connection string into a dictionary of keywords and values. - - Uses libpq's ``PQconninfoParse`` to parse the string according to - accepted format(s) and check for supported keywords. - - Example:: - - >>> psycopg2.extensions.parse_dsn('dbname=test user=postgres password=secret') - {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} +Instances of these classes are usually returned by factory functions or +attributes. Their definitions are exposed here to allow subclassing, +introspection etc. .. class:: connection(dsn, async=False) @@ -34,6 +29,7 @@ functionalities defined by the |DBAPI|_. For a complete description of the class, see `connection`. + .. class:: cursor(conn, name=None) It is the class usually returned by the `connection.cursor()` @@ -44,6 +40,7 @@ functionalities defined by the |DBAPI|_. For a complete description of the class, see `cursor`. + .. class:: lobject(conn [, oid [, mode [, new_oid [, new_file ]]]]) Wrapper for a PostgreSQL large object. See :ref:`large-objects` for an @@ -200,39 +197,6 @@ functionalities defined by the |DBAPI|_. server versions. -.. autofunction:: set_wait_callback(f) - - .. versionadded:: 2.2.0 - -.. autofunction:: get_wait_callback() - - .. versionadded:: 2.2.0 - -.. function:: libpq_version() - - Return the version number of the ``libpq`` dynamic library loaded as an - integer, in the same format of `~connection.server_version`. - - Raise `~psycopg2.NotSupportedError` if the ``psycopg2`` module was - compiled with a ``libpq`` version lesser than 9.1 (which can be detected - by the `~psycopg2.__libpq_version__` constant). - - .. seealso:: libpq docs for `PQlibVersion()`__. - - .. __: http://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQLIBVERSION - -.. function:: quote_ident(str, scope) - - Return quoted identifier according to PostgreSQL quoting rules. - - The *scope* must be a `connection` or a `cursor`, the underlying - connection encoding is used for any necessary character conversion. - - Requires libpq >= 9.0. - - .. seealso:: libpq docs for `PQescapeIdentifier()`__ - - .. __: http://www.postgresql.org/docs/current/static/libpq-exec.html#LIBPQ-PQESCAPEIDENTIFIER .. _sql-adaptation-objects: @@ -492,6 +456,106 @@ The module exports a few exceptions in addition to the :ref:`standard ones +.. _coroutines-functions: + +Coroutines support functions +---------------------------- + +These functions are used to set and retrieve the callback function for +:ref:`cooperation with coroutine libraries `. + +.. versionadded:: 2.2.0 + +.. autofunction:: set_wait_callback(f) + +.. autofunction:: get_wait_callback() + + + +Other functions +--------------- + +.. function:: libpq_version() + + Return the version number of the ``libpq`` dynamic library loaded as an + integer, in the same format of `~connection.server_version`. + + Raise `~psycopg2.NotSupportedError` if the ``psycopg2`` module was + compiled with a ``libpq`` version lesser than 9.1 (which can be detected + by the `~psycopg2.__libpq_version__` constant). + + .. versionadded:: 2.7 + + .. seealso:: libpq docs for `PQlibVersion()`__. + + .. __: http://www.postgresql.org/docs/current/static/libpq-misc.html#LIBPQ-PQLIBVERSION + + +.. function:: make_dsn(dsn=None, \*\*kwargs) + + Create a valid connection string from arguments. + + Put together the arguments in *kwargs* into a connection string. If *dsn* + is specified too, merge the arguments coming from both the sources. If the + same argument name is specified in both the sources, the *kwargs* value + overrides the *dsn* value. + + The input arguments are validated: the output should always be a valid + connection string (as far as `parse_dsn()` is concerned). If not raise + `~psycopg2.ProgrammingError`. + + Example:: + + >>> from psycopg2.extensions import make_dsn + >>> make_dsn('dbname=foo host=example.com', password="s3cr3t") + 'host=example.com password=s3cr3t dbname=foo' + + .. versionadded:: 2.7 + + +.. function:: parse_dsn(dsn) + + Parse connection string into a dictionary of keywords and values. + + Parsing is delegated to the libpq: different versions of the client + library may support different formats or parameters (for example, + `connection URIs`__ are only supported from libpq 9.2). Raise + `~psycopg2.ProgrammingError` if the *dsn* is not valid. + + .. __: http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING + + Example:: + + >>> from psycopg2.extensions import parse_dsn + >>> parse_dsn('dbname=test user=postgres password=secret') + {'password': 'secret', 'user': 'postgres', 'dbname': 'test'} + >>> parse_dsn("postgresql://someone@example.com/somedb?connect_timeout=10") + {'host': 'example.com', 'user': 'someone', 'dbname': 'somedb', 'connect_timeout': '10'} + + .. versionadded:: 2.7 + + .. seealso:: libpq docs for `PQconninfoParse()`__. + + .. __: http://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PQCONNINFOPARSE + + +.. function:: quote_ident(str, scope) + + Return quoted identifier according to PostgreSQL quoting rules. + + The *scope* must be a `connection` or a `cursor`, the underlying + connection encoding is used for any necessary character conversion. + + Requires libpq >= 9.0. + + .. versionadded:: 2.7 + + .. seealso:: libpq docs for `PQescapeIdentifier()`__ + + .. __: http://www.postgresql.org/docs/current/static/libpq-exec.html#LIBPQ-PQESCAPEIDENTIFIER + + + .. index:: pair: Isolation level; Constants diff --git a/doc/src/extras.rst b/doc/src/extras.rst index 0e21ae58..d33b8eed 100644 --- a/doc/src/extras.rst +++ b/doc/src/extras.rst @@ -143,6 +143,374 @@ Logging cursor +.. _replication-objects: + +Replication connection and cursor classes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +See :ref:`replication-support` for an introduction to the topic. + + +The following replication types are defined: + +.. data:: REPLICATION_LOGICAL +.. data:: REPLICATION_PHYSICAL + + +.. index:: + pair: Connection; replication + +.. autoclass:: LogicalReplicationConnection + + This connection factory class can be used to open a special type of + connection that is used for logical replication. + + Example:: + + from psycopg2.extras import LogicalReplicationConnection + log_conn = psycopg2.connect(dsn, connection_factory=LogicalReplicationConnection) + log_cur = log_conn.cursor() + + +.. autoclass:: PhysicalReplicationConnection + + This connection factory class can be used to open a special type of + connection that is used for physical replication. + + Example:: + + from psycopg2.extras import PhysicalReplicationConnection + phys_conn = psycopg2.connect(dsn, connection_factory=PhysicalReplicationConnection) + phys_cur = phys_conn.cursor() + + Both `LogicalReplicationConnection` and `PhysicalReplicationConnection` use + `ReplicationCursor` for actual communication with the server. + + +.. index:: + pair: Message; replication + +The individual messages in the replication stream are represented by +`ReplicationMessage` objects (both logical and physical type): + +.. autoclass:: ReplicationMessage + + .. attribute:: payload + + The actual data received from the server. + + An instance of either `bytes()` or `unicode()`, depending on the value + of `decode` option passed to `~ReplicationCursor.start_replication()` + on the connection. See `~ReplicationCursor.read_message()` for + details. + + .. attribute:: data_size + + The raw size of the message payload (before possible unicode + conversion). + + .. attribute:: data_start + + LSN position of the start of the message. + + .. attribute:: wal_end + + LSN position of the current end of WAL on the server. + + .. attribute:: send_time + + A `~datetime` object representing the server timestamp at the moment + when the message was sent. + + .. attribute:: cursor + + A reference to the corresponding `ReplicationCursor` object. + + +.. index:: + pair: Cursor; replication + +.. autoclass:: ReplicationCursor + + .. method:: create_replication_slot(slot_name, slot_type=None, output_plugin=None) + + Create streaming replication slot. + + :param slot_name: name of the replication slot to be created + :param slot_type: type of replication: should be either + `REPLICATION_LOGICAL` or `REPLICATION_PHYSICAL` + :param output_plugin: name of the logical decoding output plugin to be + used by the slot; required for logical + replication connections, disallowed for physical + + Example:: + + log_cur.create_replication_slot("logical1", "test_decoding") + phys_cur.create_replication_slot("physical1") + + # either logical or physical replication connection + cur.create_replication_slot("slot1", slot_type=REPLICATION_LOGICAL) + + When creating a slot on a logical replication connection, a logical + replication slot is created by default. Logical replication requires + name of the logical decoding output plugin to be specified. + + When creating a slot on a physical replication connection, a physical + replication slot is created by default. No output plugin parameter is + required or allowed when creating a physical replication slot. + + In either case the type of slot being created can be specified + explicitly using *slot_type* parameter. + + Replication slots are a feature of PostgreSQL server starting with + version 9.4. + + .. method:: drop_replication_slot(slot_name) + + Drop streaming replication slot. + + :param slot_name: name of the replication slot to drop + + Example:: + + # either logical or physical replication connection + cur.drop_replication_slot("slot1") + + Replication slots are a feature of PostgreSQL server starting with + version 9.4. + + .. method:: start_replication(slot_name=None, slot_type=None, start_lsn=0, timeline=0, options=None, decode=False) + + Start replication on the connection. + + :param slot_name: name of the replication slot to use; required for + logical replication, physical replication can work + with or without a slot + :param slot_type: type of replication: should be either + `REPLICATION_LOGICAL` or `REPLICATION_PHYSICAL` + :param start_lsn: the optional LSN position to start replicating from, + can be an integer or a string of hexadecimal digits + in the form ``XXX/XXX`` + :param timeline: WAL history timeline to start streaming from (optional, + can only be used with physical replication) + :param options: a dictionary of options to pass to logical replication + slot (not allowed with physical replication) + :param decode: a flag indicating that unicode conversion should be + performed on messages received from the server + + If a *slot_name* is specified, the slot must exist on the server and + its type must match the replication type used. + + If not specified using *slot_type* parameter, the type of replication + is defined by the type of replication connection. Logical replication + is only allowed on logical replication connection, but physical + replication can be used with both types of connection. + + On the other hand, physical replication doesn't require a named + replication slot to be used, only logical replication does. In any + case logical replication and replication slots are a feature of + PostgreSQL server starting with version 9.4. Physical replication can + be used starting with 9.0. + + If *start_lsn* is specified, the requested stream will start from that + LSN. The default is `!None` which passes the LSN ``0/0`` causing + replay to begin at the last point for which the server got flush + confirmation from the client, or the oldest available point for a new + slot. + + The server might produce an error if a WAL file for the given LSN has + already been recycled or it may silently start streaming from a later + position: the client can verify the actual position using information + provided by the `ReplicationMessage` attributes. The exact server + behavior depends on the type of replication and use of slots. + + The *timeline* parameter can only be specified with physical + replication and only starting with server version 9.3. + + A dictionary of *options* may be passed to the logical decoding plugin + on a logical replication slot. The set of supported options depends + on the output plugin that was used to create the slot. Must be + `!None` for physical replication. + + If *decode* is set to `!True` the messages received from the server + would be converted according to the connection `~connection.encoding`. + *This parameter should not be set with physical replication or with + logical replication plugins that produce binary output.* + + This function constructs a ``START_REPLICATION`` command and calls + `start_replication_expert()` internally. + + After starting the replication, to actually consume the incoming + server messages use `consume_stream()` or implement a loop around + `read_message()` in case of :ref:`asynchronous connection + `. + + .. method:: start_replication_expert(command, decode=False) + + Start replication on the connection using provided + ``START_REPLICATION`` command. See `start_replication()` for + description of *decode* parameter. + + .. method:: consume_stream(consume, keepalive_interval=10) + + :param consume: a callable object with signature :samp:`consume({msg})` + :param keepalive_interval: interval (in seconds) to send keepalive + messages to the server + + This method can only be used with synchronous connection. For + asynchronous connections see `read_message()`. + + Before using this method to consume the stream call + `start_replication()` first. + + This method enters an endless loop reading messages from the server + and passing them to ``consume()`` one at a time, then waiting for more + messages from the server. In order to make this method break out of + the loop and return, ``consume()`` can throw a `StopReplication` + exception. Any unhandled exception will make it break out of the loop + as well. + + The *msg* object passed to ``consume()`` is an instance of + `ReplicationMessage` class. See `read_message()` for details about + message decoding. + + This method also sends keepalive messages to the server in case there + were no new data from the server for the duration of + *keepalive_interval* (in seconds). The value of this parameter must + be set to at least 1 second, but it can have a fractional part. + + After processing certain amount of messages the client should send a + confirmation message to the server. This should be done by calling + `send_feedback()` method on the corresponding replication cursor. A + reference to the cursor is provided in the `ReplicationMessage` as an + attribute. + + The following example is a sketch implementation of ``consume()`` + callable for logical replication:: + + class LogicalStreamConsumer(object): + + ... + + def __call__(self, msg): + self.process_message(msg.payload) + + if self.should_send_feedback(msg): + msg.cursor.send_feedback(flush_lsn=msg.data_start) + + consumer = LogicalStreamConsumer() + cur.consume_stream(consumer) + + .. warning:: + + When using replication with slots, failure to constantly consume + *and* report success to the server appropriately can eventually + lead to "disk full" condition on the server, because the server + retains all the WAL segments that might be needed to stream the + changes via all of the currently open replication slots. + + On the other hand, it is not recommended to send confirmation + after *every* processed message, since that will put an + unnecessary load on network and the server. A possible strategy + is to confirm after every COMMIT message. + + .. method:: send_feedback(write_lsn=0, flush_lsn=0, apply_lsn=0, reply=False) + + :param write_lsn: a LSN position up to which the client has written the data locally + :param flush_lsn: a LSN position up to which the client has processed the + data reliably (the server is allowed to discard all + and every data that predates this LSN) + :param apply_lsn: a LSN position up to which the warm standby server + has applied the changes (physical replication + master-slave protocol only) + :param reply: request the server to send back a keepalive message immediately + + Use this method to report to the server that all messages up to a + certain LSN position have been processed on the client and may be + discarded on the server. + + This method can also be called with all default parameters' values to + just send a keepalive message to the server. + + Low-level replication cursor methods for :ref:`asynchronous connection + ` operation. + + With the synchronous connection a call to `consume_stream()` handles all + the complexity of handling the incoming messages and sending keepalive + replies, but at times it might be beneficial to use low-level interface + for better control, in particular to `~select` on multiple sockets. The + following methods are provided for asynchronous operation: + + .. method:: read_message() + + Try to read the next message from the server without blocking and + return an instance of `ReplicationMessage` or `!None`, in case there + are no more data messages from the server at the moment. + + This method should be used in a loop with asynchronous connections + (after calling `start_replication()` once). For synchronous + connections see `consume_stream()`. + + The returned message's `~ReplicationMessage.payload` is an instance of + `!unicode` decoded according to connection `~connection.encoding` + *iff* *decode* was set to `!True` in the initial call to + `start_replication()` on this connection, otherwise it is an instance + of `!bytes` with no decoding. + + It is expected that the calling code will call this method repeatedly + in order to consume all of the messages that might have been buffered + until `!None` is returned. After receiving `!None` from this method + the caller should use `~select.select()` or `~select.poll()` on the + corresponding connection to block the process until there is more data + from the server. + + The server can send keepalive messages to the client periodically. + Such messages are silently consumed by this method and are never + reported to the caller. + + .. method:: fileno() + + Call the corresponding connection's `~connection.fileno()` method and + return the result. + + This is a convenience method which allows replication cursor to be + used directly in `~select.select()` or `~select.poll()` calls. + + .. attribute:: io_timestamp + + A `~datetime` object representing the timestamp at the moment of last + communication with the server (a data or keepalive message in either + direction). + + An actual example of asynchronous operation might look like this:: + + from select import select + from datetime import datetime + + def consume(msg): + ... + + keepalive_interval = 10.0 + while True: + msg = cur.read_message() + if msg: + consume(msg) + else: + now = datetime.now() + timeout = keepalive_interval - (now - cur.io_timestamp).total_seconds() + try: + sel = select([cur], [], [], max(0, timeout)) + if not any(sel): + cur.send_feedback() # timed out, send keepalive message + except InterruptedError: + pass # recalculate timeout and continue + +.. index:: + pair: Cursor; Replication + +.. autoclass:: StopReplication + + .. index:: single: Data types; Additional @@ -562,12 +930,29 @@ UUID data type .. index:: pair: INET; Data types + pair: CIDR; Data types + pair: MACADDR; Data types -:sql:`inet` data type -^^^^^^^^^^^^^^^^^^^^^^ +.. _adapt-network: -.. versionadded:: 2.0.9 -.. versionchanged:: 2.4.5 added inet array support. +Networking data types +^^^^^^^^^^^^^^^^^^^^^ + +By default Psycopg casts the PostgreSQL networking data types (:sql:`inet`, +:sql:`cidr`, :sql:`macaddr`) into ordinary strings; array of such types are +converted into lists of strings. + +.. versionchanged:: 2.7 + in previous version array of networking types were not treated as arrays. + +.. autofunction:: register_ipaddress + + +.. autofunction:: register_inet + + .. deprecated:: 2.7 + this function will not receive further development and disappear in + future versions. .. doctest:: @@ -582,10 +967,11 @@ UUID data type '192.168.0.1/24' -.. autofunction:: register_inet - .. autoclass:: Inet + .. deprecated:: 2.7 + this object will not receive further development and may disappear in + future versions. .. index:: diff --git a/doc/src/faq.rst b/doc/src/faq.rst index 69273ba5..89d8a639 100644 --- a/doc/src/faq.rst +++ b/doc/src/faq.rst @@ -73,7 +73,7 @@ Why does `!cursor.execute()` raise the exception *can't adapt*? I can't pass an integer or a float parameter to my query: it says *a number is required*, but *it is* a number! In your query string, you always have to use ``%s`` placeholders, - event when passing a number. All Python objects are converted by Psycopg + even when passing a number. All Python objects are converted by Psycopg in their SQL representation, so they get passed to the query as strings. See :ref:`query-parameters`. :: @@ -241,7 +241,7 @@ How do I interrupt a long-running query in an interactive shell? .. code-block:: pycon - >>> psycopg2.extensions.set_wait_callback(psycopg2.extensions.wait_select) + >>> psycopg2.extensions.set_wait_callback(psycopg2.extras.wait_select) >>> cnn = psycopg2.connect('') >>> cur = cnn.cursor() >>> cur.execute("select pg_sleep(10)") diff --git a/doc/src/install.rst b/doc/src/install.rst index ec1eeea8..4611537e 100644 --- a/doc/src/install.rst +++ b/doc/src/install.rst @@ -17,9 +17,10 @@ The current `!psycopg2` implementation supports: .. NOTE: keep consistent with setup.py and the /features/ page. -- Python 2 versions from 2.5 to 2.7 -- Python 3 versions from 3.1 to 3.4 -- PostgreSQL versions from 7.4 to 9.4 +- Python 2 versions from 2.6 to 2.7 +- Python 3 versions from 3.1 to 3.5 +- PostgreSQL server versions from 7.4 to 9.5 +- PostgreSQL client library version from 9.1 .. _PostgreSQL: http://www.postgresql.org/ .. _Python: http://www.python.org/ @@ -51,6 +52,16 @@ extension packages, *above all if you are a Windows or a Mac OS user*, please use a pre-compiled package and go straight to the :ref:`module usage ` avoid bothering with the gory details. +.. note:: + + Regardless of the way `!psycopg2` is installed, at runtime it will need to + use the libpq_ library. `!psycopg2` relies on the host OS to find the + library file (usually ``libpq.so`` or ``libpq.dll``): if the library is + installed in a standard location there is usually no problem; if the + library is in a non-standard location you will have to tell somehow + psycopg how to find it, which is OS-dependent (for instance setting a + suitable :envvar:`LD_LIBRARY_PATH` on Linux). + .. _install-from-package: @@ -95,7 +106,17 @@ Install from a package pair: Install; Windows **Microsoft Windows** - Jason Erickson maintains a packaged `Windows port of Psycopg`__ with + There are two options to install a precompiled `psycopg2` package under windows: + + **Option 1:** Using `pip`__ (Included in python 2.7.9+ and python 3.4+) + and a binary wheel package. Launch windows' command prompt (`cmd.exe`) + and execute the following command:: + + pip install psycopg2 + + .. __: https://pip.pypa.io/en/stable/installing/ + + **Option 2:** Jason Erickson maintains a packaged `Windows port of Psycopg`__ with installation executable. Download. Double click. Done. .. __: http://www.stickpeople.com/projects/python/win-psycopg/ diff --git a/doc/src/module.rst b/doc/src/module.rst index 6950b703..97fbdf19 100644 --- a/doc/src/module.rst +++ b/doc/src/module.rst @@ -17,37 +17,34 @@ The module interface respects the standard defined in the |DBAPI|_. single: DSN (Database Source Name) .. function:: - connect(dsn, connection_factory=None, cursor_factory=None, async=False) - connect(\*\*kwargs, connection_factory=None, cursor_factory=None, async=False) + connect(dsn=None, connection_factory=None, cursor_factory=None, async=False, \*\*kwargs) Create a new database session and return a new `connection` object. - The connection parameters can be specified either as a `libpq connection + The connection parameters can be specified as a `libpq connection string`__ using the *dsn* parameter:: conn = psycopg2.connect("dbname=test user=postgres password=secret") or using a set of keyword arguments:: - conn = psycopg2.connect(database="test", user="postgres", password="secret") + conn = psycopg2.connect(dbname"test", user="postgres", password="secret") - The two call styles are mutually exclusive: you cannot specify connection - parameters as keyword arguments together with a connection string; only - the parameters not needed for the database connection (*i.e.* - *connection_factory*, *cursor_factory*, and *async*) are supported - together with the *dsn* argument. + or using a mix of both: if the same parameter name is specified in both + sources, the *kwargs* value will have precedence over the *dsn* value. + Note that either the *dsn* or at least one connection-related keyword + argument is required. The basic connection parameters are: - - `!dbname` -- the database name (only in the *dsn* string) - - `!database` -- the database name (only as keyword argument) + - `!dbname` -- the database name (`!database` is a deprecated alias) - `!user` -- user name used to authenticate - `!password` -- password used to authenticate - `!host` -- database host address (defaults to UNIX socket if not provided) - `!port` -- connection port number (defaults to 5432 if not provided) Any other connection parameter supported by the client library/server can - be passed either in the connection string or as keywords. The PostgreSQL + be passed either in the connection string or as a keyword. The PostgreSQL documentation contains the complete list of the `supported parameters`__. Also note that the same parameters can be passed to the client library using `environment variables`__. @@ -76,6 +73,9 @@ The module interface respects the standard defined in the |DBAPI|_. .. versionchanged:: 2.5 added the *cursor_factory* parameter. + .. versionchanged:: 2.7 + both *dsn* and keyword arguments can be specified. + .. seealso:: - `~psycopg2.extensions.parse_dsn` @@ -89,8 +89,8 @@ The module interface respects the standard defined in the |DBAPI|_. .. extension:: - The parameters *connection_factory* and *async* are Psycopg extensions - to the |DBAPI|. + The non-connection-related keyword parameters are Psycopg extensions + to the |DBAPI|_. .. data:: apilevel diff --git a/doc/src/usage.rst b/doc/src/usage.rst index 9dd31df2..e768f372 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -264,7 +264,10 @@ types: +--------------------+-------------------------+--------------------------+ | Anything\ |tm| | :sql:`json` | :ref:`adapt-json` | +--------------------+-------------------------+--------------------------+ - | `uuid` | :sql:`uuid` | :ref:`adapt-uuid` | + | `~uuid.UUID` | :sql:`uuid` | :ref:`adapt-uuid` | + +--------------------+-------------------------+--------------------------+ + | `ipaddress` | | :sql:`inet` | :ref:`adapt-network` | + | objects | | :sql:`cidr` | | +--------------------+-------------------------+--------------------------+ .. |tm| unicode:: U+2122 @@ -864,11 +867,19 @@ Using COPY TO and COPY FROM Psycopg `cursor` objects provide an interface to the efficient PostgreSQL |COPY|__ command to move data from files to tables and back. + +Currently no adaptation is provided between Python and PostgreSQL types on +|COPY|: the file can be any Python file-like object but its format must be in +the format accepted by `PostgreSQL COPY command`__ (data fromat, escaped +characters, etc). + +.. __: COPY_ + The methods exposed are: `~cursor.copy_from()` Reads data *from* a file-like object appending them to a database table - (:sql:`COPY table FROM file` syntax). The source file must have both + (:sql:`COPY table FROM file` syntax). The source file must provide both `!read()` and `!readline()` method. `~cursor.copy_to()` diff --git a/lib/__init__.py b/lib/__init__.py index 994b15a8..fb22b4c0 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -47,19 +47,20 @@ Homepage: http://initd.org/projects/psycopg2 # Import the DBAPI-2.0 stuff into top-level module. -from psycopg2._psycopg import BINARY, NUMBER, STRING, DATETIME, ROWID +from psycopg2._psycopg import ( # noqa + BINARY, NUMBER, STRING, DATETIME, ROWID, -from psycopg2._psycopg import Binary, Date, Time, Timestamp -from psycopg2._psycopg import DateFromTicks, TimeFromTicks, TimestampFromTicks + Binary, Date, Time, Timestamp, + DateFromTicks, TimeFromTicks, TimestampFromTicks, -from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, ProgrammingError -from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError -from psycopg2._psycopg import NotSupportedError, OperationalError + Error, Warning, DataError, DatabaseError, ProgrammingError, IntegrityError, + InterfaceError, InternalError, NotSupportedError, OperationalError, -from psycopg2._psycopg import _connect, apilevel, threadsafety, paramstyle -from psycopg2._psycopg import __version__, __libpq_version__ + _connect, apilevel, threadsafety, paramstyle, + __version__, __libpq_version__, +) -from psycopg2 import tz +from psycopg2 import tz # noqa # Register default adapters. @@ -80,32 +81,13 @@ else: _ext.register_adapter(Decimal, Adapter) del Decimal, Adapter -import re -def _param_escape(s, - re_escape=re.compile(r"([\\'])"), - re_space=re.compile(r'\s')): - """ - Apply the escaping rule required by PQconnectdb - """ - if not s: return "''" - - s = re_escape.sub(r'\\\1', s) - if re_space.search(s): - s = "'" + s + "'" - - return s - -del re - - -def connect(dsn=None, - database=None, user=None, password=None, host=None, port=None, - connection_factory=None, cursor_factory=None, async=False, **kwargs): +def connect(dsn=None, connection_factory=None, cursor_factory=None, + async=False, **kwargs): """ Create a new database connection. - The connection parameters can be specified either as a string: + The connection parameters can be specified as a string: conn = psycopg2.connect("dbname=test user=postgres password=secret") @@ -113,9 +95,9 @@ def connect(dsn=None, conn = psycopg2.connect(database="test", user="postgres", password="secret") - The basic connection parameters are: + Or as a mix of both. The basic connection parameters are: - - *dbname*: the database name (only in dsn string) + - *dbname*: the database name - *database*: the database name (only as keyword argument) - *user*: user name used to authenticate - *password*: password used to authenticate @@ -135,32 +117,10 @@ def connect(dsn=None, library: the list of supported parameters depends on the library version. """ - items = [] - if database is not None: - items.append(('dbname', database)) - if user is not None: - items.append(('user', user)) - if password is not None: - items.append(('password', password)) - if host is not None: - items.append(('host', host)) - if port is not None: - items.append(('port', port)) - - items.extend([(k, v) for (k, v) in kwargs.iteritems() if v is not None]) - - if dsn is not None and items: - raise TypeError( - "'%s' is an invalid keyword argument when the dsn is specified" - % items[0][0]) - - if dsn is None: - if not items: - raise TypeError('missing dsn and no parameters') - else: - dsn = " ".join(["%s=%s" % (k, _param_escape(str(v))) - for (k, v) in items]) + if dsn is None and not kwargs: + raise TypeError('missing dsn and no parameters') + dsn = _ext.make_dsn(dsn, **kwargs) conn = _connect(dsn, connection_factory=connection_factory, async=async) if cursor_factory is not None: conn.cursor_factory = cursor_factory diff --git a/lib/_ipaddress.py b/lib/_ipaddress.py new file mode 100644 index 00000000..ee05a260 --- /dev/null +++ b/lib/_ipaddress.py @@ -0,0 +1,89 @@ +"""Implementation of the ipaddres-based network types adaptation +""" + +# psycopg/_ipaddress.py - Ipaddres-based network types adaptation +# +# Copyright (C) 2016 Daniele Varrazzo +# +# psycopg2 is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# In addition, as a special exception, the copyright holders give +# permission to link this program with the OpenSSL library (or with +# modified versions of OpenSSL that use the same license as OpenSSL), +# and distribute linked combinations including the two. +# +# You must obey the GNU Lesser General Public License in all respects for +# all of the code used other than OpenSSL. +# +# psycopg2 is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +from psycopg2.extensions import ( + new_type, new_array_type, register_type, register_adapter, QuotedString) + +# The module is imported on register_ipaddress +ipaddress = None + +# The typecasters are created only once +_casters = None + + +def register_ipaddress(conn_or_curs=None): + """ + Register conversion support between `ipaddress` objects and `network types`__. + + :param conn_or_curs: the scope where to register the type casters. + If `!None` register them globally. + + After the function is called, PostgreSQL :sql:`inet` values will be + converted into `~ipaddress.IPv4Interface` or `~ipaddress.IPv6Interface` + objects, :sql:`cidr` values into into `~ipaddress.IPv4Network` or + `~ipaddress.IPv6Network`. + + .. __: https://www.postgresql.org/docs/current/static/datatype-net-types.html + """ + global ipaddress + import ipaddress + + global _casters + if _casters is None: + _casters = _make_casters() + + for c in _casters: + register_type(c, conn_or_curs) + + for t in [ipaddress.IPv4Interface, ipaddress.IPv6Interface, + ipaddress.IPv4Network, ipaddress.IPv6Network]: + register_adapter(t, adapt_ipaddress) + + +def _make_casters(): + inet = new_type((869,), 'INET', cast_interface) + ainet = new_array_type((1041,), 'INET[]', inet) + + cidr = new_type((650,), 'CIDR', cast_network) + acidr = new_array_type((651,), 'CIDR[]', cidr) + + return [inet, ainet, cidr, acidr] + + +def cast_interface(s, cur=None): + if s is None: + return None + # Py2 version force the use of unicode. meh. + return ipaddress.ip_interface(unicode(s)) + + +def cast_network(s, cur=None): + if s is None: + return None + return ipaddress.ip_network(unicode(s)) + + +def adapt_ipaddress(obj): + return QuotedString(str(obj)) diff --git a/lib/_json.py b/lib/_json.py index 26e32f2f..b137a2d9 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -34,7 +34,7 @@ from psycopg2._psycopg import new_type, new_array_type, register_type # import the best json implementation available -if sys.version_info[:2] >= (2,6): +if sys.version_info[:2] >= (2, 6): import json else: try: @@ -51,6 +51,7 @@ JSONARRAY_OID = 199 JSONB_OID = 3802 JSONBARRAY_OID = 3807 + class Json(object): """ An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to @@ -106,7 +107,7 @@ class Json(object): def register_json(conn_or_curs=None, globally=False, loads=None, - oid=None, array_oid=None, name='json'): + oid=None, array_oid=None, name='json'): """Create and register typecasters converting :sql:`json` type to Python objects. :param conn_or_curs: a connection or cursor used to find the :sql:`json` @@ -143,6 +144,7 @@ def register_json(conn_or_curs=None, globally=False, loads=None, return JSON, JSONARRAY + def register_default_json(conn_or_curs=None, globally=False, loads=None): """ Create and register :sql:`json` typecasters for PostgreSQL 9.2 and following. @@ -155,6 +157,7 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) + def register_default_jsonb(conn_or_curs=None, globally=False, loads=None): """ Create and register :sql:`jsonb` typecasters for PostgreSQL 9.4 and following. @@ -167,6 +170,7 @@ def register_default_jsonb(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSONB_OID, array_oid=JSONBARRAY_OID, name='jsonb') + def _create_json_typecasters(oid, array_oid, loads=None, name='JSON'): """Create typecasters for json data type.""" if loads is None: @@ -188,6 +192,7 @@ def _create_json_typecasters(oid, array_oid, loads=None, name='JSON'): return JSON, JSONARRAY + def _get_json_oids(conn_or_curs, name='json'): # lazy imports from psycopg2.extensions import STATUS_IN_TRANSACTION @@ -204,7 +209,7 @@ def _get_json_oids(conn_or_curs, name='json'): # get the oid for the hstore curs.execute( "SELECT t.oid, %s FROM pg_type t WHERE t.typname = %%s;" - % typarray, (name,)) + % typarray, (name,)) r = curs.fetchone() # revert the status of the connection as before the command @@ -215,6 +220,3 @@ def _get_json_oids(conn_or_curs, name='json'): raise conn.ProgrammingError("%s data type not found" % name) return r - - - diff --git a/lib/_range.py b/lib/_range.py index 47b82086..ee9c329e 100644 --- a/lib/_range.py +++ b/lib/_range.py @@ -27,9 +27,10 @@ import re from psycopg2._psycopg import ProgrammingError, InterfaceError -from psycopg2.extensions import ISQLQuote, adapt, register_adapter, b +from psycopg2.extensions import ISQLQuote, adapt, register_adapter from psycopg2.extensions import new_type, new_array_type, register_type + class Range(object): """Python representation for a PostgreSQL |range|_ type. @@ -78,42 +79,50 @@ class Range(object): @property def lower_inf(self): """`!True` if the range doesn't have a lower bound.""" - if self._bounds is None: return False + if self._bounds is None: + return False return self._lower is None @property def upper_inf(self): """`!True` if the range doesn't have an upper bound.""" - if self._bounds is None: return False + if self._bounds is None: + return False return self._upper is None @property def lower_inc(self): """`!True` if the lower bound is included in the range.""" - if self._bounds is None: return False - if self._lower is None: return False + if self._bounds is None or self._lower is None: + return False return self._bounds[0] == '[' @property def upper_inc(self): """`!True` if the upper bound is included in the range.""" - if self._bounds is None: return False - if self._upper is None: return False + if self._bounds is None or self._upper is None: + return False return self._bounds[1] == ']' def __contains__(self, x): - if self._bounds is None: return False + if self._bounds is None: + return False + if self._lower is not None: if self._bounds[0] == '[': - if x < self._lower: return False + if x < self._lower: + return False else: - if x <= self._lower: return False + if x <= self._lower: + return False if self._upper is not None: if self._bounds[1] == ']': - if x > self._upper: return False + if x > self._upper: + return False else: - if x >= self._upper: return False + if x >= self._upper: + return False return True @@ -171,6 +180,17 @@ class Range(object): else: return self.__gt__(other) + def __getstate__(self): + return dict( + (slot, getattr(self, slot)) + for slot in self.__slots__ + if hasattr(self, slot) + ) + + def __setstate__(self, state): + for slot, value in state.items(): + setattr(self, slot, value) + def register_range(pgrange, pyrange, conn_or_curs, globally=False): """Create and register an adapter and the typecasters to convert between @@ -229,7 +249,7 @@ class RangeAdapter(object): r = self.adapted if r.isempty: - return b("'empty'::" + self.name) + return b"'empty'::" + self.name.encode('utf8') if r.lower is not None: a = adapt(r.lower) @@ -237,7 +257,7 @@ class RangeAdapter(object): a.prepare(self._conn) lower = a.getquoted() else: - lower = b('NULL') + lower = b'NULL' if r.upper is not None: a = adapt(r.upper) @@ -245,10 +265,10 @@ class RangeAdapter(object): a.prepare(self._conn) upper = a.getquoted() else: - upper = b('NULL') + upper = b'NULL' - return b(self.name + '(') + lower + b(', ') + upper \ - + b(", '%s')" % r._bounds) + return self.name.encode('utf8') + b'(' + lower + b', ' + upper \ + + b", '" + r._bounds.encode('utf8') + b"')" class RangeCaster(object): @@ -284,7 +304,8 @@ class RangeCaster(object): self.adapter.name = pgrange else: try: - if issubclass(pgrange, RangeAdapter) and pgrange is not RangeAdapter: + if issubclass(pgrange, RangeAdapter) \ + and pgrange is not RangeAdapter: self.adapter = pgrange except TypeError: pass @@ -425,14 +446,17 @@ class NumericRange(Range): """ pass + class DateRange(Range): """Represents :sql:`daterange` values.""" pass + class DateTimeRange(Range): """Represents :sql:`tsrange` values.""" pass + class DateTimeTZRange(Range): """Represents :sql:`tstzrange` values.""" pass @@ -448,7 +472,7 @@ class NumberRangeAdapter(RangeAdapter): def getquoted(self): r = self.adapted if r.isempty: - return b("'empty'") + return b"'empty'" if not r.lower_inf: # not exactly: we are relying that none of these object is really @@ -497,5 +521,3 @@ tsrange_caster._register() tstzrange_caster = RangeCaster('tstzrange', DateTimeTZRange, oid=3910, subtype_oid=1184, array_oid=3911) tstzrange_caster._register() - - diff --git a/lib/errorcodes.py b/lib/errorcodes.py index 12c300f6..f56e25ab 100644 --- a/lib/errorcodes.py +++ b/lib/errorcodes.py @@ -29,6 +29,7 @@ This module contains symbolic names for all PostgreSQL error codes. # http://www.postgresql.org/docs/current/static/errcodes-appendix.html # + def lookup(code, _cache={}): """Lookup an error code or class code and return its symbolic name. @@ -38,11 +39,17 @@ def lookup(code, _cache={}): return _cache[code] # Generate the lookup map at first usage. + tmp = {} for k, v in globals().iteritems(): if isinstance(v, str) and len(v) in (2, 5): - _cache[v] = k + tmp[v] = k - return lookup(code) + assert tmp + + # Atomic update, to avoid race condition on import (bug #382) + _cache.update(tmp) + + return _cache[code] # autogenerated data: do not edit below this point. @@ -193,6 +200,8 @@ INVALID_ESCAPE_SEQUENCE = '22025' STRING_DATA_LENGTH_MISMATCH = '22026' TRIM_ERROR = '22027' ARRAY_SUBSCRIPT_ERROR = '2202E' +INVALID_TABLESAMPLE_REPEAT = '2202G' +INVALID_TABLESAMPLE_ARGUMENT = '2202H' FLOATING_POINT_EXCEPTION = '22P01' INVALID_TEXT_REPRESENTATION = '22P02' INVALID_BINARY_REPRESENTATION = '22P03' @@ -265,6 +274,7 @@ INVALID_SQLSTATE_RETURNED = '39001' NULL_VALUE_NOT_ALLOWED = '39004' TRIGGER_PROTOCOL_VIOLATED = '39P01' SRF_PROTOCOL_VIOLATED = '39P02' +EVENT_TRIGGER_PROTOCOL_VIOLATED = '39P03' # Class 3B - Savepoint Exception SAVEPOINT_EXCEPTION = '3B000' @@ -402,6 +412,7 @@ PLPGSQL_ERROR = 'P0000' RAISE_EXCEPTION = 'P0001' NO_DATA_FOUND = 'P0002' TOO_MANY_ROWS = 'P0003' +ASSERT_FAILURE = 'P0004' # Class XX - Internal Error INTERNAL_ERROR = 'XX000' diff --git a/lib/extensions.py b/lib/extensions.py index b40e28b8..b123e881 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -7,7 +7,7 @@ This module holds all the extensions to the DBAPI-2.0 provided by psycopg. - `lobject` -- the new-type inheritable large object class - `adapt()` -- exposes the PEP-246_ compatible adapting mechanism used by psycopg to adapt Python types to PostgreSQL ones - + .. _PEP-246: http://www.python.org/peps/pep-0246.html """ # psycopg/extensions.py - DBAPI-2.0 extensions specific to psycopg @@ -32,81 +32,74 @@ This module holds all the extensions to the DBAPI-2.0 provided by psycopg. # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. -from psycopg2._psycopg import UNICODE, INTEGER, LONGINTEGER, BOOLEAN, FLOAT -from psycopg2._psycopg import TIME, DATE, INTERVAL, DECIMAL -from psycopg2._psycopg import BINARYARRAY, BOOLEANARRAY, DATEARRAY, DATETIMEARRAY -from psycopg2._psycopg import DECIMALARRAY, FLOATARRAY, INTEGERARRAY, INTERVALARRAY -from psycopg2._psycopg import LONGINTEGERARRAY, ROWIDARRAY, STRINGARRAY, TIMEARRAY -from psycopg2._psycopg import UNICODEARRAY +import re as _re + +from psycopg2._psycopg import ( # noqa + BINARYARRAY, BOOLEAN, BOOLEANARRAY, DATE, DATEARRAY, DATETIMEARRAY, + DECIMAL, DECIMALARRAY, FLOAT, FLOATARRAY, INTEGER, INTEGERARRAY, + INTERVAL, INTERVALARRAY, LONGINTEGER, LONGINTEGERARRAY, ROWIDARRAY, + STRINGARRAY, TIME, TIMEARRAY, UNICODE, UNICODEARRAY, + AsIs, Binary, Boolean, Float, Int, QuotedString, ) -from psycopg2._psycopg import Binary, Boolean, Int, Float, QuotedString, AsIs try: - from psycopg2._psycopg import MXDATE, MXDATETIME, MXINTERVAL, MXTIME - from psycopg2._psycopg import MXDATEARRAY, MXDATETIMEARRAY, MXINTERVALARRAY, MXTIMEARRAY - from psycopg2._psycopg import DateFromMx, TimeFromMx, TimestampFromMx - from psycopg2._psycopg import IntervalFromMx + from psycopg2._psycopg import ( # noqa + MXDATE, MXDATETIME, MXINTERVAL, MXTIME, + MXDATEARRAY, MXDATETIMEARRAY, MXINTERVALARRAY, MXTIMEARRAY, + DateFromMx, TimeFromMx, TimestampFromMx, IntervalFromMx, ) except ImportError: pass try: - from psycopg2._psycopg import PYDATE, PYDATETIME, PYINTERVAL, PYTIME - from psycopg2._psycopg import PYDATEARRAY, PYDATETIMEARRAY, PYINTERVALARRAY, PYTIMEARRAY - from psycopg2._psycopg import DateFromPy, TimeFromPy, TimestampFromPy - from psycopg2._psycopg import IntervalFromPy + from psycopg2._psycopg import ( # noqa + PYDATE, PYDATETIME, PYINTERVAL, PYTIME, + PYDATEARRAY, PYDATETIMEARRAY, PYINTERVALARRAY, PYTIMEARRAY, + DateFromPy, TimeFromPy, TimestampFromPy, IntervalFromPy, ) except ImportError: pass -from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid, libpq_version, parse_dsn, quote_ident -from psycopg2._psycopg import string_types, binary_types, new_type, new_array_type, register_type -from psycopg2._psycopg import ISQLQuote, Notify, Diagnostics, Column +from psycopg2._psycopg import ( # noqa + adapt, adapters, encodings, connection, cursor, + lobject, Xid, libpq_version, parse_dsn, quote_ident, + string_types, binary_types, new_type, new_array_type, register_type, + ISQLQuote, Notify, Diagnostics, Column, + QueryCanceledError, TransactionRollbackError, + set_wait_callback, get_wait_callback, ) -from psycopg2._psycopg import QueryCanceledError, TransactionRollbackError - -try: - from psycopg2._psycopg import set_wait_callback, get_wait_callback -except ImportError: - pass """Isolation level values.""" -ISOLATION_LEVEL_AUTOCOMMIT = 0 -ISOLATION_LEVEL_READ_UNCOMMITTED = 4 -ISOLATION_LEVEL_READ_COMMITTED = 1 -ISOLATION_LEVEL_REPEATABLE_READ = 2 -ISOLATION_LEVEL_SERIALIZABLE = 3 +ISOLATION_LEVEL_AUTOCOMMIT = 0 +ISOLATION_LEVEL_READ_UNCOMMITTED = 4 +ISOLATION_LEVEL_READ_COMMITTED = 1 +ISOLATION_LEVEL_REPEATABLE_READ = 2 +ISOLATION_LEVEL_SERIALIZABLE = 3 + """psycopg connection status values.""" -STATUS_SETUP = 0 -STATUS_READY = 1 -STATUS_BEGIN = 2 -STATUS_SYNC = 3 # currently unused -STATUS_ASYNC = 4 # currently unused +STATUS_SETUP = 0 +STATUS_READY = 1 +STATUS_BEGIN = 2 +STATUS_SYNC = 3 # currently unused +STATUS_ASYNC = 4 # currently unused STATUS_PREPARED = 5 # This is a useful mnemonic to check if the connection is in a transaction STATUS_IN_TRANSACTION = STATUS_BEGIN + """psycopg asynchronous connection polling values""" -POLL_OK = 0 -POLL_READ = 1 +POLL_OK = 0 +POLL_READ = 1 POLL_WRITE = 2 POLL_ERROR = 3 + """Backend transaction status values.""" -TRANSACTION_STATUS_IDLE = 0 -TRANSACTION_STATUS_ACTIVE = 1 +TRANSACTION_STATUS_IDLE = 0 +TRANSACTION_STATUS_ACTIVE = 1 TRANSACTION_STATUS_INTRANS = 2 TRANSACTION_STATUS_INERROR = 3 TRANSACTION_STATUS_UNKNOWN = 4 -import sys as _sys - -# Return bytes from a string -if _sys.version_info[0] < 3: - def b(s): - return s -else: - def b(s): - return s.encode('utf8') def register_adapter(typ, callable): """Register 'callable' as an ISQLQuote adapter for type 'typ'.""" @@ -132,7 +125,7 @@ class SQL_IN(object): if hasattr(obj, 'prepare'): obj.prepare(self._conn) qobjs = [o.getquoted() for o in pobjs] - return b('(') + b(', ').join(qobjs) + b(')') + return b'(' + b', '.join(qobjs) + b')' def __str__(self): return str(self.getquoted()) @@ -147,12 +140,59 @@ class NoneAdapter(object): def __init__(self, obj): pass - def getquoted(self, _null=b("NULL")): + def getquoted(self, _null=b"NULL"): return _null +def make_dsn(dsn=None, **kwargs): + """Convert a set of keywords into a connection strings.""" + if dsn is None and not kwargs: + return '' + + # If no kwarg is specified don't mung the dsn, but verify it + if not kwargs: + parse_dsn(dsn) + return dsn + + # Override the dsn with the parameters + if 'database' in kwargs: + if 'dbname' in kwargs: + raise TypeError( + "you can't specify both 'database' and 'dbname' arguments") + kwargs['dbname'] = kwargs.pop('database') + + if dsn is not None: + tmp = parse_dsn(dsn) + tmp.update(kwargs) + kwargs = tmp + + dsn = " ".join(["%s=%s" % (k, _param_escape(str(v))) + for (k, v) in kwargs.iteritems()]) + + # verify that the returned dsn is valid + parse_dsn(dsn) + + return dsn + + +def _param_escape(s, + re_escape=_re.compile(r"([\\'])"), + re_space=_re.compile(r'\s')): + """ + Apply the escaping rule required by PQconnectdb + """ + if not s: + return "''" + + s = re_escape.sub(r'\\\1', s) + if re_space.search(s): + s = "'" + s + "'" + + return s + + # Create default json typecasters for PostgreSQL 9.2 oids -from psycopg2._json import register_default_json, register_default_jsonb +from psycopg2._json import register_default_json, register_default_jsonb # noqa try: JSON, JSONARRAY = register_default_json() @@ -164,7 +204,7 @@ del register_default_json, register_default_jsonb # Create default Range typecasters -from psycopg2. _range import Range +from psycopg2. _range import Range # noqa del Range diff --git a/lib/extras.py b/lib/extras.py index 2713d6fc..7fc853a6 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -39,8 +39,28 @@ import psycopg2 from psycopg2 import extensions as _ext from psycopg2.extensions import cursor as _cursor from psycopg2.extensions import connection as _connection -from psycopg2.extensions import adapt as _A -from psycopg2.extensions import b +from psycopg2.extensions import adapt as _A, quote_ident + +from psycopg2._psycopg import ( # noqa + REPLICATION_PHYSICAL, REPLICATION_LOGICAL, + ReplicationConnection as _replicationConnection, + ReplicationCursor as _replicationCursor, + ReplicationMessage) + + +# expose the json adaptation stuff into the module +from psycopg2._json import ( # noqa + json, Json, register_json, register_default_json, register_default_jsonb) + + +# Expose range-related objects +from psycopg2._range import ( # noqa + Range, NumericRange, DateRange, DateTimeRange, DateTimeTZRange, + register_range, RangeAdapter, RangeCaster) + + +# Expose ipaddress-related objects +from psycopg2._ipaddress import register_ipaddress # noqa class DictCursorBase(_cursor): @@ -106,6 +126,7 @@ class DictConnection(_connection): kwargs.setdefault('cursor_factory', DictCursor) return super(DictConnection, self).cursor(*args, **kwargs) + class DictCursor(DictCursorBase): """A cursor that keeps a list of column name -> index mappings.""" @@ -130,6 +151,7 @@ class DictCursor(DictCursorBase): self.index[self.description[i][0]] = i self._query_executed = 0 + class DictRow(list): """A row object that allow by-column-name access to data.""" @@ -192,10 +214,10 @@ class DictRow(list): # drop the crusty Py2 methods if _sys.version_info[0] > 2: - items = iteritems; del iteritems - keys = iterkeys; del iterkeys - values = itervalues; del itervalues - del has_key + items = iteritems # noqa + keys = iterkeys # noqa + values = itervalues # noqa + del iteritems, iterkeys, itervalues, has_key class RealDictConnection(_connection): @@ -204,6 +226,7 @@ class RealDictConnection(_connection): kwargs.setdefault('cursor_factory', RealDictCursor) return super(RealDictConnection, self).cursor(*args, **kwargs) + class RealDictCursor(DictCursorBase): """A cursor that uses a real dict as the base type for rows. @@ -233,6 +256,7 @@ class RealDictCursor(DictCursorBase): self.column_mapping.append(self.description[i][0]) self._query_executed = 0 + class RealDictRow(dict): """A `!dict` subclass representing a data record.""" @@ -265,6 +289,7 @@ class NamedTupleConnection(_connection): kwargs.setdefault('cursor_factory', NamedTupleCursor) return super(NamedTupleConnection, self).cursor(*args, **kwargs) + class NamedTupleCursor(_cursor): """A cursor that generates results as `~collections.namedtuple`. @@ -369,11 +394,13 @@ class LoggingConnection(_connection): def _logtofile(self, msg, curs): msg = self.filter(msg, curs) - if msg: self._logobj.write(msg + _os.linesep) + if msg: + self._logobj.write(msg + _os.linesep) def _logtologger(self, msg, curs): msg = self.filter(msg, curs) - if msg: self._logobj.debug(msg) + if msg: + self._logobj.debug(msg) def _check(self): if not hasattr(self, '_logobj'): @@ -385,6 +412,7 @@ class LoggingConnection(_connection): kwargs.setdefault('cursor_factory', LoggingCursor) return super(LoggingConnection, self).cursor(*args, **kwargs) + class LoggingCursor(_cursor): """A cursor that logs queries using its connection logging facilities.""" @@ -425,6 +453,7 @@ class MinTimeLoggingConnection(LoggingConnection): kwargs.setdefault('cursor_factory', MinTimeLoggingCursor) return LoggingConnection.cursor(self, *args, **kwargs) + class MinTimeLoggingCursor(LoggingCursor): """The cursor sub-class companion to `MinTimeLoggingConnection`.""" @@ -437,6 +466,133 @@ class MinTimeLoggingCursor(LoggingCursor): return LoggingCursor.callproc(self, procname, vars) +class LogicalReplicationConnection(_replicationConnection): + + def __init__(self, *args, **kwargs): + kwargs['replication_type'] = REPLICATION_LOGICAL + super(LogicalReplicationConnection, self).__init__(*args, **kwargs) + + +class PhysicalReplicationConnection(_replicationConnection): + + def __init__(self, *args, **kwargs): + kwargs['replication_type'] = REPLICATION_PHYSICAL + super(PhysicalReplicationConnection, self).__init__(*args, **kwargs) + + +class StopReplication(Exception): + """ + Exception used to break out of the endless loop in + `~ReplicationCursor.consume_stream()`. + + Subclass of `~exceptions.Exception`. Intentionally *not* inherited from + `~psycopg2.Error` as occurrence of this exception does not indicate an + error. + """ + pass + + +class ReplicationCursor(_replicationCursor): + """A cursor used for communication on replication connections.""" + + def create_replication_slot(self, slot_name, slot_type=None, output_plugin=None): + """Create streaming replication slot.""" + + command = "CREATE_REPLICATION_SLOT %s " % quote_ident(slot_name, self) + + if slot_type is None: + slot_type = self.connection.replication_type + + if slot_type == REPLICATION_LOGICAL: + if output_plugin is None: + raise psycopg2.ProgrammingError( + "output plugin name is required to create " + "logical replication slot") + + command += "LOGICAL %s" % quote_ident(output_plugin, self) + + elif slot_type == REPLICATION_PHYSICAL: + if output_plugin is not None: + raise psycopg2.ProgrammingError( + "cannot specify output plugin name when creating " + "physical replication slot") + + command += "PHYSICAL" + + else: + raise psycopg2.ProgrammingError( + "unrecognized replication type: %s" % repr(slot_type)) + + self.execute(command) + + def drop_replication_slot(self, slot_name): + """Drop streaming replication slot.""" + + command = "DROP_REPLICATION_SLOT %s" % quote_ident(slot_name, self) + self.execute(command) + + def start_replication(self, slot_name=None, slot_type=None, start_lsn=0, + timeline=0, options=None, decode=False): + """Start replication stream.""" + + command = "START_REPLICATION " + + if slot_type is None: + slot_type = self.connection.replication_type + + if slot_type == REPLICATION_LOGICAL: + if slot_name: + command += "SLOT %s " % quote_ident(slot_name, self) + else: + raise psycopg2.ProgrammingError( + "slot name is required for logical replication") + + command += "LOGICAL " + + elif slot_type == REPLICATION_PHYSICAL: + if slot_name: + command += "SLOT %s " % quote_ident(slot_name, self) + # don't add "PHYSICAL", before 9.4 it was just START_REPLICATION XXX/XXX + + else: + raise psycopg2.ProgrammingError( + "unrecognized replication type: %s" % repr(slot_type)) + + if type(start_lsn) is str: + lsn = start_lsn.split('/') + lsn = "%X/%08X" % (int(lsn[0], 16), int(lsn[1], 16)) + else: + lsn = "%X/%08X" % ((start_lsn >> 32) & 0xFFFFFFFF, + start_lsn & 0xFFFFFFFF) + + command += lsn + + if timeline != 0: + if slot_type == REPLICATION_LOGICAL: + raise psycopg2.ProgrammingError( + "cannot specify timeline for logical replication") + + command += " TIMELINE %d" % timeline + + if options: + if slot_type == REPLICATION_PHYSICAL: + raise psycopg2.ProgrammingError( + "cannot specify output plugin options for physical replication") + + command += " (" + for k, v in options.iteritems(): + if not command.endswith('('): + command += ", " + command += "%s %s" % (quote_ident(k, self), _A(str(v))) + command += ")" + + self.start_replication_expert(command, decode=decode) + + # allows replication cursors to be used in select.select() directly + def fileno(self): + return self.connection.fileno() + + # a dbtype and adapter for Python UUID type class UUID_adapter(object): @@ -454,11 +610,12 @@ class UUID_adapter(object): return self def getquoted(self): - return b("'%s'::uuid" % self._uuid) + return ("'%s'::uuid" % self._uuid).encode('utf8') def __str__(self): return "'%s'::uuid" % self._uuid + def register_uuid(oids=None, conn_or_curs=None): """Create the UUID type and an uuid.UUID adapter. @@ -514,7 +671,7 @@ class Inet(object): obj = _A(self.addr) if hasattr(obj, 'prepare'): obj.prepare(self._conn) - return obj.getquoted() + b("::inet") + return obj.getquoted() + b"::inet" def __conform__(self, proto): if proto is _ext.ISQLQuote: @@ -523,6 +680,7 @@ class Inet(object): def __str__(self): return str(self.addr) + def register_inet(oid=None, conn_or_curs=None): """Create the INET type and an Inet adapter. @@ -532,6 +690,11 @@ def register_inet(oid=None, conn_or_curs=None): :param conn_or_curs: where to register the typecaster. If not specified, register it globally. """ + import warnings + warnings.warn( + "the inet adapter is deprecated, it's not very useful", + DeprecationWarning) + if not oid: oid1 = 869 oid2 = 1041 @@ -621,7 +784,7 @@ class HstoreAdapter(object): def _getquoted_8(self): """Use the operators available in PG pre-9.0.""" if not self.wrapped: - return b("''::hstore") + return b"''::hstore" adapt = _ext.adapt rv = [] @@ -635,23 +798,23 @@ class HstoreAdapter(object): v.prepare(self.conn) v = v.getquoted() else: - v = b('NULL') + v = b'NULL' # XXX this b'ing is painfully inefficient! - rv.append(b("(") + k + b(" => ") + v + b(")")) + rv.append(b"(" + k + b" => " + v + b")") - return b("(") + b('||').join(rv) + b(")") + return b"(" + b'||'.join(rv) + b")" def _getquoted_9(self): """Use the hstore(text[], text[]) function.""" if not self.wrapped: - return b("''::hstore") + return b"''::hstore" k = _ext.adapt(self.wrapped.keys()) k.prepare(self.conn) v = _ext.adapt(self.wrapped.values()) v.prepare(self.conn) - return b("hstore(") + k.getquoted() + b(", ") + v.getquoted() + b(")") + return b"hstore(" + k.getquoted() + b", " + v.getquoted() + b")" getquoted = _getquoted_9 @@ -742,9 +905,10 @@ WHERE typname = 'hstore'; return tuple(rv0), tuple(rv1) + def register_hstore(conn_or_curs, globally=False, unicode=False, - oid=None, array_oid=None): - """Register adapter and typecaster for `!dict`\-\ |hstore| conversions. + oid=None, array_oid=None): + r"""Register adapter and typecaster for `!dict`\-\ |hstore| conversions. :param conn_or_curs: a connection or cursor: the typecaster will be registered only on this object unless *globally* is set to `!True` @@ -822,8 +986,8 @@ class CompositeCaster(object): self.oid = oid self.array_oid = array_oid - self.attnames = [ a[0] for a in attrs ] - self.atttypes = [ a[1] for a in attrs ] + self.attnames = [a[0] for a in attrs] + self.atttypes = [a[1] for a in attrs] self._create_type(name, self.attnames) self.typecaster = _ext.new_type((oid,), name, self.parse) if array_oid: @@ -842,8 +1006,8 @@ class CompositeCaster(object): "expecting %d components for the type %s, %d found instead" % (len(self.atttypes), self.name, len(tokens))) - values = [ curs.cast(oid, token) - for oid, token in zip(self.atttypes, tokens) ] + values = [curs.cast(oid, token) + for oid, token in zip(self.atttypes, tokens)] return self.make(values) @@ -937,11 +1101,12 @@ ORDER BY attnum; type_oid = recs[0][0] array_oid = recs[0][1] - type_attrs = [ (r[2], r[3]) for r in recs ] + type_attrs = [(r[2], r[3]) for r in recs] return self(tname, type_oid, type_attrs, array_oid=array_oid, schema=schema) + def register_composite(name, conn_or_curs, globally=False, factory=None): """Register a typecaster to convert a composite type into a tuple. @@ -964,17 +1129,7 @@ def register_composite(name, conn_or_curs, globally=False, factory=None): _ext.register_type(caster.typecaster, not globally and conn_or_curs or None) if caster.array_typecaster is not None: - _ext.register_type(caster.array_typecaster, not globally and conn_or_curs or None) + _ext.register_type( + caster.array_typecaster, not globally and conn_or_curs or None) return caster - - -# expose the json adaptation stuff into the module -from psycopg2._json import json, Json, register_json -from psycopg2._json import register_default_json, register_default_jsonb - - -# Expose range-related objects -from psycopg2._range import Range, NumericRange -from psycopg2._range import DateRange, DateTimeRange, DateTimeTZRange -from psycopg2._range import register_range, RangeAdapter, RangeCaster diff --git a/lib/pool.py b/lib/pool.py index 8d7c4afb..e57875c8 100644 --- a/lib/pool.py +++ b/lib/pool.py @@ -40,18 +40,18 @@ class AbstractConnectionPool(object): New 'minconn' connections are created immediately calling 'connfunc' with given parameters. The connection pool will support a maximum of - about 'maxconn' connections. + about 'maxconn' connections. """ self.minconn = int(minconn) self.maxconn = int(maxconn) self.closed = False - + self._args = args self._kwargs = kwargs self._pool = [] self._used = {} - self._rused = {} # id(conn) -> key map + self._rused = {} # id(conn) -> key map self._keys = 0 for i in range(self.minconn): @@ -71,12 +71,14 @@ class AbstractConnectionPool(object): """Return a new unique key.""" self._keys += 1 return self._keys - + def _getconn(self, key=None): """Get a free connection and assign it to 'key' if not None.""" - if self.closed: raise PoolError("connection pool is closed") - if key is None: key = self._getkey() - + if self.closed: + raise PoolError("connection pool is closed") + if key is None: + key = self._getkey() + if key in self._used: return self._used[key] @@ -88,11 +90,13 @@ class AbstractConnectionPool(object): if len(self._used) == self.maxconn: raise PoolError("connection pool exhausted") return self._connect(key) - + def _putconn(self, conn, key=None, close=False): """Put away a connection.""" - if self.closed: raise PoolError("connection pool is closed") - if key is None: key = self._rused.get(id(conn)) + if self.closed: + raise PoolError("connection pool is closed") + if key is None: + key = self._rused.get(id(conn)) if not key: raise PoolError("trying to put unkeyed connection") @@ -129,21 +133,22 @@ class AbstractConnectionPool(object): an already closed connection. If you call .closeall() make sure your code can deal with it. """ - if self.closed: raise PoolError("connection pool is closed") + if self.closed: + raise PoolError("connection pool is closed") for conn in self._pool + list(self._used.values()): try: conn.close() except: pass self.closed = True - + class SimpleConnectionPool(AbstractConnectionPool): """A connection pool that can't be shared across different threads.""" getconn = AbstractConnectionPool._getconn putconn = AbstractConnectionPool._putconn - closeall = AbstractConnectionPool._closeall + closeall = AbstractConnectionPool._closeall class ThreadedConnectionPool(AbstractConnectionPool): @@ -182,7 +187,7 @@ class ThreadedConnectionPool(AbstractConnectionPool): class PersistentConnectionPool(AbstractConnectionPool): - """A pool that assigns persistent connections to different threads. + """A pool that assigns persistent connections to different threads. Note that this connection pool generates by itself the required keys using the current thread id. This means that until a thread puts away @@ -204,7 +209,7 @@ class PersistentConnectionPool(AbstractConnectionPool): # we we'll need the thread module, to determine thread ids, so we # import it here and copy it in an instance variable - import thread as _thread # work around for 2to3 bug - see ticket #348 + import thread as _thread # work around for 2to3 bug - see ticket #348 self.__thread = _thread def getconn(self): @@ -221,7 +226,8 @@ class PersistentConnectionPool(AbstractConnectionPool): key = self.__thread.get_ident() self._lock.acquire() try: - if not conn: conn = self._used[key] + if not conn: + conn = self._used[key] self._putconn(conn, key, close) finally: self._lock.release() diff --git a/lib/psycopg1.py b/lib/psycopg1.py index 7a24c5f2..3808aaaf 100644 --- a/lib/psycopg1.py +++ b/lib/psycopg1.py @@ -28,24 +28,26 @@ old code while porting to psycopg 2. Import it as follows:: # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. -import _psycopg as _2psycopg +import psycopg2._psycopg as _2psycopg # noqa from psycopg2.extensions import cursor as _2cursor from psycopg2.extensions import connection as _2connection -from psycopg2 import * +from psycopg2 import * # noqa import psycopg2.extensions as _ext _2connect = connect + def connect(*args, **kwargs): """connect(dsn, ...) -> new psycopg 1.1.x compatible connection object""" kwargs['connection_factory'] = connection conn = _2connect(*args, **kwargs) conn.set_isolation_level(_ext.ISOLATION_LEVEL_READ_COMMITTED) return conn - + + class connection(_2connection): """psycopg 1.1.x connection.""" - + def cursor(self): """cursor() -> new psycopg 1.1.x compatible cursor object""" return _2connection.cursor(self, cursor_factory=cursor) @@ -56,7 +58,7 @@ class connection(_2connection): self.set_isolation_level(_ext.ISOLATION_LEVEL_AUTOCOMMIT) else: self.set_isolation_level(_ext.ISOLATION_LEVEL_READ_COMMITTED) - + class cursor(_2cursor): """psycopg 1.1.x cursor. @@ -71,25 +73,24 @@ class cursor(_2cursor): for i in range(len(self.description)): res[self.description[i][0]] = row[i] return res - + def dictfetchone(self): row = _2cursor.fetchone(self) if row: return self.__build_dict(row) else: return row - + def dictfetchmany(self, size): res = [] rows = _2cursor.fetchmany(self, size) for row in rows: res.append(self.__build_dict(row)) return res - + def dictfetchall(self): res = [] rows = _2cursor.fetchall(self) for row in rows: res.append(self.__build_dict(row)) return res - diff --git a/lib/tz.py b/lib/tz.py index 695a9253..92a16041 100644 --- a/lib/tz.py +++ b/lib/tz.py @@ -2,7 +2,7 @@ This module holds two different tzinfo implementations that can be used as the 'tzinfo' argument to datetime constructors, directly passed to psycopg -functions or used to set the .tzinfo_factory attribute in cursors. +functions or used to set the .tzinfo_factory attribute in cursors. """ # psycopg/tz.py - tzinfo implementation # @@ -31,6 +31,7 @@ import time ZERO = datetime.timedelta(0) + class FixedOffsetTimezone(datetime.tzinfo): """Fixed offset in minutes east from UTC. @@ -52,7 +53,7 @@ class FixedOffsetTimezone(datetime.tzinfo): def __init__(self, offset=None, name=None): if offset is not None: - self._offset = datetime.timedelta(minutes = offset) + self._offset = datetime.timedelta(minutes=offset) if name is not None: self._name = name @@ -85,7 +86,7 @@ class FixedOffsetTimezone(datetime.tzinfo): else: seconds = self._offset.seconds + self._offset.days * 86400 hours, seconds = divmod(seconds, 3600) - minutes = seconds/60 + minutes = seconds / 60 if minutes: return "%+03d:%d" % (hours, minutes) else: @@ -95,13 +96,14 @@ class FixedOffsetTimezone(datetime.tzinfo): return ZERO -STDOFFSET = datetime.timedelta(seconds = -time.timezone) +STDOFFSET = datetime.timedelta(seconds=-time.timezone) if time.daylight: - DSTOFFSET = datetime.timedelta(seconds = -time.altzone) + DSTOFFSET = datetime.timedelta(seconds=-time.altzone) else: DSTOFFSET = STDOFFSET DSTDIFF = DSTOFFSET - STDOFFSET + class LocalTimezone(datetime.tzinfo): """Platform idea of local timezone. diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 597048d2..1727b19a 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -39,11 +39,9 @@ static unsigned char * binary_escape(unsigned char *from, size_t from_length, size_t *to_length, PGconn *conn) { -#if PG_VERSION_NUM >= 80104 if (conn) return PQescapeByteaConn(conn, from, from_length, to_length); else -#endif return PQescapeBytea(from, from_length, to_length); } diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 0571837d..9d04df40 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -451,7 +451,7 @@ psyco_TimestampFromTicks(PyObject *self, PyObject *args) tz); exit: - Py_DECREF(tz); + Py_XDECREF(tz); Py_XDECREF(m); return res; } diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 2e3ab0ae..8c5a8f10 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -36,44 +36,56 @@ static const char *default_encoding = "latin1"; /* qstring_quote - do the quote process on plain and unicode strings */ +const char * +_qstring_get_encoding(qstringObject *self) +{ + /* if the wrapped object is an unicode object we can encode it to match + conn->encoding but if the encoding is not specified we don't know what + to do and we raise an exception */ + if (self->conn) { + return self->conn->codec; + } + else { + return self->encoding ? self->encoding : default_encoding; + } +} + static PyObject * qstring_quote(qstringObject *self) { PyObject *str = NULL; char *s, *buffer = NULL; Py_ssize_t len, qlen; - const char *encoding = default_encoding; + const char *encoding; PyObject *rv = NULL; - /* if the wrapped object is an unicode object we can encode it to match - conn->encoding but if the encoding is not specified we don't know what - to do and we raise an exception */ - if (self->conn) { - encoding = self->conn->codec; - } - + encoding = _qstring_get_encoding(self); Dprintf("qstring_quote: encoding to %s", encoding); - if (PyUnicode_Check(self->wrapped) && encoding) { - str = PyUnicode_AsEncodedString(self->wrapped, encoding, NULL); - Dprintf("qstring_quote: got encoded object at %p", str); - if (str == NULL) goto exit; + if (PyUnicode_Check(self->wrapped)) { + if (encoding) { + str = PyUnicode_AsEncodedString(self->wrapped, encoding, NULL); + Dprintf("qstring_quote: got encoded object at %p", str); + if (str == NULL) goto exit; + } + else { + PyErr_SetString(PyExc_TypeError, + "missing encoding to encode unicode object"); + goto exit; + } } -#if PY_MAJOR_VERSION < 3 - /* if the wrapped object is a simple string, we don't know how to + /* if the wrapped object is a binary string, we don't know how to (re)encode it, so we pass it as-is */ - else if (PyString_Check(self->wrapped)) { + else if (Bytes_Check(self->wrapped)) { str = self->wrapped; /* INCREF to make it ref-wise identical to unicode one */ Py_INCREF(str); } -#endif /* if the wrapped object is not a string, this is an error */ else { - PyErr_SetString(PyExc_TypeError, - "can't quote non-string object (or missing encoding)"); + PyErr_SetString(PyExc_TypeError, "can't quote non-string object"); goto exit; } @@ -150,15 +162,34 @@ qstring_conform(qstringObject *self, PyObject *args) static PyObject * qstring_get_encoding(qstringObject *self) { - const char *encoding = default_encoding; - - if (self->conn) { - encoding = self->conn->codec; - } - + const char *encoding; + encoding = _qstring_get_encoding(self); return Text_FromUTF8(encoding); } +static int +qstring_set_encoding(qstringObject *self, PyObject *pyenc) +{ + int rv = -1; + const char *tmp; + char *cenc; + + /* get a C copy of the encoding (which may come from unicode) */ + Py_INCREF(pyenc); + if (!(pyenc = psycopg_ensure_bytes(pyenc))) { goto exit; } + if (!(tmp = Bytes_AsString(pyenc))) { goto exit; } + if (0 > psycopg_strdup(&cenc, tmp, 0)) { goto exit; } + + Dprintf("qstring_set_encoding: encoding set to %s", cenc); + PyMem_Free((void *)self->encoding); + self->encoding = cenc; + rv = 0; + +exit: + Py_XDECREF(pyenc); + return rv; +} + /** the QuotedString object **/ /* object member list */ @@ -183,7 +214,7 @@ static PyMethodDef qstringObject_methods[] = { static PyGetSetDef qstringObject_getsets[] = { { "encoding", (getter)qstring_get_encoding, - (setter)NULL, + (setter)qstring_set_encoding, "current encoding of the adapter" }, {NULL} }; @@ -216,6 +247,7 @@ qstring_dealloc(PyObject* obj) Py_CLEAR(self->wrapped); Py_CLEAR(self->buffer); Py_CLEAR(self->conn); + PyMem_Free((void *)self->encoding); Dprintf("qstring_dealloc: deleted qstring object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, diff --git a/psycopg/adapter_qstring.h b/psycopg/adapter_qstring.h index b7b086f3..8abdc5f2 100644 --- a/psycopg/adapter_qstring.h +++ b/psycopg/adapter_qstring.h @@ -39,6 +39,9 @@ typedef struct { PyObject *buffer; connectionObject *conn; + + const char *encoding; + } qstringObject; #ifdef __cplusplus diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 43d0fdae..a34e5ef9 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -494,6 +494,25 @@ conn_setup_cancel(connectionObject *self, PGconn *pgconn) return 0; } +/* Return 1 if the "replication" keyword is set in the DSN, 0 otherwise */ +static int +dsn_has_replication(char *pgdsn) +{ + int ret = 0; + PQconninfoOption *connopts, *ptr; + + connopts = PQconninfoParse(pgdsn, NULL); + + for(ptr = connopts; ptr->keyword != NULL; ptr++) { + if(strcmp(ptr->keyword, "replication") == 0 && ptr->val != NULL) + ret = 1; + } + + PQconninfoFree(connopts); + + return ret; +} + /* Return 1 if the server datestyle allows us to work without problems, 0 if it needs to be set to something better, e.g. ISO. */ @@ -522,28 +541,29 @@ conn_setup(connectionObject *self, PGconn *pgconn) { PGresult *pgres = NULL; char *error = NULL; + int rv = -1; self->equote = conn_get_standard_conforming_strings(pgconn); self->server_version = conn_get_server_version(pgconn); self->protocol = conn_get_protocol_version(self->pgconn); if (3 != self->protocol) { PyErr_SetString(InterfaceError, "only protocol 3 supported"); - return -1; + goto exit; } if (0 > conn_read_encoding(self, pgconn)) { - return -1; + goto exit; } if (0 > conn_setup_cancel(self, pgconn)) { - return -1; + goto exit; } Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); Py_BLOCK_THREADS; - if (!conn_is_datestyle_ok(self->pgconn)) { + if (!dsn_has_replication(self->dsn) && !conn_is_datestyle_ok(self->pgconn)) { int res; Py_UNBLOCK_THREADS; res = pq_set_guc_locked(self, "datestyle", "ISO", @@ -551,18 +571,23 @@ conn_setup(connectionObject *self, PGconn *pgconn) Py_BLOCK_THREADS; if (res < 0) { pq_complete_error(self, &pgres, &error); - return -1; + goto unlock; } } /* for reset */ self->autocommit = 0; + /* success */ + rv = 0; + +unlock: Py_UNBLOCK_THREADS; pthread_mutex_unlock(&self->lock); Py_END_ALLOW_THREADS; - return 0; +exit: + return rv; } /* conn_connect - execute a connection to the database */ @@ -859,8 +884,11 @@ _conn_poll_setup_async(connectionObject *self) self->autocommit = 1; /* If the datestyle is ISO or anything else good, - * we can skip the CONN_STATUS_DATESTYLE step. */ - if (!conn_is_datestyle_ok(self->pgconn)) { + * we can skip the CONN_STATUS_DATESTYLE step. + * Note that we cannot change the datestyle on a replication + * connection. + */ + if (!dsn_has_replication(self->dsn) && !conn_is_datestyle_ok(self->pgconn)) { Dprintf("conn_poll: status -> CONN_STATUS_DATESTYLE"); self->status = CONN_STATUS_DATESTYLE; if (0 == pq_send_query(self, psyco_datestyle)) { diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 2c1dddf2..485a92b7 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -733,6 +733,37 @@ psyco_conn_get_parameter_status(connectionObject *self, PyObject *args) return conn_text_from_chars(self, val); } +/* get_dsn_parameters method - Get connection parameters */ + +#define psyco_conn_get_dsn_parameters_doc \ +"get_dsn_parameters() -- Get effective connection parameters.\n\n" + +static PyObject * +psyco_conn_get_dsn_parameters(connectionObject *self) +{ +#if PG_VERSION_NUM >= 90300 + PyObject *res = NULL; + PQconninfoOption *options = NULL; + + EXC_IF_CONN_CLOSED(self); + + if (!(options = PQconninfo(self->pgconn))) { + PyErr_NoMemory(); + goto exit; + } + + res = psycopg_dict_from_conninfo_options(options, /* include_password = */ 0); + +exit: + PQconninfoFree(options); + + return res; +#else + PyErr_SetString(NotSupportedError, "PQconninfo not available in libpq < 9.3"); + return NULL; +#endif +} + /* lobject method - allocate a new lobject */ @@ -977,6 +1008,8 @@ static struct PyMethodDef connectionObject_methods[] = { METH_NOARGS, psyco_conn_get_transaction_status_doc}, {"get_parameter_status", (PyCFunction)psyco_conn_get_parameter_status, METH_VARARGS, psyco_conn_get_parameter_status_doc}, + {"get_dsn_parameters", (PyCFunction)psyco_conn_get_dsn_parameters, + METH_NOARGS, psyco_conn_get_dsn_parameters_doc}, {"get_backend_pid", (PyCFunction)psyco_conn_get_backend_pid, METH_NOARGS, psyco_conn_get_backend_pid_doc}, {"lobject", (PyCFunction)psyco_conn_lobject, @@ -1171,7 +1204,7 @@ connection_repr(connectionObject *self) { return PyString_FromFormat( "", - self, self->dsn, self->closed); + self, (self->dsn ? self->dsn : ""), self->closed); } static int diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index e205ba2a..c47af43b 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -335,7 +335,7 @@ _psyco_curs_merge_query_args(cursorObject *self, PyErr_Fetch(&err, &arg, &trace); if (err && PyErr_GivenExceptionMatches(err, PyExc_TypeError)) { - Dprintf("psyco_curs_execute: TypeError exception catched"); + Dprintf("psyco_curs_execute: TypeError exception caught"); PyErr_NormalizeException(&err, &arg, &trace); if (PyObject_HasAttrString(arg, "args")) { diff --git a/psycopg/libpq_support.c b/psycopg/libpq_support.c new file mode 100644 index 00000000..6c0b5f8e --- /dev/null +++ b/psycopg/libpq_support.c @@ -0,0 +1,104 @@ +/* libpq_support.c - functions not provided by libpq, but which are + * required for advanced communication with the server, such as + * streaming replication + * + * Copyright (C) 2003-2015 Federico Di Gregorio + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/libpq_support.h" + +/* htonl(), ntohl() */ +#ifdef _WIN32 +#include +/* gettimeofday() */ +#include "psycopg/win32_support.h" +#else +#include +#endif + +/* support routines taken from pg_basebackup/streamutil.c */ + +/* + * Frontend version of GetCurrentTimestamp(), since we are not linked with + * backend code. The protocol always uses integer timestamps, regardless of + * server setting. + */ +int64_t +feGetCurrentTimestamp(void) +{ + int64_t result; + struct timeval tp; + + gettimeofday(&tp, NULL); + + result = (int64_t) tp.tv_sec - + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY); + + result = (result * USECS_PER_SEC) + tp.tv_usec; + + return result; +} + +/* + * Converts an int64 to network byte order. + */ +void +fe_sendint64(int64_t i, char *buf) +{ + uint32_t n32; + + /* High order half first, since we're doing MSB-first */ + n32 = (uint32_t) (i >> 32); + n32 = htonl(n32); + memcpy(&buf[0], &n32, 4); + + /* Now the low order half */ + n32 = (uint32_t) i; + n32 = htonl(n32); + memcpy(&buf[4], &n32, 4); +} + +/* + * Converts an int64 from network byte order to native format. + */ +int64_t +fe_recvint64(char *buf) +{ + int64_t result; + uint32_t h32; + uint32_t l32; + + memcpy(&h32, buf, 4); + memcpy(&l32, buf + 4, 4); + h32 = ntohl(h32); + l32 = ntohl(l32); + + result = h32; + result <<= 32; + result |= l32; + + return result; +} diff --git a/psycopg/libpq_support.h b/psycopg/libpq_support.h new file mode 100644 index 00000000..c8f10665 --- /dev/null +++ b/psycopg/libpq_support.h @@ -0,0 +1,48 @@ +/* libpq_support.h - definitions for libpq_support.c + * + * Copyright (C) 2003-2015 Federico Di Gregorio + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ +#ifndef PSYCOPG_LIBPQ_SUPPORT_H +#define PSYCOPG_LIBPQ_SUPPORT_H 1 + +#include "psycopg/config.h" + +/* type and constant definitions from internal postgres include */ +typedef unsigned PG_INT64_TYPE XLogRecPtr; + +/* have to use lowercase %x, as PyString_FromFormat can't do %X */ +#define XLOGFMTSTR "%x/%x" +#define XLOGFMTARGS(x) ((uint32_t)((x) >> 32)), ((uint32_t)((x) & 0xFFFFFFFF)) + +/* Julian-date equivalents of Day 0 in Unix and Postgres reckoning */ +#define UNIX_EPOCH_JDATE 2440588 /* == date2j(1970, 1, 1) */ +#define POSTGRES_EPOCH_JDATE 2451545 /* == date2j(2000, 1, 1) */ + +#define SECS_PER_DAY 86400 +#define USECS_PER_SEC 1000000LL + +HIDDEN int64_t feGetCurrentTimestamp(void); +HIDDEN void fe_sendint64(int64_t i, char *buf); +HIDDEN int64_t fe_recvint64(char *buf); + +#endif /* !defined(PSYCOPG_LIBPQ_SUPPORT_H) */ diff --git a/psycopg/lobject.h b/psycopg/lobject.h index b9c8c3d8..73cf6192 100644 --- a/psycopg/lobject.h +++ b/psycopg/lobject.h @@ -60,8 +60,8 @@ RAISES_NEG HIDDEN int lobject_export(lobjectObject *self, const char *filename); RAISES_NEG HIDDEN Py_ssize_t lobject_read(lobjectObject *self, char *buf, size_t len); RAISES_NEG HIDDEN Py_ssize_t lobject_write(lobjectObject *self, const char *buf, size_t len); -RAISES_NEG HIDDEN long lobject_seek(lobjectObject *self, long pos, int whence); -RAISES_NEG HIDDEN long lobject_tell(lobjectObject *self); +RAISES_NEG HIDDEN Py_ssize_t lobject_seek(lobjectObject *self, Py_ssize_t pos, int whence); +RAISES_NEG HIDDEN Py_ssize_t lobject_tell(lobjectObject *self); RAISES_NEG HIDDEN int lobject_truncate(lobjectObject *self, size_t len); RAISES_NEG HIDDEN int lobject_close(lobjectObject *self); diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 8788c100..b954a76b 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -376,12 +376,12 @@ lobject_read(lobjectObject *self, char *buf, size_t len) /* lobject_seek - move the current position in the lo */ -RAISES_NEG long -lobject_seek(lobjectObject *self, long pos, int whence) +RAISES_NEG Py_ssize_t +lobject_seek(lobjectObject *self, Py_ssize_t pos, int whence) { PGresult *pgres = NULL; char *error = NULL; - long where; + Py_ssize_t where; Dprintf("lobject_seek: fd = %d, pos = %ld, whence = %d", self->fd, pos, whence); @@ -391,12 +391,12 @@ lobject_seek(lobjectObject *self, long pos, int whence) #ifdef HAVE_LO64 if (self->conn->server_version < 90300) { - where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); + where = (Py_ssize_t)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); } else { - where = lo_lseek64(self->conn->pgconn, self->fd, pos, whence); + where = (Py_ssize_t)lo_lseek64(self->conn->pgconn, self->fd, pos, whence); } #else - where = (long)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); + where = (Py_ssize_t)lo_lseek(self->conn->pgconn, self->fd, (int)pos, whence); #endif Dprintf("lobject_seek: where = %ld", where); if (where < 0) @@ -412,12 +412,12 @@ lobject_seek(lobjectObject *self, long pos, int whence) /* lobject_tell - tell the current position in the lo */ -RAISES_NEG long +RAISES_NEG Py_ssize_t lobject_tell(lobjectObject *self) { PGresult *pgres = NULL; char *error = NULL; - long where; + Py_ssize_t where; Dprintf("lobject_tell: fd = %d", self->fd); @@ -426,12 +426,12 @@ lobject_tell(lobjectObject *self) #ifdef HAVE_LO64 if (self->conn->server_version < 90300) { - where = (long)lo_tell(self->conn->pgconn, self->fd); + where = (Py_ssize_t)lo_tell(self->conn->pgconn, self->fd); } else { - where = lo_tell64(self->conn->pgconn, self->fd); + where = (Py_ssize_t)lo_tell64(self->conn->pgconn, self->fd); } #else - where = (long)lo_tell(self->conn->pgconn, self->fd); + where = (Py_ssize_t)lo_tell(self->conn->pgconn, self->fd); #endif Dprintf("lobject_tell: where = %ld", where); if (where < 0) @@ -474,8 +474,6 @@ lobject_export(lobjectObject *self, const char *filename) return retvalue; } -#if PG_VERSION_NUM >= 80300 - RAISES_NEG int lobject_truncate(lobjectObject *self, size_t len) { @@ -510,5 +508,3 @@ lobject_truncate(lobjectObject *self, size_t len) return retvalue; } - -#endif /* PG_VERSION_NUM >= 80300 */ diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index a43325d4..ddda0daf 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -105,7 +105,7 @@ psyco_lobj_write(lobjectObject *self, PyObject *args) goto exit; } - rv = PyInt_FromLong((long)res); + rv = PyInt_FromSsize_t((Py_ssize_t)res); exit: Py_XDECREF(data); @@ -121,7 +121,7 @@ static PyObject * psyco_lobj_read(lobjectObject *self, PyObject *args) { PyObject *res; - long where, end; + Py_ssize_t where, end; Py_ssize_t size = -1; char *buffer; @@ -165,10 +165,10 @@ psyco_lobj_read(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_seek(lobjectObject *self, PyObject *args) { - long offset, pos=0; + Py_ssize_t offset, pos=0; int whence=0; - if (!PyArg_ParseTuple(args, "l|i", &offset, &whence)) + if (!PyArg_ParseTuple(args, "n|i", &offset, &whence)) return NULL; EXC_IF_LOBJ_CLOSED(self); @@ -187,8 +187,8 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) #else if (offset < INT_MIN || offset > INT_MAX) { PyErr_Format(InterfaceError, - "offset out of range (%ld): this psycopg version was not built " - "with lobject 64 API support", + "offset out of range (" FORMAT_CODE_PY_SSIZE_T "): " + "this psycopg version was not built with lobject 64 API support", offset); return NULL; } @@ -197,7 +197,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) if ((pos = lobject_seek(self, offset, whence)) < 0) return NULL; - return PyLong_FromLong(pos); + return PyInt_FromSsize_t(pos); } /* tell method - tell current position in the lobject */ @@ -208,7 +208,7 @@ psyco_lobj_seek(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_tell(lobjectObject *self, PyObject *args) { - long pos; + Py_ssize_t pos; EXC_IF_LOBJ_CLOSED(self); EXC_IF_LOBJ_LEVEL0(self); @@ -217,7 +217,7 @@ psyco_lobj_tell(lobjectObject *self, PyObject *args) if ((pos = lobject_tell(self)) < 0) return NULL; - return PyLong_FromLong(pos); + return PyInt_FromSsize_t(pos); } /* unlink method - unlink (destroy) the lobject */ @@ -266,17 +266,15 @@ psyco_lobj_get_closed(lobjectObject *self, void *closure) return closed; } -#if PG_VERSION_NUM >= 80300 - #define psyco_lobj_truncate_doc \ "truncate(len=0) -- Truncate large object to given size." static PyObject * psyco_lobj_truncate(lobjectObject *self, PyObject *args) { - long len = 0; + Py_ssize_t len = 0; - if (!PyArg_ParseTuple(args, "|l", &len)) + if (!PyArg_ParseTuple(args, "|n", &len)) return NULL; EXC_IF_LOBJ_CLOSED(self); @@ -286,16 +284,16 @@ psyco_lobj_truncate(lobjectObject *self, PyObject *args) #ifdef HAVE_LO64 if (len > INT_MAX && self->conn->server_version < 90300) { PyErr_Format(NotSupportedError, - "len out of range (%ld): server version %d " - "does not support the lobject 64 API", + "len out of range (" FORMAT_CODE_PY_SSIZE_T "): " + "server version %d does not support the lobject 64 API", len, self->conn->server_version); return NULL; } #else if (len > INT_MAX) { PyErr_Format(InterfaceError, - "len out of range (%ld): this psycopg version was not built " - "with lobject 64 API support", + "len out of range (" FORMAT_CODE_PY_SSIZE_T "): " + "this psycopg version was not built with lobject 64 API support", len); return NULL; } @@ -327,10 +325,8 @@ static struct PyMethodDef lobjectObject_methods[] = { METH_NOARGS, psyco_lobj_unlink_doc}, {"export",(PyCFunction)psyco_lobj_export, METH_VARARGS, psyco_lobj_export_doc}, -#if PG_VERSION_NUM >= 80300 {"truncate",(PyCFunction)psyco_lobj_truncate, METH_VARARGS, psyco_lobj_truncate_doc}, -#endif /* PG_VERSION_NUM >= 80300 */ {NULL} }; @@ -475,6 +471,3 @@ PyTypeObject lobjectType = { 0, /*tp_alloc*/ lobject_new, /*tp_new*/ }; - -#endif - diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index b643512d..d02cb708 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -35,13 +35,22 @@ #include "psycopg/pqpath.h" #include "psycopg/connection.h" #include "psycopg/cursor.h" +#include "psycopg/replication_cursor.h" +#include "psycopg/replication_message.h" #include "psycopg/green.h" #include "psycopg/typecast.h" #include "psycopg/pgtypes.h" #include "psycopg/error.h" -#include +#include "psycopg/libpq_support.h" +#include "libpq-fe.h" +#ifdef _WIN32 +/* select() */ +#include +/* gettimeofday() */ +#include "win32_support.h" +#endif extern HIDDEN PyObject *psyco_DescriptionType; @@ -161,11 +170,11 @@ pq_raise(connectionObject *conn, cursorObject *curs, PGresult **pgres) if (conn == NULL) { PyErr_SetString(DatabaseError, - "psycopg went psycotic and raised a null error"); + "psycopg went psychotic and raised a null error"); return; } - /* if the connection has somehow beed broken, we mark the connection + /* if the connection has somehow been broken, we mark the connection object as closed but requiring cleanup */ if (conn->pgconn != NULL && PQstatus(conn->pgconn) == CONNECTION_BAD) conn->closed = 2; @@ -907,7 +916,7 @@ pq_execute(cursorObject *curs, const char *query, int async, int no_result, int PyErr_SetString(OperationalError, PQerrorMessage(curs->conn->pgconn)); return -1; } - Dprintf("curs_execute: pg connection at %p OK", curs->conn->pgconn); + Dprintf("pq_execute: pg connection at %p OK", curs->conn->pgconn); Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(curs->conn->lock)); @@ -932,7 +941,7 @@ pq_execute(cursorObject *curs, const char *query, int async, int no_result, int Py_UNBLOCK_THREADS; } - /* dont let pgres = NULL go to pq_fetch() */ + /* don't let pgres = NULL go to pq_fetch() */ if (curs->pgres == NULL) { pthread_mutex_unlock(&(curs->conn->lock)); Py_BLOCK_THREADS; @@ -1056,6 +1065,13 @@ pq_get_last_result(connectionObject *conn) PQclear(result); } result = res; + + /* After entering copy both mode, libpq will make a phony + * PGresult for us every time we query for it, so we need to + * break out of this endless loop. */ + if (PQresultStatus(result) == PGRES_COPY_BOTH) { + break; + } } return result; @@ -1393,7 +1409,11 @@ _pq_copy_in_v3(cursorObject *curs) Py_DECREF(str); } } - PyErr_Restore(t, ex, tb); + /* Clear the Py exception: it will be re-raised from the libpq */ + Py_XDECREF(t); + Py_XDECREF(ex); + Py_XDECREF(tb); + PyErr_Clear(); } res = PQputCopyEnd(curs->conn->pgconn, buf); } @@ -1516,6 +1536,281 @@ exit: return ret; } +/* Tries to read the next message from the replication stream, without + blocking, in both sync and async connection modes. If no message + is ready in the CopyData buffer, tries to read from the server, + again without blocking. If that doesn't help, returns Py_None. + The caller is then supposed to block on the socket(s) and call this + function again. + + Any keepalive messages from the server are silently consumed and + are never returned to the caller. + */ +int +pq_read_replication_message(replicationCursorObject *repl, replicationMessageObject **msg) +{ + cursorObject *curs = &repl->cur; + connectionObject *conn = curs->conn; + PGconn *pgconn = conn->pgconn; + char *buffer = NULL; + int len, data_size, consumed, hdr, reply; + XLogRecPtr data_start, wal_end; + int64_t send_time; + PyObject *str = NULL, *result = NULL; + int ret = -1; + + Dprintf("pq_read_replication_message"); + + *msg = NULL; + consumed = 0; + +retry: + len = PQgetCopyData(pgconn, &buffer, 1 /* async */); + + if (len == 0) { + /* If we've tried reading some data, but there was none, bail out. */ + if (consumed) { + ret = 0; + goto exit; + } + /* We should only try reading more data when there is nothing + available at the moment. Otherwise, with a really highly loaded + server we might be reading a number of messages for every single + one we process, thus overgrowing the internal buffer until the + client system runs out of memory. */ + if (!PQconsumeInput(pgconn)) { + pq_raise(conn, curs, NULL); + goto exit; + } + /* But PQconsumeInput() doesn't tell us if it has actually read + anything into the internal buffer and there is no (supported) way + to ask libpq about this directly. The way we check is setting the + flag and re-trying PQgetCopyData(): if that returns 0 again, + there's no more data available in the buffer, so we return None. */ + consumed = 1; + goto retry; + } + + if (len == -2) { + /* serious error */ + pq_raise(conn, curs, NULL); + goto exit; + } + if (len == -1) { + /* EOF */ + curs->pgres = PQgetResult(pgconn); + + if (curs->pgres && PQresultStatus(curs->pgres) == PGRES_FATAL_ERROR) { + pq_raise(conn, curs, NULL); + goto exit; + } + + CLEARPGRES(curs->pgres); + ret = 0; + goto exit; + } + + /* It also makes sense to set this flag here to make us return early in + case of retry due to keepalive message. Any pending data on the socket + will trigger read condition in select() in the calling code anyway. */ + consumed = 1; + + /* ok, we did really read something: update the io timestamp */ + gettimeofday(&repl->last_io, NULL); + + Dprintf("pq_read_replication_message: msg=%c, len=%d", buffer[0], len); + if (buffer[0] == 'w') { + /* XLogData: msgtype(1), dataStart(8), walEnd(8), sendTime(8) */ + hdr = 1 + 8 + 8 + 8; + if (len < hdr + 1) { + psyco_set_error(OperationalError, curs, "data message header too small"); + goto exit; + } + + data_size = len - hdr; + data_start = fe_recvint64(buffer + 1); + wal_end = fe_recvint64(buffer + 1 + 8); + send_time = fe_recvint64(buffer + 1 + 8 + 8); + + Dprintf("pq_read_replication_message: data_start="XLOGFMTSTR", wal_end="XLOGFMTSTR, + XLOGFMTARGS(data_start), XLOGFMTARGS(wal_end)); + + Dprintf("pq_read_replication_message: >>%.*s<<", data_size, buffer + hdr); + + if (repl->decode) { + str = PyUnicode_Decode(buffer + hdr, data_size, conn->codec, NULL); + } else { + str = Bytes_FromStringAndSize(buffer + hdr, data_size); + } + if (!str) { goto exit; } + + result = PyObject_CallFunctionObjArgs((PyObject *)&replicationMessageType, + curs, str, NULL); + Py_DECREF(str); + if (!result) { goto exit; } + + *msg = (replicationMessageObject *)result; + (*msg)->data_size = data_size; + (*msg)->data_start = data_start; + (*msg)->wal_end = wal_end; + (*msg)->send_time = send_time; + } + else if (buffer[0] == 'k') { + /* Primary keepalive message: msgtype(1), walEnd(8), sendTime(8), reply(1) */ + hdr = 1 + 8 + 8; + if (len < hdr + 1) { + psyco_set_error(OperationalError, curs, "keepalive message header too small"); + goto exit; + } + + reply = buffer[hdr]; + if (reply && pq_send_replication_feedback(repl, 0) < 0) { + goto exit; + } + + PQfreemem(buffer); + buffer = NULL; + goto retry; + } + else { + psyco_set_error(OperationalError, curs, "unrecognized replication message type"); + goto exit; + } + + ret = 0; + +exit: + if (buffer) { + PQfreemem(buffer); + } + + return ret; +} + +int +pq_send_replication_feedback(replicationCursorObject *repl, int reply_requested) +{ + cursorObject *curs = &repl->cur; + connectionObject *conn = curs->conn; + PGconn *pgconn = conn->pgconn; + char replybuf[1 + 8 + 8 + 8 + 8 + 1]; + int len = 0; + + Dprintf("pq_send_replication_feedback: write="XLOGFMTSTR", flush="XLOGFMTSTR", apply="XLOGFMTSTR, + XLOGFMTARGS(repl->write_lsn), + XLOGFMTARGS(repl->flush_lsn), + XLOGFMTARGS(repl->apply_lsn)); + + replybuf[len] = 'r'; len += 1; + fe_sendint64(repl->write_lsn, &replybuf[len]); len += 8; + fe_sendint64(repl->flush_lsn, &replybuf[len]); len += 8; + fe_sendint64(repl->apply_lsn, &replybuf[len]); len += 8; + fe_sendint64(feGetCurrentTimestamp(), &replybuf[len]); len += 8; + replybuf[len] = reply_requested ? 1 : 0; len += 1; + + if (PQputCopyData(pgconn, replybuf, len) <= 0 || PQflush(pgconn) != 0) { + pq_raise(conn, curs, NULL); + return -1; + } + gettimeofday(&repl->last_io, NULL); + + return 0; +} + +/* Calls pq_read_replication_message in an endless loop, until + stop_replication is called or a fatal error occurs. The messages + are passed to the consumer object. + + When no message is available, blocks on the connection socket, but + manages to send keepalive messages to the server as needed. +*/ +int +pq_copy_both(replicationCursorObject *repl, PyObject *consume, double keepalive_interval) +{ + cursorObject *curs = &repl->cur; + connectionObject *conn = curs->conn; + PGconn *pgconn = conn->pgconn; + replicationMessageObject *msg = NULL; + PyObject *tmp = NULL; + int fd, sel, ret = -1; + fd_set fds; + struct timeval keep_intr, curr_time, ping_time, timeout; + + if (!PyCallable_Check(consume)) { + Dprintf("pq_copy_both: expected callable consume object"); + goto exit; + } + + CLEARPGRES(curs->pgres); + + keep_intr.tv_sec = (int)keepalive_interval; + keep_intr.tv_usec = (keepalive_interval - keep_intr.tv_sec)*1.0e6; + + while (1) { + if (pq_read_replication_message(repl, &msg) < 0) { + goto exit; + } + else if (msg == NULL) { + fd = PQsocket(pgconn); + if (fd < 0) { + pq_raise(conn, curs, NULL); + goto exit; + } + + FD_ZERO(&fds); + FD_SET(fd, &fds); + + /* how long can we wait before we need to send a keepalive? */ + gettimeofday(&curr_time, NULL); + + timeradd(&repl->last_io, &keep_intr, &ping_time); + timersub(&ping_time, &curr_time, &timeout); + + if (timeout.tv_sec >= 0) { + Py_BEGIN_ALLOW_THREADS; + sel = select(fd + 1, &fds, NULL, NULL, &timeout); + Py_END_ALLOW_THREADS; + } + else { + sel = 0; /* we're past target time, pretend select() timed out */ + } + + if (sel < 0) { + if (errno != EINTR) { + PyErr_SetFromErrno(PyExc_OSError); + goto exit; + } + if (PyErr_CheckSignals()) { + goto exit; + } + continue; + } + + if (sel == 0) { + if (pq_send_replication_feedback(repl, 0) < 0) { + goto exit; + } + } + continue; + } + else { + tmp = PyObject_CallFunctionObjArgs(consume, msg, NULL); + Py_DECREF(msg); + + if (tmp == NULL) { + Dprintf("pq_copy_both: consume returned NULL"); + goto exit; + } + Py_DECREF(tmp); + } + } + + ret = 1; + +exit: + return ret; +} + int pq_fetch(cursorObject *curs, int no_result) { @@ -1575,6 +1870,17 @@ pq_fetch(cursorObject *curs, int no_result) CLEARPGRES(curs->pgres); break; + case PGRES_COPY_BOTH: + Dprintf("pq_fetch: data from a streaming replication slot (no tuples)"); + curs->rowcount = -1; + ex = 0; + /* Nothing to do here: pq_copy_both will be called separately. + + Also don't clear the result status: it's checked in + consume_stream. */ + /*CLEARPGRES(curs->pgres);*/ + break; + case PGRES_TUPLES_OK: if (!no_result) { Dprintf("pq_fetch: got tuples"); @@ -1607,7 +1913,7 @@ pq_fetch(cursorObject *curs, int no_result) break; default: - /* PGRES_COPY_BOTH, PGRES_SINGLE_TUPLE, future statuses */ + /* PGRES_SINGLE_TUPLE, future statuses */ Dprintf("pq_fetch: got unsupported result: status = %d pgconn = %p", pgstatus, curs->conn); PyErr_Format(NotSupportedError, diff --git a/psycopg/pqpath.h b/psycopg/pqpath.h index bd3293f8..5cf22309 100644 --- a/psycopg/pqpath.h +++ b/psycopg/pqpath.h @@ -28,6 +28,8 @@ #include "psycopg/cursor.h" #include "psycopg/connection.h" +#include "psycopg/replication_cursor.h" +#include "psycopg/replication_message.h" /* macro to clean the pg result */ #define CLEARPGRES(pgres) do { PQclear(pgres); pgres = NULL; } while (0) @@ -72,4 +74,11 @@ HIDDEN int pq_execute_command_locked(connectionObject *conn, RAISES HIDDEN void pq_complete_error(connectionObject *conn, PGresult **pgres, char **error); +/* replication protocol support */ +HIDDEN int pq_copy_both(replicationCursorObject *repl, PyObject *consumer, + double keepalive_interval); +HIDDEN int pq_read_replication_message(replicationCursorObject *repl, + replicationMessageObject **msg); +HIDDEN int pq_send_replication_feedback(replicationCursorObject *repl, int reply_requested); + #endif /* !defined(PSYCOPG_PQPATH_H) */ diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index 1a16e8af..fb46a1d6 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -26,6 +26,10 @@ #ifndef PSYCOPG_H #define PSYCOPG_H 1 +#if PG_VERSION_NUM < 90100 +#error "Psycopg requires PostgreSQL client library (libpq) >= 9.1" +#endif + #define PY_SSIZE_T_CLEAN #include #include @@ -117,6 +121,7 @@ HIDDEN PyObject *psyco_GetDecimalType(void); /* forward declarations */ typedef struct cursorObject cursorObject; typedef struct connectionObject connectionObject; +typedef struct replicationMessageObject replicationMessageObject; /* some utility functions */ RAISES HIDDEN PyObject *psyco_set_error(PyObject *exc, cursorObject *curs, const char *msg); @@ -132,6 +137,9 @@ STEALS(1) HIDDEN PyObject * psycopg_ensure_bytes(PyObject *obj); STEALS(1) HIDDEN PyObject * psycopg_ensure_text(PyObject *obj); +HIDDEN PyObject *psycopg_dict_from_conninfo_options(PQconninfoOption *options, + int include_password); + /* Exceptions docstrings */ #define Error_doc \ "Base class for error exceptions." diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 0ecfcc9c..a7f7aef3 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -28,6 +28,9 @@ #include "psycopg/connection.h" #include "psycopg/cursor.h" +#include "psycopg/replication_connection.h" +#include "psycopg/replication_cursor.h" +#include "psycopg/replication_message.h" #include "psycopg/green.h" #include "psycopg/lobject.h" #include "psycopg/notify.h" @@ -111,14 +114,16 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) return conn; } -#define psyco_parse_dsn_doc "parse_dsn(dsn) -> dict" + +#define psyco_parse_dsn_doc \ +"parse_dsn(dsn) -> dict -- parse a connection string into parameters" static PyObject * psyco_parse_dsn(PyObject *self, PyObject *args, PyObject *kwargs) { char *err = NULL; - PQconninfoOption *options = NULL, *o; - PyObject *dict = NULL, *res = NULL, *dsn; + PQconninfoOption *options = NULL; + PyObject *res = NULL, *dsn; static char *kwlist[] = {"dsn", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O", kwlist, &dsn)) { @@ -131,7 +136,7 @@ psyco_parse_dsn(PyObject *self, PyObject *args, PyObject *kwargs) options = PQconninfoParse(Bytes_AS_STRING(dsn), &err); if (options == NULL) { if (err != NULL) { - PyErr_Format(ProgrammingError, "error parsing the dsn: %s", err); + PyErr_Format(ProgrammingError, "invalid dsn: %s", err); PQfreemem(err); } else { PyErr_SetString(OperationalError, "PQconninfoParse() failed"); @@ -139,26 +144,10 @@ psyco_parse_dsn(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } - if (!(dict = PyDict_New())) { goto exit; } - for (o = options; o->keyword != NULL; o++) { - if (o->val != NULL) { - PyObject *value; - if (!(value = Text_FromUTF8(o->val))) { goto exit; } - if (PyDict_SetItemString(dict, o->keyword, value) != 0) { - Py_DECREF(value); - goto exit; - } - Py_DECREF(value); - } - } - - /* success */ - res = dict; - dict = NULL; + res = psycopg_dict_from_conninfo_options(options, /* include_password = */ 1); exit: PQconninfoFree(options); /* safe on null */ - Py_XDECREF(dict); Py_XDECREF(dsn); return res; @@ -289,9 +278,7 @@ psyco_libcrypto_threads_init(void) if ((m = PyImport_ImportModule("ssl"))) { /* disable libcrypto setup in libpq, so it won't stomp on the callbacks that have already been set up */ -#if PG_VERSION_NUM >= 80400 PQinitOpenSSL(1, 0); -#endif Py_DECREF(m); } else { @@ -909,6 +896,15 @@ INIT_MODULE(_psycopg)(void) Py_TYPE(&cursorType) = &PyType_Type; if (PyType_Ready(&cursorType) == -1) goto exit; + Py_TYPE(&replicationConnectionType) = &PyType_Type; + if (PyType_Ready(&replicationConnectionType) == -1) goto exit; + + Py_TYPE(&replicationCursorType) = &PyType_Type; + if (PyType_Ready(&replicationCursorType) == -1) goto exit; + + Py_TYPE(&replicationMessageType) = &PyType_Type; + if (PyType_Ready(&replicationMessageType) == -1) goto exit; + Py_TYPE(&typecastType) = &PyType_Type; if (PyType_Ready(&typecastType) == -1) goto exit; @@ -989,6 +985,8 @@ INIT_MODULE(_psycopg)(void) /* Initialize the PyDateTimeAPI everywhere is used */ PyDateTime_IMPORT; if (psyco_adapter_datetime_init()) { goto exit; } + if (psyco_repl_curs_datetime_init()) { goto exit; } + if (psyco_replmsg_datetime_init()) { goto exit; } Py_TYPE(&pydatetimeType) = &PyType_Type; if (PyType_Ready(&pydatetimeType) == -1) goto exit; @@ -1024,6 +1022,8 @@ INIT_MODULE(_psycopg)(void) PyModule_AddStringConstant(module, "__version__", PSYCOPG_VERSION); PyModule_AddStringConstant(module, "__doc__", "psycopg PostgreSQL driver"); PyModule_AddIntConstant(module, "__libpq_version__", PG_VERSION_NUM); + PyModule_AddIntMacro(module, REPLICATION_PHYSICAL); + PyModule_AddIntMacro(module, REPLICATION_LOGICAL); PyModule_AddObject(module, "apilevel", Text_FromUTF8(APILEVEL)); PyModule_AddObject(module, "threadsafety", PyInt_FromLong(THREADSAFETY)); PyModule_AddObject(module, "paramstyle", Text_FromUTF8(PARAMSTYLE)); @@ -1031,6 +1031,9 @@ INIT_MODULE(_psycopg)(void) /* put new types in module dictionary */ PyModule_AddObject(module, "connection", (PyObject*)&connectionType); PyModule_AddObject(module, "cursor", (PyObject*)&cursorType); + PyModule_AddObject(module, "ReplicationConnection", (PyObject*)&replicationConnectionType); + PyModule_AddObject(module, "ReplicationCursor", (PyObject*)&replicationCursorType); + PyModule_AddObject(module, "ReplicationMessage", (PyObject*)&replicationMessageType); PyModule_AddObject(module, "ISQLQuote", (PyObject*)&isqlquoteType); PyModule_AddObject(module, "Notify", (PyObject*)¬ifyType); PyModule_AddObject(module, "Xid", (PyObject*)&xidType); @@ -1070,6 +1073,9 @@ INIT_MODULE(_psycopg)(void) if (0 != psyco_errors_init()) { goto exit; } psyco_errors_fill(dict); + replicationPhysicalConst = PyDict_GetItemString(dict, "REPLICATION_PHYSICAL"); + replicationLogicalConst = PyDict_GetItemString(dict, "REPLICATION_LOGICAL"); + Dprintf("initpsycopg: module initialization complete"); exit: diff --git a/psycopg/python.h b/psycopg/python.h index 90c82516..cfb8dad3 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -31,8 +31,8 @@ #include #endif -#if PY_VERSION_HEX < 0x02050000 -# error "psycopg requires Python >= 2.5" +#if PY_VERSION_HEX < 0x02060000 +# error "psycopg requires Python >= 2.6" #endif /* hash() return size changed around version 3.2a4 on 64bit platforms. Before diff --git a/psycopg/replication_connection.h b/psycopg/replication_connection.h new file mode 100644 index 00000000..e693038a --- /dev/null +++ b/psycopg/replication_connection.h @@ -0,0 +1,55 @@ +/* replication_connection.h - definition for the psycopg replication connection type + * + * Copyright (C) 2015 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#ifndef PSYCOPG_REPLICATION_CONNECTION_H +#define PSYCOPG_REPLICATION_CONNECTION_H 1 + +#include "psycopg/connection.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern HIDDEN PyTypeObject replicationConnectionType; + +typedef struct replicationConnectionObject { + connectionObject conn; + + long int type; +} replicationConnectionObject; + +/* The funny constant values should help to avoid mixups with some + commonly used numbers like 1 and 2. */ +#define REPLICATION_PHYSICAL 12345678 +#define REPLICATION_LOGICAL 87654321 + +extern HIDDEN PyObject *replicationPhysicalConst; +extern HIDDEN PyObject *replicationLogicalConst; + +#ifdef __cplusplus +} +#endif + +#endif /* !defined(PSYCOPG_REPLICATION_CONNECTION_H) */ diff --git a/psycopg/replication_connection_type.c b/psycopg/replication_connection_type.c new file mode 100644 index 00000000..5e5d2229 --- /dev/null +++ b/psycopg/replication_connection_type.c @@ -0,0 +1,221 @@ +/* replication_connection_type.c - python interface to replication connection objects + * + * Copyright (C) 2015 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/replication_connection.h" +#include "psycopg/replication_message.h" +#include "psycopg/green.h" +#include "psycopg/pqpath.h" + +#include +#include + + +#define psyco_repl_conn_type_doc \ +"replication_type -- the replication connection type" + +static PyObject * +psyco_repl_conn_get_type(replicationConnectionObject *self) +{ + connectionObject *conn = &self->conn; + PyObject *res = NULL; + + EXC_IF_CONN_CLOSED(conn); + + if (self->type == REPLICATION_PHYSICAL) { + res = replicationPhysicalConst; + } else if (self->type == REPLICATION_LOGICAL) { + res = replicationLogicalConst; + } else { + PyErr_Format(PyExc_TypeError, "unknown replication type constant: %ld", self->type); + } + + Py_XINCREF(res); + return res; +} + + +static int +replicationConnection_init(PyObject *obj, PyObject *args, PyObject *kwargs) +{ + replicationConnectionObject *self = (replicationConnectionObject *)obj; + PyObject *dsn = NULL, *replication_type = NULL, + *item = NULL, *ext = NULL, *make_dsn = NULL, + *extras = NULL, *cursor = NULL; + int async = 0; + int ret = -1; + + /* 'replication_type' is not actually optional, but there's no + good way to put it before 'async' in the list */ + static char *kwlist[] = {"dsn", "async", "replication_type", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|iO", kwlist, + &dsn, &async, &replication_type)) { return ret; } + + /* + We have to call make_dsn() to add replication-specific + connection parameters, because the DSN might be an URI (if there + were no keyword arguments to connect() it is passed unchanged). + */ + /* we reuse args and kwargs to call make_dsn() and parent type's tp_init() */ + if (!(kwargs = PyDict_New())) { return ret; } + Py_INCREF(args); + + /* we also reuse the dsn to hold the result of the make_dsn() call */ + Py_INCREF(dsn); + + if (!(ext = PyImport_ImportModule("psycopg2.extensions"))) { goto exit; } + if (!(make_dsn = PyObject_GetAttrString(ext, "make_dsn"))) { goto exit; } + + /* all the nice stuff is located in python-level ReplicationCursor class */ + if (!(extras = PyImport_ImportModule("psycopg2.extras"))) { goto exit; } + if (!(cursor = PyObject_GetAttrString(extras, "ReplicationCursor"))) { goto exit; } + + /* checking the object reference helps to avoid recognizing + unrelated integer constants as valid input values */ + if (replication_type == replicationPhysicalConst) { + self->type = REPLICATION_PHYSICAL; + +#define SET_ITEM(k, v) \ + if (!(item = Text_FromUTF8(#v))) { goto exit; } \ + if (PyDict_SetItemString(kwargs, #k, item) != 0) { goto exit; } \ + Py_DECREF(item); \ + item = NULL; + + SET_ITEM(replication, true); + SET_ITEM(dbname, replication); /* required for .pgpass lookup */ + } else if (replication_type == replicationLogicalConst) { + self->type = REPLICATION_LOGICAL; + + SET_ITEM(replication, database); +#undef SET_ITEM + } else { + PyErr_SetString(PyExc_TypeError, + "replication_type must be either REPLICATION_PHYSICAL or REPLICATION_LOGICAL"); + goto exit; + } + + Py_DECREF(args); + if (!(args = PyTuple_Pack(1, dsn))) { goto exit; } + + Py_DECREF(dsn); + if (!(dsn = PyObject_Call(make_dsn, args, kwargs))) { goto exit; } + + Py_DECREF(args); + if (!(args = Py_BuildValue("(Oi)", dsn, async))) { goto exit; } + + /* only attempt the connection once we've handled all possible errors */ + if ((ret = connectionType.tp_init(obj, args, NULL)) < 0) { goto exit; } + + self->conn.autocommit = 1; + Py_INCREF(self->conn.cursor_factory = cursor); + +exit: + Py_XDECREF(item); + Py_XDECREF(ext); + Py_XDECREF(make_dsn); + Py_XDECREF(extras); + Py_XDECREF(cursor); + Py_XDECREF(dsn); + Py_XDECREF(args); + Py_XDECREF(kwargs); + + return ret; +} + +static PyObject * +replicationConnection_repr(replicationConnectionObject *self) +{ + return PyString_FromFormat( + "", + self, self->conn.dsn, self->conn.closed); +} + + +/* object calculated member list */ + +static struct PyGetSetDef replicationConnectionObject_getsets[] = { + /* override to prevent user tweaking these: */ + { "autocommit", NULL, NULL, NULL }, + { "isolation_level", NULL, NULL, NULL }, + { "set_session", NULL, NULL, NULL }, + { "set_isolation_level", NULL, NULL, NULL }, + { "reset", NULL, NULL, NULL }, + /* an actual getter */ + { "replication_type", + (getter)psyco_repl_conn_get_type, NULL, + psyco_repl_conn_type_doc, NULL }, + {NULL} +}; + +/* object type */ + +#define replicationConnectionType_doc \ +"A replication connection." + +PyTypeObject replicationConnectionType = { + PyVarObject_HEAD_INIT(NULL, 0) + "psycopg2.extensions.ReplicationConnection", + sizeof(replicationConnectionObject), 0, + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + (reprfunc)replicationConnection_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + (reprfunc)replicationConnection_repr, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_ITER | + Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + replicationConnectionType_doc, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + replicationConnectionObject_getsets, /*tp_getset*/ + &connectionType, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + replicationConnection_init, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /*tp_new*/ +}; + +PyObject *replicationPhysicalConst; +PyObject *replicationLogicalConst; diff --git a/psycopg/replication_cursor.h b/psycopg/replication_cursor.h new file mode 100644 index 00000000..71c6e190 --- /dev/null +++ b/psycopg/replication_cursor.h @@ -0,0 +1,59 @@ +/* replication_cursor.h - definition for the psycopg replication cursor type + * + * Copyright (C) 2015 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#ifndef PSYCOPG_REPLICATION_CURSOR_H +#define PSYCOPG_REPLICATION_CURSOR_H 1 + +#include "psycopg/cursor.h" +#include "libpq_support.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern HIDDEN PyTypeObject replicationCursorType; + +typedef struct replicationCursorObject { + cursorObject cur; + + int consuming:1; /* if running the consume loop */ + int decode:1; /* if we should use character decoding on the messages */ + + struct timeval last_io; /* timestamp of the last exchange with the server */ + struct timeval keepalive_interval; /* interval for keepalive messages in replication mode */ + + XLogRecPtr write_lsn; /* LSNs for replication feedback messages */ + XLogRecPtr flush_lsn; + XLogRecPtr apply_lsn; +} replicationCursorObject; + + +RAISES_NEG int psyco_repl_curs_datetime_init(void); + +#ifdef __cplusplus +} +#endif + +#endif /* !defined(PSYCOPG_REPLICATION_CURSOR_H) */ diff --git a/psycopg/replication_cursor_type.c b/psycopg/replication_cursor_type.c new file mode 100644 index 00000000..d66bec36 --- /dev/null +++ b/psycopg/replication_cursor_type.c @@ -0,0 +1,315 @@ +/* replication_cursor_type.c - python interface to replication cursor objects + * + * Copyright (C) 2015 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/replication_cursor.h" +#include "psycopg/replication_message.h" +#include "psycopg/green.h" +#include "psycopg/pqpath.h" + +#include +#include + +/* python */ +#include "datetime.h" + + +#define psyco_repl_curs_start_replication_expert_doc \ +"start_replication_expert(command, decode=False) -- Start replication with a given command." + +static PyObject * +psyco_repl_curs_start_replication_expert(replicationCursorObject *self, + PyObject *args, PyObject *kwargs) +{ + cursorObject *curs = &self->cur; + connectionObject *conn = self->cur.conn; + PyObject *res = NULL; + char *command; + long int decode = 0; + static char *kwlist[] = {"command", "decode", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|l", kwlist, &command, &decode)) { + return NULL; + } + + EXC_IF_CURS_CLOSED(curs); + EXC_IF_GREEN(start_replication_expert); + EXC_IF_TPC_PREPARED(conn, start_replication_expert); + + Dprintf("psyco_repl_curs_start_replication_expert: '%s'; decode: %ld", command, decode); + + if (pq_execute(curs, command, conn->async, 1 /* no_result */, 1 /* no_begin */) >= 0) { + res = Py_None; + Py_INCREF(res); + + self->decode = decode; + gettimeofday(&self->last_io, NULL); + } + + return res; +} + +#define psyco_repl_curs_consume_stream_doc \ +"consume_stream(consumer, keepalive_interval=10) -- Consume replication stream." + +static PyObject * +psyco_repl_curs_consume_stream(replicationCursorObject *self, + PyObject *args, PyObject *kwargs) +{ + cursorObject *curs = &self->cur; + PyObject *consume = NULL, *res = NULL; + double keepalive_interval = 10; + static char *kwlist[] = {"consume", "keepalive_interval", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|id", kwlist, + &consume, &keepalive_interval)) { + return NULL; + } + + EXC_IF_CURS_CLOSED(curs); + EXC_IF_CURS_ASYNC(curs, consume_stream); + EXC_IF_GREEN(consume_stream); + EXC_IF_TPC_PREPARED(self->cur.conn, consume_stream); + + Dprintf("psyco_repl_curs_consume_stream"); + + if (keepalive_interval < 1.0) { + psyco_set_error(ProgrammingError, curs, "keepalive_interval must be >= 1 (sec)"); + return NULL; + } + + if (self->consuming) { + PyErr_SetString(ProgrammingError, + "consume_stream cannot be used when already in the consume loop"); + return NULL; + } + + if (curs->pgres == NULL || PQresultStatus(curs->pgres) != PGRES_COPY_BOTH) { + PyErr_SetString(ProgrammingError, + "consume_stream: not replicating, call start_replication first"); + return NULL; + } + CLEARPGRES(curs->pgres); + + self->consuming = 1; + + if (pq_copy_both(self, consume, keepalive_interval) >= 0) { + res = Py_None; + Py_INCREF(res); + } + + self->consuming = 0; + + return res; +} + +#define psyco_repl_curs_read_message_doc \ +"read_message() -- Try reading a replication message from the server (non-blocking)." + +static PyObject * +psyco_repl_curs_read_message(replicationCursorObject *self) +{ + cursorObject *curs = &self->cur; + replicationMessageObject *msg = NULL; + + EXC_IF_CURS_CLOSED(curs); + EXC_IF_GREEN(read_message); + EXC_IF_TPC_PREPARED(self->cur.conn, read_message); + + if (pq_read_replication_message(self, &msg) < 0) { + return NULL; + } + if (msg) { + return (PyObject *)msg; + } + + Py_RETURN_NONE; +} + +#define psyco_repl_curs_send_feedback_doc \ +"send_feedback(write_lsn=0, flush_lsn=0, apply_lsn=0, reply=False) -- Try sending a replication feedback message to the server and optionally request a reply." + +static PyObject * +psyco_repl_curs_send_feedback(replicationCursorObject *self, + PyObject *args, PyObject *kwargs) +{ + cursorObject *curs = &self->cur; + XLogRecPtr write_lsn = 0, flush_lsn = 0, apply_lsn = 0; + int reply = 0; + static char* kwlist[] = {"write_lsn", "flush_lsn", "apply_lsn", "reply", NULL}; + + EXC_IF_CURS_CLOSED(curs); + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|KKKi", kwlist, + &write_lsn, &flush_lsn, &apply_lsn, &reply)) { + return NULL; + } + + if (write_lsn > self->write_lsn) + self->write_lsn = write_lsn; + + if (flush_lsn > self->flush_lsn) + self->flush_lsn = flush_lsn; + + if (apply_lsn > self->apply_lsn) + self->apply_lsn = apply_lsn; + + if (pq_send_replication_feedback(self, reply) < 0) { + return NULL; + } + + Py_RETURN_NONE; +} + + +RAISES_NEG int +psyco_repl_curs_datetime_init(void) +{ + Dprintf("psyco_repl_curs_datetime_init: datetime init"); + + PyDateTime_IMPORT; + + if (!PyDateTimeAPI) { + PyErr_SetString(PyExc_ImportError, "datetime initialization failed"); + return -1; + } + return 0; +} + +#define psyco_repl_curs_io_timestamp_doc \ +"io_timestamp -- the timestamp of latest IO with the server" + +static PyObject * +psyco_repl_curs_get_io_timestamp(replicationCursorObject *self) +{ + cursorObject *curs = &self->cur; + PyObject *tval, *res = NULL; + double seconds; + + EXC_IF_CURS_CLOSED(curs); + + seconds = self->last_io.tv_sec + self->last_io.tv_usec / 1.0e6; + + tval = Py_BuildValue("(d)", seconds); + if (tval) { + res = PyDateTime_FromTimestamp(tval); + Py_DECREF(tval); + } + return res; +} + +/* object method list */ + +static struct PyMethodDef replicationCursorObject_methods[] = { + {"start_replication_expert", (PyCFunction)psyco_repl_curs_start_replication_expert, + METH_VARARGS|METH_KEYWORDS, psyco_repl_curs_start_replication_expert_doc}, + {"consume_stream", (PyCFunction)psyco_repl_curs_consume_stream, + METH_VARARGS|METH_KEYWORDS, psyco_repl_curs_consume_stream_doc}, + {"read_message", (PyCFunction)psyco_repl_curs_read_message, + METH_NOARGS, psyco_repl_curs_read_message_doc}, + {"send_feedback", (PyCFunction)psyco_repl_curs_send_feedback, + METH_VARARGS|METH_KEYWORDS, psyco_repl_curs_send_feedback_doc}, + {NULL} +}; + +/* object calculated member list */ + +static struct PyGetSetDef replicationCursorObject_getsets[] = { + { "io_timestamp", + (getter)psyco_repl_curs_get_io_timestamp, NULL, + psyco_repl_curs_io_timestamp_doc, NULL }, + {NULL} +}; + +static int +replicationCursor_init(PyObject *obj, PyObject *args, PyObject *kwargs) +{ + replicationCursorObject *self = (replicationCursorObject *)obj; + + self->consuming = 0; + self->decode = 0; + + self->write_lsn = 0; + self->flush_lsn = 0; + self->apply_lsn = 0; + + return cursorType.tp_init(obj, args, kwargs); +} + +static PyObject * +replicationCursor_repr(replicationCursorObject *self) +{ + return PyString_FromFormat( + "", self, self->cur.closed); +} + + +/* object type */ + +#define replicationCursorType_doc \ +"A database replication cursor." + +PyTypeObject replicationCursorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "psycopg2.extensions.ReplicationCursor", + sizeof(replicationCursorObject), 0, + 0, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + (reprfunc)replicationCursor_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + (reprfunc)replicationCursor_repr, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_ITER | + Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + replicationCursorType_doc, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + replicationCursorObject_methods, /*tp_methods*/ + 0, /*tp_members*/ + replicationCursorObject_getsets, /*tp_getset*/ + &cursorType, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + replicationCursor_init, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /*tp_new*/ +}; diff --git a/psycopg/replication_message.h b/psycopg/replication_message.h new file mode 100644 index 00000000..b4d93d67 --- /dev/null +++ b/psycopg/replication_message.h @@ -0,0 +1,57 @@ +/* replication_message.h - definition for the psycopg ReplicationMessage type + * + * Copyright (C) 2003-2015 Federico Di Gregorio + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#ifndef PSYCOPG_REPLICATION_MESSAGE_H +#define PSYCOPG_REPLICATION_MESSAGE_H 1 + +#include "cursor.h" +#include "libpq_support.h" + +#ifdef __cplusplus +extern "C" { +#endif + +extern HIDDEN PyTypeObject replicationMessageType; + +/* the typedef is forward-declared in psycopg.h */ +struct replicationMessageObject { + PyObject_HEAD + + cursorObject *cursor; + PyObject *payload; + + int data_size; + XLogRecPtr data_start; + XLogRecPtr wal_end; + int64_t send_time; +}; + +RAISES_NEG int psyco_replmsg_datetime_init(void); + +#ifdef __cplusplus +} +#endif + +#endif /* !defined(PSYCOPG_REPLICATION_MESSAGE_H) */ diff --git a/psycopg/replication_message_type.c b/psycopg/replication_message_type.c new file mode 100644 index 00000000..b37c402e --- /dev/null +++ b/psycopg/replication_message_type.c @@ -0,0 +1,191 @@ +/* replication_message_type.c - python interface to ReplcationMessage objects + * + * Copyright (C) 2003-2015 Federico Di Gregorio + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/replication_message.h" + +#include "datetime.h" + +RAISES_NEG int +psyco_replmsg_datetime_init(void) +{ + Dprintf("psyco_replmsg_datetime_init: datetime init"); + + PyDateTime_IMPORT; + + if (!PyDateTimeAPI) { + PyErr_SetString(PyExc_ImportError, "datetime initialization failed"); + return -1; + } + return 0; +} + + +static PyObject * +replmsg_repr(replicationMessageObject *self) +{ + return PyString_FromFormat( + "", + self, self->data_size, XLOGFMTARGS(self->data_start), XLOGFMTARGS(self->wal_end), + (long int)self->send_time); +} + +static int +replmsg_init(PyObject *obj, PyObject *args, PyObject *kwargs) +{ + replicationMessageObject *self = (replicationMessageObject*) obj; + + if (!PyArg_ParseTuple(args, "O!O", &cursorType, &self->cursor, &self->payload)) + return -1; + Py_XINCREF(self->cursor); + Py_XINCREF(self->payload); + + self->data_size = 0; + self->data_start = 0; + self->wal_end = 0; + self->send_time = 0; + + return 0; +} + +static int +replmsg_traverse(replicationMessageObject *self, visitproc visit, void *arg) +{ + Py_VISIT((PyObject* )self->cursor); + Py_VISIT(self->payload); + return 0; +} + +static int +replmsg_clear(replicationMessageObject *self) +{ + Py_CLEAR(self->cursor); + Py_CLEAR(self->payload); + return 0; +} + +static void +replmsg_dealloc(PyObject* obj) +{ + PyObject_GC_UnTrack(obj); + + replmsg_clear((replicationMessageObject*) obj); + + Py_TYPE(obj)->tp_free(obj); +} + +#define psyco_replmsg_send_time_doc \ +"send_time - Timestamp of the replication message departure from the server." + +static PyObject * +psyco_replmsg_get_send_time(replicationMessageObject *self) +{ + PyObject *tval, *res = NULL; + double t; + + t = (double)self->send_time / USECS_PER_SEC + + ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY); + + tval = Py_BuildValue("(d)", t); + if (tval) { + res = PyDateTime_FromTimestamp(tval); + Py_DECREF(tval); + } + + return res; +} + +#define OFFSETOF(x) offsetof(replicationMessageObject, x) + +/* object member list */ + +static struct PyMemberDef replicationMessageObject_members[] = { + {"cursor", T_OBJECT, OFFSETOF(cursor), READONLY, + "Related ReplcationCursor object."}, + {"payload", T_OBJECT, OFFSETOF(payload), READONLY, + "The actual message data."}, + {"data_size", T_INT, OFFSETOF(data_size), READONLY, + "Raw size of the message data in bytes."}, + {"data_start", T_ULONGLONG, OFFSETOF(data_start), READONLY, + "LSN position of the start of this message."}, + {"wal_end", T_ULONGLONG, OFFSETOF(wal_end), READONLY, + "LSN position of the current end of WAL on the server."}, + {NULL} +}; + +static struct PyGetSetDef replicationMessageObject_getsets[] = { + { "send_time", (getter)psyco_replmsg_get_send_time, NULL, + psyco_replmsg_send_time_doc, NULL }, + {NULL} +}; + +/* object type */ + +#define replicationMessageType_doc \ +"A replication protocol message." + +PyTypeObject replicationMessageType = { + PyVarObject_HEAD_INIT(NULL, 0) + "psycopg2.extensions.ReplicationMessage", + sizeof(replicationMessageObject), 0, + replmsg_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + (reprfunc)replmsg_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + replicationMessageType_doc, /*tp_doc*/ + (traverseproc)replmsg_traverse, /*tp_traverse*/ + (inquiry)replmsg_clear, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + replicationMessageObject_members, /*tp_members*/ + replicationMessageObject_getsets, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + replmsg_init, /*tp_init*/ + 0, /*tp_alloc*/ + PyType_GenericNew, /*tp_new*/ +}; diff --git a/psycopg/typecast_builtins.c b/psycopg/typecast_builtins.c index a104b7c4..fa548a73 100644 --- a/psycopg/typecast_builtins.c +++ b/psycopg/typecast_builtins.c @@ -25,6 +25,9 @@ static long int typecast_DATEARRAY_types[] = {1182, 0}; static long int typecast_INTERVALARRAY_types[] = {1187, 0}; static long int typecast_BINARYARRAY_types[] = {1001, 0}; static long int typecast_ROWIDARRAY_types[] = {1028, 1013, 0}; +static long int typecast_INETARRAY_types[] = {1041, 0}; +static long int typecast_CIDRARRAY_types[] = {651, 0}; +static long int typecast_MACADDRARRAY_types[] = {1040, 0}; static long int typecast_UNKNOWN_types[] = {705, 0}; @@ -57,6 +60,9 @@ static typecastObject_initlist typecast_builtins[] = { {"BINARYARRAY", typecast_BINARYARRAY_types, typecast_BINARYARRAY_cast, "BINARY"}, {"ROWIDARRAY", typecast_ROWIDARRAY_types, typecast_ROWIDARRAY_cast, "ROWID"}, {"UNKNOWN", typecast_UNKNOWN_types, typecast_UNKNOWN_cast, NULL}, + {"INETARRAY", typecast_INETARRAY_types, typecast_STRINGARRAY_cast, "STRING"}, + {"CIDRARRAY", typecast_CIDRARRAY_types, typecast_STRINGARRAY_cast, "STRING"}, + {"MACADDRARRAY", typecast_MACADDRARRAY_types, typecast_STRINGARRAY_cast, "STRING"}, {NULL, NULL, NULL, NULL} }; diff --git a/psycopg/utils.c b/psycopg/utils.c index e224a0d5..44fec6d1 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -50,8 +50,13 @@ psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, Py_ssize_t ql; int eq = (conn && (conn->equote)) ? 1 : 0; - if (len == 0) + if (len == 0) { len = strlen(from); + } else if (strchr(from, '\0') != from + len) { + PyErr_Format(PyExc_ValueError, "A string literal cannot contain NUL (0x00) characters."); + + return NULL; + } if (to == NULL) { to = (char *)PyMem_Malloc((len * 2 + 4) * sizeof(char)); @@ -62,12 +67,10 @@ psycopg_escape_string(connectionObject *conn, const char *from, Py_ssize_t len, } { - #if PG_VERSION_NUM >= 80104 int err; if (conn && conn->pgconn) ql = PQescapeStringConn(conn->pgconn, to+eq+1, from, len, &err); else - #endif ql = PQescapeString(to+eq+1, from, len); } @@ -276,3 +279,32 @@ psycopg_is_text_file(PyObject *f) } } +/* Make a dict out of PQconninfoOption array */ +PyObject * +psycopg_dict_from_conninfo_options(PQconninfoOption *options, int include_password) +{ + PyObject *dict, *res = NULL; + PQconninfoOption *o; + + if (!(dict = PyDict_New())) { goto exit; } + for (o = options; o->keyword != NULL; o++) { + if (o->val != NULL && + (include_password || strcmp(o->keyword, "password") != 0)) { + PyObject *value; + if (!(value = Text_FromUTF8(o->val))) { goto exit; } + if (PyDict_SetItemString(dict, o->keyword, value) != 0) { + Py_DECREF(value); + goto exit; + } + Py_DECREF(value); + } + } + + res = dict; + dict = NULL; + +exit: + Py_XDECREF(dict); + + return res; +} diff --git a/psycopg/win32_support.c b/psycopg/win32_support.c new file mode 100644 index 00000000..d508b220 --- /dev/null +++ b/psycopg/win32_support.c @@ -0,0 +1,76 @@ +/* win32_support.c - emulate some functions missing on Win32 + * + * Copyright (C) 2003-2015 Federico Di Gregorio + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/win32_support.h" + +#ifdef _WIN32 + +#ifndef __MINGW32__ +/* millisecond-precision port of gettimeofday for Win32, taken from + src/port/gettimeofday.c in PostgreSQL core */ + +/* FILETIME of Jan 1 1970 00:00:00. */ +static const unsigned __int64 epoch = 116444736000000000ULL; + +/* + * timezone information is stored outside the kernel so tzp isn't used anymore. + * + * Note: this function is not for Win32 high precision timing purpose. See + * elapsed_time(). + */ +int +gettimeofday(struct timeval * tp, struct timezone * tzp) +{ + FILETIME file_time; + SYSTEMTIME system_time; + ULARGE_INTEGER ularge; + + GetSystemTime(&system_time); + SystemTimeToFileTime(&system_time, &file_time); + ularge.LowPart = file_time.dwLowDateTime; + ularge.HighPart = file_time.dwHighDateTime; + + tp->tv_sec = (long) ((ularge.QuadPart - epoch) / 10000000L); + tp->tv_usec = (long) (system_time.wMilliseconds * 1000); + + return 0; +} +#endif /* !defined(__MINGW32__) */ + +/* timersub is missing on mingw */ +void +timersub(struct timeval *a, struct timeval *b, struct timeval *c) +{ + c->tv_sec = a->tv_sec - b->tv_sec; + c->tv_usec = a->tv_usec - b->tv_usec; + if (tv_usec < 0) { + c->tv_usec += 1000000; + c->tv_sec -= 1; + } +} +#endif /* defined(_WIN32) */ diff --git a/psycopg/win32_support.h b/psycopg/win32_support.h new file mode 100644 index 00000000..be963df5 --- /dev/null +++ b/psycopg/win32_support.h @@ -0,0 +1,40 @@ +/* win32_support.h - definitions for win32_support.c + * + * Copyright (C) 2003-2015 Federico Di Gregorio + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ +#ifndef PSYCOPG_WIN32_SUPPORT_H +#define PSYCOPG_WIN32_SUPPORT_H 1 + +#include "psycopg/config.h" + +#include + +#ifdef _WIN32 +#ifndef __MINGW32__ +HIDDEN int gettimeofday(struct timeval * tp, struct timezone * tzp); +#endif + +HIDDEN void timersub(struct timeval *a, struct timeval *b, struct timeval *c); +#endif + +#endif /* !defined(PSYCOPG_WIN32_SUPPORT_H) */ diff --git a/psycopg2.cproj b/psycopg2.cproj index 7755b961..f6f85c72 100644 --- a/psycopg2.cproj +++ b/psycopg2.cproj @@ -85,14 +85,19 @@ + + + + + @@ -124,6 +129,7 @@ + @@ -217,10 +223,14 @@ + + + + @@ -229,6 +239,7 @@ + @@ -251,4 +262,4 @@ - \ No newline at end of file + diff --git a/scripts/buildtypes.py b/scripts/buildtypes.py index d50a6b66..5ae6c947 100644 --- a/scripts/buildtypes.py +++ b/scripts/buildtypes.py @@ -19,8 +19,8 @@ # code defines the DBAPITypeObject fundamental types and warns for # undefined types. -import sys, os, string, copy -from string import split, join, strip +import sys +from string import split, strip # here is the list of the foundamental types we want to import from @@ -37,7 +37,7 @@ basic_types = (['NUMBER', ['INT8', 'INT4', 'INT2', 'FLOAT8', 'FLOAT4', ['STRING', ['NAME', 'CHAR', 'TEXT', 'BPCHAR', 'VARCHAR']], ['BOOLEAN', ['BOOL']], - ['DATETIME', ['TIMESTAMP', 'TIMESTAMPTZ', + ['DATETIME', ['TIMESTAMP', 'TIMESTAMPTZ', 'TINTERVAL', 'INTERVAL']], ['TIME', ['TIME', 'TIMETZ']], ['DATE', ['DATE']], @@ -73,8 +73,7 @@ FOOTER = """ {NULL, NULL, NULL, NULL}\n};\n""" # useful error reporting function def error(msg): """Report an error on stderr.""" - sys.stderr.write(msg+'\n') - + sys.stderr.write(msg + '\n') # read couples from stdin and build list read_types = [] @@ -91,14 +90,14 @@ for t in basic_types: for v in t[1]: found = filter(lambda x, y=v: x[0] == y, read_types) if len(found) == 0: - error(v+': value not found') + error(v + ': value not found') elif len(found) > 1: - error(v+': too many values') + error(v + ': too many values') else: found_types[k].append(int(found[0][1])) # now outputs to stdout the right C-style definitions -stypes = "" ; sstruct = "" +stypes = sstruct = "" for t in basic_types: k = t[0] s = str(found_types[k]) @@ -108,7 +107,7 @@ for t in basic_types: % (k, k, k)) for t in array_types: kt = t[0] - ka = t[0]+'ARRAY' + ka = t[0] + 'ARRAY' s = str(t[1]) s = '{' + s[1:-1] + ', 0}' stypes = stypes + ('static long int typecast_%s_types[] = %s;\n' % (ka, s)) diff --git a/scripts/fix_b.py b/scripts/fix_b.py deleted file mode 100644 index ccc8e1c9..00000000 --- a/scripts/fix_b.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Fixer to change b('string') into b'string'.""" -# Author: Daniele Varrazzo - -import token -from lib2to3 import fixer_base -from lib2to3.pytree import Leaf - -class FixB(fixer_base.BaseFix): - - PATTERN = """ - power< wrapper='b' trailer< '(' arg=[any] ')' > rest=any* > - """ - - def transform(self, node, results): - arg = results['arg'] - wrapper = results["wrapper"] - if len(arg) == 1 and arg[0].type == token.STRING: - b = Leaf(token.STRING, 'b' + arg[0].value, prefix=wrapper.prefix) - node.children = [ b ] + results['rest'] - diff --git a/scripts/make_errorcodes.py b/scripts/make_errorcodes.py index 122e0d56..26f7e68a 100755 --- a/scripts/make_errorcodes.py +++ b/scripts/make_errorcodes.py @@ -23,6 +23,7 @@ from collections import defaultdict from BeautifulSoup import BeautifulSoup as BS + def main(): if len(sys.argv) != 2: print >>sys.stderr, "usage: %s /path/to/errorcodes.py" % sys.argv[0] @@ -33,7 +34,7 @@ def main(): file_start = read_base_file(filename) # If you add a version to the list fix the docs (errorcodes.rst, err.rst) classes, errors = fetch_errors( - ['8.1', '8.2', '8.3', '8.4', '9.0', '9.1', '9.2', '9.3', '9.4']) + ['8.1', '8.2', '8.3', '8.4', '9.0', '9.1', '9.2', '9.3', '9.4', '9.5']) f = open(filename, "w") for line in file_start: @@ -41,6 +42,7 @@ def main(): for line in generate_module_data(classes, errors): print >>f, line + def read_base_file(filename): rv = [] for line in open(filename): @@ -50,6 +52,7 @@ def read_base_file(filename): raise ValueError("can't find the separator. Is this the right file?") + def parse_errors_txt(url): classes = {} errors = defaultdict(dict) @@ -84,6 +87,7 @@ def parse_errors_txt(url): return classes, errors + def parse_errors_sgml(url): page = BS(urllib2.urlopen(url)) table = page('table')[1]('tbody')[0] @@ -92,7 +96,7 @@ def parse_errors_sgml(url): errors = defaultdict(dict) for tr in table('tr'): - if tr.td.get('colspan'): # it's a class + if tr.td.get('colspan'): # it's a class label = ' '.join(' '.join(tr(text=True)).split()) \ .replace(u'\u2014', '-').encode('ascii') assert label.startswith('Class') @@ -100,7 +104,7 @@ def parse_errors_sgml(url): assert len(class_) == 2 classes[class_] = label - else: # it's an error + else: # it's an error errcode = tr.tt.string.encode("ascii") assert len(errcode) == 5 @@ -124,11 +128,12 @@ def parse_errors_sgml(url): return classes, errors errors_sgml_url = \ - "http://www.postgresql.org/docs/%s/static/errcodes-appendix.html" + "http://www.postgresql.org/docs/%s/static/errcodes-appendix.html" errors_txt_url = \ - "http://git.postgresql.org/gitweb/?p=postgresql.git;a=blob_plain;" \ - "f=src/backend/utils/errcodes.txt;hb=REL%s_STABLE" + "http://git.postgresql.org/gitweb/?p=postgresql.git;a=blob_plain;" \ + "f=src/backend/utils/errcodes.txt;hb=REL%s_STABLE" + def fetch_errors(versions): classes = {} @@ -148,14 +153,15 @@ def fetch_errors(versions): return classes, errors + def generate_module_data(classes, errors): yield "" yield "# Error classes" for clscode, clslabel in sorted(classes.items()): err = clslabel.split(" - ")[1].split("(")[0] \ - .strip().replace(" ", "_").replace('/', "_").upper() + .strip().replace(" ", "_").replace('/', "_").upper() yield "CLASS_%s = %r" % (err, clscode) - + for clscode, clslabel in sorted(classes.items()): yield "" yield "# %s" % clslabel @@ -163,7 +169,6 @@ def generate_module_data(classes, errors): for errcode, errlabel in sorted(errors[clscode].items()): yield "%s = %r" % (errlabel, errcode) + if __name__ == '__main__': sys.exit(main()) - - diff --git a/scripts/refcounter.py b/scripts/refcounter.py index 38544fe0..9e900cf7 100755 --- a/scripts/refcounter.py +++ b/scripts/refcounter.py @@ -25,6 +25,7 @@ import unittest from pprint import pprint from collections import defaultdict + def main(): opt = parse_args() @@ -58,6 +59,7 @@ def main(): return rv + def parse_args(): import optparse @@ -83,7 +85,7 @@ def dump(i, opt): c[type(o)] += 1 pprint( - sorted(((v,str(k)) for k,v in c.items()), reverse=True), + sorted(((v, str(k)) for k, v in c.items()), reverse=True), stream=open("debug-%02d.txt" % i, "w")) if opt.objs: @@ -95,7 +97,7 @@ def dump(i, opt): # TODO: very incomplete if t is dict: - co.sort(key = lambda d: d.items()) + co.sort(key=lambda d: d.items()) else: co.sort() @@ -104,4 +106,3 @@ def dump(i, opt): if __name__ == '__main__': sys.exit(main()) - diff --git a/scripts/travis_prepare.sh b/scripts/travis_prepare.sh new file mode 100755 index 00000000..2b1e12eb --- /dev/null +++ b/scripts/travis_prepare.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +# Prepare the test databases in Travis CI. +# The script should be run with sudo. +# The script is not idempotent: it assumes the machine in a clean state +# and is designed for a sudo-enabled Trusty environment. + +set_param () { + # Set a parameter in a postgresql.conf file + version=$1 + param=$2 + value=$3 + + sed -i "s/^\s*#\?\s*$param.*/$param = $value/" \ + "/etc/postgresql/$version/psycopg/postgresql.conf" +} + +create () { + version=$1 + port=$2 + dbname=psycopg2_test + + pg_createcluster -p $port --start-conf manual $version psycopg + + # for two-phase commit testing + set_param "$version" max_prepared_transactions 10 + + # for replication testing + set_param "$version" max_wal_senders 5 + set_param "$version" max_replication_slots 5 + if [ "$version" == "9.2" -o "$version" == "9.3" ] + then + set_param "$version" wal_level hot_standby + else + set_param "$version" wal_level logical + fi + + echo "local replication travis trust" \ + >> "/etc/postgresql/$version/psycopg/pg_hba.conf" + + + pg_ctlcluster "$version" psycopg start + + sudo -u postgres psql -c "create user travis replication" "port=$port" + sudo -u postgres psql -c "create database $dbname" "port=$port" + sudo -u postgres psql -c "grant create on database $dbname to travis" "port=$port" + sudo -u postgres psql -c "create extension hstore" "port=$port dbname=$dbname" +} + + +# Would give a permission denied error in the travis build dir +cd / + +create 9.6 54396 +create 9.5 54395 +create 9.4 54394 +create 9.3 54393 +create 9.2 54392 diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh new file mode 100755 index 00000000..15783088 --- /dev/null +++ b/scripts/travis_test.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Run the tests in all the databases +# The script is designed for a Trusty environment. + +set -e + +run_test () { + version=$1 + port=$2 + dbname=psycopg2_test + + printf "\n\nRunning tests against PostgreSQL $version\n\n" + export PSYCOPG2_TESTDB=$dbname + export PSYCOPG2_TESTDB_PORT=$port + export PSYCOPG2_TESTDB_USER=travis + export PSYCOPG2_TEST_REPL_DSN= + + python -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" --verbose + + printf "\n\nRunning tests against PostgreSQL $version (green mode)\n\n" + export PSYCOPG2_TEST_GREEN=1 + python -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" --verbose +} + +run_test 9.6 54396 +run_test 9.5 54395 +run_test 9.4 54394 +run_test 9.3 54393 +run_test 9.2 54392 diff --git a/setup.cfg b/setup.cfg index 90a47dd4..0d41934f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,24 +7,23 @@ define= # "pg_config" is required to locate PostgreSQL headers and libraries needed to # build psycopg2. If pg_config is not in the path or is installed under a -# different name uncomment the following option and set it to the pg_config -# full path. -#pg_config= +# different name set the following option to the pg_config full path. +pg_config= # Set to 1 to use Python datetime objects for default date/time representation. use_pydatetime=1 # If the build system does not find the mx.DateTime headers, try -# uncommenting the following line and setting its value to the right path. -#mx_include_dir= +# setting its value to the right path. +mx_include_dir= # For Windows only: # Set to 1 if the PostgreSQL library was built with OpenSSL. # Required to link in OpenSSL libraries and dependencies. have_ssl=0 -# Statically link against the postgresql client library. -#static_libpq=1 +# Set to 1 to statically link against the postgresql client library. +static_libpq=0 # Add here eventual extra libraries required to link the module. -#libraries= +libraries= diff --git a/setup.py b/setup.py index 2de8c5ef..c1065258 100644 --- a/setup.py +++ b/setup.py @@ -25,39 +25,17 @@ UPDATEs. psycopg2 also provide full asynchronous operations and support for coroutine libraries. """ -# note: if you are changing the list of supported Python version please fix -# the docs in install.rst and the /features/ page on the website. -classifiers = """\ -Development Status :: 5 - Production/Stable -Intended Audience :: Developers -License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) -License :: OSI Approved :: Zope Public License -Programming Language :: Python -Programming Language :: Python :: 2.5 -Programming Language :: Python :: 2.6 -Programming Language :: Python :: 2.7 -Programming Language :: Python :: 3 -Programming Language :: Python :: 3.1 -Programming Language :: Python :: 3.2 -Programming Language :: Python :: 3.3 -Programming Language :: Python :: 3.4 -Programming Language :: C -Programming Language :: SQL -Topic :: Database -Topic :: Database :: Front-Ends -Topic :: Software Development -Topic :: Software Development :: Libraries :: Python Modules -Operating System :: Microsoft :: Windows -Operating System :: Unix -""" - # Note: The setup.py must be compatible with both Python 2 and 3 + import os import sys import re import subprocess -from distutils.core import setup, Extension +try: + from setuptools import setup, Extension +except ImportError: + from distutils.core import setup, Extension from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc from distutils.ccompiler import get_default_compiler @@ -72,10 +50,6 @@ else: # workaround subclass for ticket #153 pass - # Configure distutils to run our custom 2to3 fixers as well - from lib2to3.refactor import get_fixers_from_package - build_py.fixer_names = get_fixers_from_package('lib2to3.fixes') \ - + [ 'fix_b' ] sys.path.insert(0, 'scripts') try: @@ -88,7 +62,34 @@ except ImportError: PSYCOPG_VERSION = '2.7.dev0' -version_flags = ['dt', 'dec'] + +# note: if you are changing the list of supported Python version please fix +# the docs in install.rst and the /features/ page on the website. +classifiers = """\ +Development Status :: 5 - Production/Stable +Intended Audience :: Developers +License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) +License :: OSI Approved :: Zope Public License +Programming Language :: Python +Programming Language :: Python :: 2.6 +Programming Language :: Python :: 2.7 +Programming Language :: Python :: 3 +Programming Language :: Python :: 3.1 +Programming Language :: Python :: 3.2 +Programming Language :: Python :: 3.3 +Programming Language :: Python :: 3.4 +Programming Language :: Python :: 3.5 +Programming Language :: C +Programming Language :: SQL +Topic :: Database +Topic :: Database :: Front-Ends +Topic :: Software Development +Topic :: Software Development :: Libraries :: Python Modules +Operating System :: Microsoft :: Windows +Operating System :: Unix +""" + +version_flags = ['dt', 'dec'] PLATFORM_IS_WINDOWS = sys.platform.lower().startswith('win') @@ -209,7 +210,7 @@ or with the pg_config option in 'setup.cfg'. # Support unicode paths, if this version of Python provides the # necessary infrastructure: if sys.version_info[0] < 3 \ - and hasattr(sys, 'getfilesystemencoding'): + and hasattr(sys, 'getfilesystemencoding'): pg_config_path = pg_config_path.encode( sys.getfilesystemencoding()) @@ -231,7 +232,7 @@ class psycopg_build_ext(build_ext): ('use-pydatetime', None, "Use Python datatime objects for date and time representation."), ('pg-config=', None, - "The name of the pg_config binary and/or full path to find it"), + "The name of the pg_config binary and/or full path to find it"), ('have-ssl', None, "Compile with OpenSSL built PostgreSQL libraries (Windows only)."), ('static-libpq', None, @@ -301,6 +302,10 @@ class psycopg_build_ext(build_ext): except AttributeError: ext_path = os.path.join(self.build_lib, 'psycopg2', '_psycopg.pyd') + # Make sure spawn() will work if compile() was never + # called. https://github.com/psycopg/psycopg2/issues/380 + if not self.compiler.initialized: + self.compiler.initialize() self.compiler.spawn( ['mt.exe', '-nologo', '-manifest', os.path.join('psycopg', manifest), @@ -343,7 +348,8 @@ class psycopg_build_ext(build_ext): self.libraries.append("advapi32") if self.compiler_is_msvc(): # MSVC requires an explicit "libpq" - self.libraries.remove("pq") + if "pq" in self.libraries: + self.libraries.remove("pq") self.libraries.append("secur32") self.libraries.append("libpq") self.libraries.append("shfolder") @@ -375,6 +381,11 @@ class psycopg_build_ext(build_ext): def finalize_options(self): """Complete the build system configuration.""" + # An empty option in the setup.cfg causes self.libraries to include + # an empty string in the list of libraries + if self.libraries is not None and not self.libraries.strip(): + self.libraries = None + build_ext.finalize_options(self) pg_config_helper = PostgresConfig(self) @@ -384,7 +395,7 @@ class psycopg_build_ext(build_ext): if not getattr(self, 'link_objects', None): self.link_objects = [] self.link_objects.append( - os.path.join(pg_config_helper.query("libdir"), "libpq.a")) + os.path.join(pg_config_helper.query("libdir"), "libpq.a")) else: self.libraries.append("pq") @@ -413,7 +424,7 @@ class psycopg_build_ext(build_ext): else: sys.stderr.write( "Error: could not determine PostgreSQL version from '%s'" - % pgversion) + % pgversion) sys.exit(1) define_macros.append(("PG_VERSION_NUM", "%d%02d%02d" % @@ -441,6 +452,7 @@ class psycopg_build_ext(build_ext): if hasattr(self, "finalize_" + sys.platform): getattr(self, "finalize_" + sys.platform)() + def is_py_64(): # sys.maxint not available since Py 3.1; # sys.maxsize not available before Py 2.6; @@ -462,9 +474,13 @@ data_files = [] sources = [ 'psycopgmodule.c', 'green.c', 'pqpath.c', 'utils.c', 'bytes_format.c', + 'libpq_support.c', 'win32_support.c', 'connection_int.c', 'connection_type.c', 'cursor_int.c', 'cursor_type.c', + 'replication_connection_type.c', + 'replication_cursor_type.c', + 'replication_message_type.c', 'diagnostics_type.c', 'error_type.c', 'lobject_int.c', 'lobject_type.c', 'notify_type.c', 'xid_type.c', @@ -480,7 +496,11 @@ depends = [ # headers 'config.h', 'pgtypes.h', 'psycopg.h', 'python.h', 'connection.h', 'cursor.h', 'diagnostics.h', 'error.h', 'green.h', 'lobject.h', + 'replication_connection.h', + 'replication_cursor.h', + 'replication_message.h', 'notify.h', 'pqpath.h', 'xid.h', + 'libpq_support.h', 'win32_support.h', 'adapter_asis.h', 'adapter_binary.h', 'adapter_datetime.h', 'adapter_list.h', 'adapter_pboolean.h', 'adapter_pdecimal.h', @@ -499,14 +519,14 @@ parser.read('setup.cfg') # Choose a datetime module have_pydatetime = True have_mxdatetime = False -use_pydatetime = int(parser.get('build_ext', 'use_pydatetime')) +use_pydatetime = int(parser.get('build_ext', 'use_pydatetime')) # check for mx package if parser.has_option('build_ext', 'mx_include_dir'): mxincludedir = parser.get('build_ext', 'mx_include_dir') else: mxincludedir = os.path.join(get_python_inc(plat_specific=1), "mx") -if os.path.exists(mxincludedir): +if mxincludedir.strip() and os.path.exists(mxincludedir): # Build the support for mx: we will check at runtime if it can be imported include_dirs.append(mxincludedir) define_macros.append(('HAVE_MXDATETIME', '1')) @@ -535,8 +555,8 @@ you probably need to install its companion -dev or -devel package.""" sys.exit(1) # generate a nice version string to avoid confusion when users report bugs -version_flags.append('pq3') # no more a choice -version_flags.append('ext') # no more a choice +version_flags.append('pq3') # no more a choice +version_flags.append('ext') # no more a choice if version_flags: PSYCOPG_VERSION_EX = PSYCOPG_VERSION + " (%s)" % ' '.join(version_flags) @@ -568,8 +588,8 @@ for define in parser.get('build_ext', 'define').split(','): # build the extension -sources = [ os.path.join('psycopg', x) for x in sources] -depends = [ os.path.join('psycopg', x) for x in depends] +sources = [os.path.join('psycopg', x) for x in sources] +depends = [os.path.join('psycopg', x) for x in depends] ext.append(Extension("psycopg2._psycopg", sources, define_macros=define_macros, diff --git a/tests/__init__.py b/tests/__init__.py index 3e677d85..1a240994 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -31,11 +31,14 @@ import test_bugX000 import test_bug_gc import test_cancel import test_connection +import test_replication import test_copy import test_cursor import test_dates +import test_errcodes import test_extras_dictcursor import test_green +import test_ipaddress import test_lobject import test_module import test_notify @@ -44,11 +47,8 @@ import test_quote import test_transaction import test_types_basic import test_types_extras +import test_with -if sys.version_info[:2] >= (2, 5): - import test_with -else: - test_with = None def test_suite(): # If connection to test db fails, bail out early. @@ -68,11 +68,14 @@ def test_suite(): suite.addTest(test_bug_gc.test_suite()) suite.addTest(test_cancel.test_suite()) suite.addTest(test_connection.test_suite()) + suite.addTest(test_replication.test_suite()) suite.addTest(test_copy.test_suite()) suite.addTest(test_cursor.test_suite()) suite.addTest(test_dates.test_suite()) + suite.addTest(test_errcodes.test_suite()) suite.addTest(test_extras_dictcursor.test_suite()) suite.addTest(test_green.test_suite()) + suite.addTest(test_ipaddress.test_suite()) suite.addTest(test_lobject.test_suite()) suite.addTest(test_module.test_suite()) suite.addTest(test_notify.test_suite()) @@ -81,8 +84,7 @@ def test_suite(): suite.addTest(test_transaction.test_suite()) suite.addTest(test_types_basic.test_suite()) suite.addTest(test_types_extras.test_suite()) - if test_with: - suite.addTest(test_with.test_suite()) + suite.addTest(test_with.test_suite()) return suite if __name__ == '__main__': diff --git a/tests/test_async.py b/tests/test_async.py index d40b9c3e..6f8fed58 100755 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -29,11 +29,11 @@ import psycopg2 from psycopg2 import extensions import time -import select import StringIO from testutils import ConnectingTestCase + class PollableStub(object): """A 'pollable' wrapper allowing analysis of the `poll()` calls.""" def __init__(self, pollable): @@ -66,24 +66,10 @@ class AsyncTests(ConnectingTestCase): )''') self.wait(curs) - def wait(self, cur_or_conn): - pollable = cur_or_conn - if not hasattr(pollable, 'poll'): - pollable = cur_or_conn.connection - while True: - state = pollable.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_READ: - select.select([pollable], [], [], 10) - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [pollable], [], 10) - else: - raise Exception("Unexpected result from poll: %r", state) - def test_connection_setup(self): cur = self.conn.cursor() sync_cur = self.sync_conn.cursor() + del cur, sync_cur self.assert_(self.conn.async) self.assert_(not self.sync_conn.async) @@ -93,7 +79,7 @@ class AsyncTests(ConnectingTestCase): # check other properties to be found on the connection self.assert_(self.conn.server_version) - self.assert_(self.conn.protocol_version in (2,3)) + self.assert_(self.conn.protocol_version in (2, 3)) self.assert_(self.conn.encoding in psycopg2.extensions.encodings) def test_async_named_cursor(self): @@ -124,6 +110,7 @@ class AsyncTests(ConnectingTestCase): def test_async_after_async(self): cur = self.conn.cursor() cur2 = self.conn.cursor() + del cur2 cur.execute("insert into table1 values (1)") @@ -438,14 +425,14 @@ class AsyncTests(ConnectingTestCase): def test_async_cursor_gone(self): import gc cur = self.conn.cursor() - cur.execute("select 42;"); + cur.execute("select 42;") del cur gc.collect() self.assertRaises(psycopg2.InterfaceError, self.wait, self.conn) # The connection is still usable cur = self.conn.cursor() - cur.execute("select 42;"); + cur.execute("select 42;") self.wait(self.conn) self.assertEqual(cur.fetchone(), (42,)) @@ -465,4 +452,3 @@ def test_suite(): if __name__ == "__main__": unittest.main() - diff --git a/tests/test_bugX000.py b/tests/test_bugX000.py index efa593ec..fbd2a9f6 100755 --- a/tests/test_bugX000.py +++ b/tests/test_bugX000.py @@ -26,15 +26,17 @@ import psycopg2 import time import unittest + class DateTimeAllocationBugTestCase(unittest.TestCase): def test_date_time_allocation_bug(self): - d1 = psycopg2.Date(2002,12,25) - d2 = psycopg2.DateFromTicks(time.mktime((2002,12,25,0,0,0,0,0,0))) - t1 = psycopg2.Time(13,45,30) - t2 = psycopg2.TimeFromTicks(time.mktime((2001,1,1,13,45,30,0,0,0))) - t1 = psycopg2.Timestamp(2002,12,25,13,45,30) + d1 = psycopg2.Date(2002, 12, 25) + d2 = psycopg2.DateFromTicks(time.mktime((2002, 12, 25, 0, 0, 0, 0, 0, 0))) + t1 = psycopg2.Time(13, 45, 30) + t2 = psycopg2.TimeFromTicks(time.mktime((2001, 1, 1, 13, 45, 30, 0, 0, 0))) + t1 = psycopg2.Timestamp(2002, 12, 25, 13, 45, 30) t2 = psycopg2.TimestampFromTicks( - time.mktime((2002,12,25,13,45,30,0,0,0))) + time.mktime((2002, 12, 25, 13, 45, 30, 0, 0, 0))) + del d1, d2, t1, t2 def test_suite(): diff --git a/tests/test_bug_gc.py b/tests/test_bug_gc.py index 1551dc47..084236ef 100755 --- a/tests/test_bug_gc.py +++ b/tests/test_bug_gc.py @@ -29,6 +29,7 @@ import gc from testutils import ConnectingTestCase, skip_if_no_uuid + class StolenReferenceTestCase(ConnectingTestCase): @skip_if_no_uuid def test_stolen_reference_bug(self): @@ -41,8 +42,10 @@ class StolenReferenceTestCase(ConnectingTestCase): curs.execute("select 'b5219e01-19ab-4994-b71e-149225dc51e4'::uuid") curs.fetchone() + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_cancel.py b/tests/test_cancel.py index 0ffa742a..a8eb7506 100755 --- a/tests/test_cancel.py +++ b/tests/test_cancel.py @@ -32,6 +32,7 @@ from psycopg2 import extras from testconfig import dsn from testutils import unittest, ConnectingTestCase, skip_before_postgres + class CancelTests(ConnectingTestCase): def setUp(self): @@ -71,6 +72,7 @@ class CancelTests(ConnectingTestCase): except Exception, e: errors.append(e) raise + del cur thread1 = threading.Thread(target=neverending, args=(self.conn, )) # wait a bit to make sure that the other thread is already in diff --git a/tests/test_connection.py b/tests/test_connection.py index 68bb6f05..833751b9 100755 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -27,16 +27,16 @@ import sys import time import threading from operator import attrgetter -from StringIO import StringIO import psycopg2 import psycopg2.errorcodes -import psycopg2.extensions +from psycopg2 import extensions as ext + +from testutils import ( + unittest, decorate_all_tests, skip_if_no_superuser, + skip_before_postgres, skip_after_postgres, skip_before_libpq, + ConnectingTestCase, skip_if_tpc_disabled, skip_if_windows) -from testutils import unittest, decorate_all_tests, skip_if_no_superuser -from testutils import skip_before_postgres, skip_after_postgres, skip_before_libpq -from testutils import ConnectingTestCase, skip_if_tpc_disabled -from testutils import skip_if_windows from testconfig import dsn, dbname @@ -111,8 +111,14 @@ class ConnectionTests(ConnectingTestCase): cur = conn.cursor() if self.conn.server_version >= 90300: cur.execute("set client_min_messages=debug1") - cur.execute("create temp table table1 (id serial); create temp table table2 (id serial);") - cur.execute("create temp table table3 (id serial); create temp table table4 (id serial);") + cur.execute(""" + create temp table table1 (id serial); + create temp table table2 (id serial); + """) + cur.execute(""" + create temp table table3 (id serial); + create temp table table4 (id serial); + """) self.assertEqual(4, len(conn.notices)) self.assert_('table1' in conn.notices[0]) self.assert_('table2' in conn.notices[1]) @@ -125,7 +131,8 @@ class ConnectionTests(ConnectingTestCase): if self.conn.server_version >= 90300: cur.execute("set client_min_messages=debug1") for i in range(0, 100, 10): - sql = " ".join(["create temp table table%d (id serial);" % j for j in range(i, i+10)]) + sql = " ".join(["create temp table table%d (id serial);" % j + for j in range(i, i + 10)]) cur.execute(sql) self.assertEqual(50, len(conn.notices)) @@ -140,8 +147,13 @@ class ConnectionTests(ConnectingTestCase): if self.conn.server_version >= 90300: cur.execute("set client_min_messages=debug1") - cur.execute("create temp table table1 (id serial); create temp table table2 (id serial);") - cur.execute("create temp table table3 (id serial); create temp table table4 (id serial);") + cur.execute(""" + create temp table table1 (id serial); + create temp table table2 (id serial); + """) + cur.execute(""" + create temp table table3 (id serial); + create temp table table4 (id serial);""") self.assertEqual(len(conn.notices), 4) self.assert_('table1' in conn.notices.popleft()) self.assert_('table2' in conn.notices.popleft()) @@ -151,7 +163,8 @@ class ConnectionTests(ConnectingTestCase): # not limited, but no error for i in range(0, 100, 10): - sql = " ".join(["create temp table table2_%d (id serial);" % j for j in range(i, i+10)]) + sql = " ".join(["create temp table table2_%d (id serial);" % j + for j in range(i, i + 10)]) cur.execute(sql) self.assertEqual(len([n for n in conn.notices if 'CREATE TABLE' in n]), @@ -172,7 +185,7 @@ class ConnectionTests(ConnectingTestCase): self.assert_(self.conn.server_version) def test_protocol_version(self): - self.assert_(self.conn.protocol_version in (2,3), + self.assert_(self.conn.protocol_version in (2, 3), self.conn.protocol_version) def test_tpc_unsupported(self): @@ -252,7 +265,7 @@ class ConnectionTests(ConnectingTestCase): t1.start() i = 1 for i in range(1000): - cur.execute("select %s;",(i,)) + cur.execute("select %s;", (i,)) conn.commit() while conn.notices: notices.append((1, conn.notices.pop())) @@ -313,18 +326,19 @@ class ConnectionTests(ConnectingTestCase): class ParseDsnTestCase(ConnectingTestCase): def test_parse_dsn(self): from psycopg2 import ProgrammingError - from psycopg2.extensions import parse_dsn - self.assertEqual(parse_dsn('dbname=test user=tester password=secret'), - dict(user='tester', password='secret', dbname='test'), - "simple DSN parsed") + self.assertEqual( + ext.parse_dsn('dbname=test user=tester password=secret'), + dict(user='tester', password='secret', dbname='test'), + "simple DSN parsed") - self.assertRaises(ProgrammingError, parse_dsn, + self.assertRaises(ProgrammingError, ext.parse_dsn, "dbname=test 2 user=tester password=secret") - self.assertEqual(parse_dsn("dbname='test 2' user=tester password=secret"), - dict(user='tester', password='secret', dbname='test 2'), - "DSN with quoting parsed") + self.assertEqual( + ext.parse_dsn("dbname='test 2' user=tester password=secret"), + dict(user='tester', password='secret', dbname='test 2'), + "DSN with quoting parsed") # Can't really use assertRaisesRegexp() here since we need to # make sure that secret is *not* exposed in the error messgage @@ -332,7 +346,7 @@ class ParseDsnTestCase(ConnectingTestCase): raised = False try: # unterminated quote after dbname: - parse_dsn("dbname='test 2 user=tester password=secret") + ext.parse_dsn("dbname='test 2 user=tester password=secret") except ProgrammingError, e: raised = True self.assertTrue(str(e).find('secret') < 0, @@ -343,16 +357,14 @@ class ParseDsnTestCase(ConnectingTestCase): @skip_before_libpq(9, 2) def test_parse_dsn_uri(self): - from psycopg2.extensions import parse_dsn - - self.assertEqual(parse_dsn('postgresql://tester:secret@/test'), + self.assertEqual(ext.parse_dsn('postgresql://tester:secret@/test'), dict(user='tester', password='secret', dbname='test'), "valid URI dsn parsed") raised = False try: # extra '=' after port value - parse_dsn(dsn='postgresql://tester:secret@/test?port=1111=x') + ext.parse_dsn(dsn='postgresql://tester:secret@/test?port=1111=x') except psycopg2.ProgrammingError, e: raised = True self.assertTrue(str(e).find('secret') < 0, @@ -362,24 +374,98 @@ class ParseDsnTestCase(ConnectingTestCase): self.assertTrue(raised, "ProgrammingError raised due to invalid URI") def test_unicode_value(self): - from psycopg2.extensions import parse_dsn snowman = u"\u2603" - d = parse_dsn('dbname=' + snowman) + d = ext.parse_dsn('dbname=' + snowman) if sys.version_info[0] < 3: self.assertEqual(d['dbname'], snowman.encode('utf8')) else: self.assertEqual(d['dbname'], snowman) def test_unicode_key(self): - from psycopg2.extensions import parse_dsn snowman = u"\u2603" - self.assertRaises(psycopg2.ProgrammingError, parse_dsn, + self.assertRaises(psycopg2.ProgrammingError, ext.parse_dsn, snowman + '=' + snowman) def test_bad_param(self): - from psycopg2.extensions import parse_dsn - self.assertRaises(TypeError, parse_dsn, None) - self.assertRaises(TypeError, parse_dsn, 42) + self.assertRaises(TypeError, ext.parse_dsn, None) + self.assertRaises(TypeError, ext.parse_dsn, 42) + + +class MakeDsnTestCase(ConnectingTestCase): + def assertDsnEqual(self, dsn1, dsn2): + self.assertEqual(set(dsn1.split()), set(dsn2.split())) + + def test_empty_arguments(self): + self.assertEqual(ext.make_dsn(), '') + + def test_empty_string(self): + dsn = ext.make_dsn('') + self.assertEqual(dsn, '') + + def test_params_validation(self): + self.assertRaises(psycopg2.ProgrammingError, + ext.make_dsn, 'dbnamo=a') + self.assertRaises(psycopg2.ProgrammingError, + ext.make_dsn, dbnamo='a') + self.assertRaises(psycopg2.ProgrammingError, + ext.make_dsn, 'dbname=a', nosuchparam='b') + + def test_empty_param(self): + dsn = ext.make_dsn(dbname='sony', password='') + self.assertDsnEqual(dsn, "dbname=sony password=''") + + def test_escape(self): + dsn = ext.make_dsn(dbname='hello world') + self.assertEqual(dsn, "dbname='hello world'") + + dsn = ext.make_dsn(dbname=r'back\slash') + self.assertEqual(dsn, r"dbname=back\\slash") + + dsn = ext.make_dsn(dbname="quo'te") + self.assertEqual(dsn, r"dbname=quo\'te") + + dsn = ext.make_dsn(dbname="with\ttab") + self.assertEqual(dsn, "dbname='with\ttab'") + + dsn = ext.make_dsn(dbname=r"\every thing'") + self.assertEqual(dsn, r"dbname='\\every thing\''") + + def test_database_is_a_keyword(self): + self.assertEqual(ext.make_dsn(database='sigh'), "dbname=sigh") + + def test_params_merging(self): + dsn = ext.make_dsn('dbname=foo host=bar', host='baz') + self.assertDsnEqual(dsn, 'dbname=foo host=baz') + + dsn = ext.make_dsn('dbname=foo', user='postgres') + self.assertDsnEqual(dsn, 'dbname=foo user=postgres') + + def test_no_dsn_munging(self): + dsnin = 'dbname=a host=b user=c password=d' + dsn = ext.make_dsn(dsnin) + self.assertEqual(dsn, dsnin) + + @skip_before_libpq(9, 2) + def test_url_is_cool(self): + url = 'postgresql://tester:secret@/test?application_name=wat' + dsn = ext.make_dsn(url) + self.assertEqual(dsn, url) + + dsn = ext.make_dsn(url, application_name='woot') + self.assertDsnEqual(dsn, + 'dbname=test user=tester password=secret application_name=woot') + + self.assertRaises(psycopg2.ProgrammingError, + ext.make_dsn, 'postgresql://tester:secret@/test?nosuch=param') + self.assertRaises(psycopg2.ProgrammingError, + ext.make_dsn, url, nosuch="param") + + @skip_before_libpq(9, 3) + def test_get_dsn_parameters(self): + conn = self.connect() + d = conn.get_dsn_parameters() + self.assertEqual(d['dbname'], dbname) # the only param we can check reliably + self.assert_('password' not in d, d) class IsolationLevelsTestCase(ConnectingTestCase): @@ -413,7 +499,8 @@ class IsolationLevelsTestCase(ConnectingTestCase): levels = [ (None, psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT), - ('read uncommitted', psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED), + ('read uncommitted', + psycopg2.extensions.ISOLATION_LEVEL_READ_UNCOMMITTED), ('read committed', psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED), ('repeatable read', psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ), ('serializable', psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE), @@ -587,7 +674,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn.close() return - gids = [ r[0] for r in cur ] + gids = [r[0] for r in cur] for gid in gids: cur.execute("rollback prepared %s;", (gid,)) cnn.close() @@ -761,13 +848,13 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): def test_status_after_recover(self): cnn = self.connect() self.assertEqual(psycopg2.extensions.STATUS_READY, cnn.status) - xns = cnn.tpc_recover() + cnn.tpc_recover() self.assertEqual(psycopg2.extensions.STATUS_READY, cnn.status) cur = cnn.cursor() cur.execute("select 1") self.assertEqual(psycopg2.extensions.STATUS_BEGIN, cnn.status) - xns = cnn.tpc_recover() + cnn.tpc_recover() self.assertEqual(psycopg2.extensions.STATUS_BEGIN, cnn.status) def test_recovered_xids(self): @@ -789,12 +876,12 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn = self.connect() xids = cnn.tpc_recover() - xids = [ xid for xid in xids if xid.database == dbname ] + xids = [xid for xid in xids if xid.database == dbname] xids.sort(key=attrgetter('gtrid')) # check the values returned self.assertEqual(len(okvals), len(xids)) - for (xid, (gid, prepared, owner, database)) in zip (xids, okvals): + for (xid, (gid, prepared, owner, database)) in zip(xids, okvals): self.assertEqual(xid.gtrid, gid) self.assertEqual(xid.prepared, prepared) self.assertEqual(xid.owner, owner) @@ -825,8 +912,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn.close() cnn = self.connect() - xids = [ xid for xid in cnn.tpc_recover() - if xid.database == dbname ] + xids = [x for x in cnn.tpc_recover() if x.database == dbname] self.assertEqual(1, len(xids)) xid = xids[0] self.assertEqual(xid.format_id, fid) @@ -847,8 +933,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn.close() cnn = self.connect() - xids = [ xid for xid in cnn.tpc_recover() - if xid.database == dbname ] + xids = [x for x in cnn.tpc_recover() if x.database == dbname] self.assertEqual(1, len(xids)) xid = xids[0] self.assertEqual(xid.format_id, None) @@ -893,8 +978,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn.tpc_begin(x1) cnn.tpc_prepare() cnn.reset() - xid = [ xid for xid in cnn.tpc_recover() - if xid.database == dbname ][0] + xid = [x for x in cnn.tpc_recover() if x.database == dbname][0] self.assertEqual(10, xid.format_id) self.assertEqual('uni', xid.gtrid) self.assertEqual('code', xid.bqual) @@ -909,8 +993,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn.tpc_prepare() cnn.reset() - xid = [ xid for xid in cnn.tpc_recover() - if xid.database == dbname ][0] + xid = [x for x in cnn.tpc_recover() if x.database == dbname][0] self.assertEqual(None, xid.format_id) self.assertEqual('transaction-id', xid.gtrid) self.assertEqual(None, xid.bqual) @@ -929,7 +1012,7 @@ class ConnectionTwoPhaseTests(ConnectingTestCase): cnn.reset() xids = cnn.tpc_recover() - xid = [ xid for xid in xids if xid.database == dbname ][0] + xid = [x for x in xids if x.database == dbname][0] self.assertEqual(None, xid.format_id) self.assertEqual('dict-connection', xid.gtrid) self.assertEqual(None, xid.bqual) @@ -1178,17 +1261,6 @@ class AutocommitTests(ConnectingTestCase): self.assertEqual(cur.fetchone()[0], 'on') -class ReplicationTest(ConnectingTestCase): - @skip_before_postgres(9, 0) - def test_replication_not_supported(self): - conn = self.repl_connect() - if conn is None: return - cur = conn.cursor() - f = StringIO() - self.assertRaises(psycopg2.NotSupportedError, - cur.copy_expert, "START_REPLICATION 0/0", f) - - def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) diff --git a/tests/test_copy.py b/tests/test_copy.py index 32134215..ac42c980 100755 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -39,7 +39,8 @@ from testconfig import dsn if sys.version_info[0] < 3: _base = object else: - from io import TextIOBase as _base + from io import TextIOBase as _base + class MinimalRead(_base): """A file wrapper exposing the minimal interface to copy from.""" @@ -52,6 +53,7 @@ class MinimalRead(_base): def readline(self): return self.f.readline() + class MinimalWrite(_base): """A file wrapper exposing the minimal interface to copy to.""" def __init__(self, f): @@ -78,7 +80,7 @@ class CopyTests(ConnectingTestCase): def test_copy_from(self): curs = self.conn.cursor() try: - self._copy_from(curs, nrecs=1024, srec=10*1024, copykw={}) + self._copy_from(curs, nrecs=1024, srec=10 * 1024, copykw={}) finally: curs.close() @@ -86,8 +88,8 @@ class CopyTests(ConnectingTestCase): # Trying to trigger a "would block" error curs = self.conn.cursor() try: - self._copy_from(curs, nrecs=10*1024, srec=10*1024, - copykw={'size': 20*1024*1024}) + self._copy_from(curs, nrecs=10 * 1024, srec=10 * 1024, + copykw={'size': 20 * 1024 * 1024}) finally: curs.close() @@ -110,6 +112,7 @@ class CopyTests(ConnectingTestCase): f.write("%s\n" % (i,)) f.seek(0) + def cols(): raise ZeroDivisionError() yield 'id' @@ -120,8 +123,8 @@ class CopyTests(ConnectingTestCase): def test_copy_to(self): curs = self.conn.cursor() try: - self._copy_from(curs, nrecs=1024, srec=10*1024, copykw={}) - self._copy_to(curs, srec=10*1024) + self._copy_from(curs, nrecs=1024, srec=10 * 1024, copykw={}) + self._copy_to(curs, srec=10 * 1024) finally: curs.close() @@ -209,9 +212,11 @@ class CopyTests(ConnectingTestCase): exp_size = 123 # hack here to leave file as is, only check size when reading real_read = f.read + def read(_size, f=f, exp_size=exp_size): self.assertEqual(_size, exp_size) return real_read(_size) + f.read = read curs.copy_expert('COPY tcopy (data) FROM STDIN', f, size=exp_size) curs.execute("select data from tcopy;") @@ -221,7 +226,7 @@ class CopyTests(ConnectingTestCase): f = StringIO() for i, c in izip(xrange(nrecs), cycle(string.ascii_letters)): l = c * srec - f.write("%s\t%s\n" % (i,l)) + f.write("%s\t%s\n" % (i, l)) f.seek(0) curs.copy_from(MinimalRead(f), "tcopy", **copykw) @@ -258,24 +263,24 @@ class CopyTests(ConnectingTestCase): curs.copy_expert, 'COPY tcopy (data) FROM STDIN', f) def test_copy_no_column_limit(self): - cols = [ "c%050d" % i for i in range(200) ] + cols = ["c%050d" % i for i in range(200)] curs = self.conn.cursor() curs.execute('CREATE TEMPORARY TABLE manycols (%s)' % ',\n'.join( - [ "%s int" % c for c in cols])) + ["%s int" % c for c in cols])) curs.execute("INSERT INTO manycols DEFAULT VALUES") f = StringIO() - curs.copy_to(f, "manycols", columns = cols) + curs.copy_to(f, "manycols", columns=cols) f.seek(0) self.assertEqual(f.read().split(), ['\\N'] * len(cols)) f.seek(0) - curs.copy_from(f, "manycols", columns = cols) + curs.copy_from(f, "manycols", columns=cols) curs.execute("select count(*) from manycols;") self.assertEqual(curs.fetchone()[0], 2) - @skip_before_postgres(8, 2) # they don't send the count + @skip_before_postgres(8, 2) # they don't send the count def test_copy_rowcount(self): curs = self.conn.cursor() @@ -316,7 +321,7 @@ try: except psycopg2.ProgrammingError: pass conn.close() -""" % { 'dsn': dsn,}) +""" % {'dsn': dsn}) proc = Popen([sys.executable, '-c', script_to_py3(script)]) proc.communicate() @@ -334,7 +339,7 @@ try: except psycopg2.ProgrammingError: pass conn.close() -""" % { 'dsn': dsn,}) +""" % {'dsn': dsn}) proc = Popen([sys.executable, '-c', script_to_py3(script)], stdout=PIPE) proc.communicate() @@ -343,10 +348,10 @@ conn.close() def test_copy_from_propagate_error(self): class BrokenRead(_base): def read(self, size): - return 1/0 + return 1 / 0 def readline(self): - return 1/0 + return 1 / 0 curs = self.conn.cursor() # It seems we cannot do this, but now at least we propagate the error @@ -360,7 +365,7 @@ conn.close() def test_copy_to_propagate_error(self): class BrokenWrite(_base): def write(self, data): - return 1/0 + return 1 / 0 curs = self.conn.cursor() curs.execute("insert into tcopy values (10, 'hi')") diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 552b29c0..fc924c4b 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -26,10 +26,10 @@ import time import pickle import psycopg2 import psycopg2.extensions -from psycopg2.extensions import b from testutils import unittest, ConnectingTestCase, skip_before_postgres from testutils import skip_if_no_namedtuple, skip_if_no_getrefcount + class CursorTests(ConnectingTestCase): def test_close_idempotent(self): @@ -48,8 +48,10 @@ class CursorTests(ConnectingTestCase): conn = self.conn cur = conn.cursor() cur.execute("create temp table test_exc (data int);") + def buggygen(): - yield 1//0 + yield 1 // 0 + self.assertRaises(ZeroDivisionError, cur.executemany, "insert into test_exc values (%s)", buggygen()) cur.close() @@ -63,28 +65,34 @@ class CursorTests(ConnectingTestCase): # unicode query containing only ascii data cur.execute(u"SELECT 'foo';") self.assertEqual('foo', cur.fetchone()[0]) - self.assertEqual(b("SELECT 'foo';"), cur.mogrify(u"SELECT 'foo';")) + self.assertEqual(b"SELECT 'foo';", cur.mogrify(u"SELECT 'foo';")) conn.set_client_encoding('UTF8') snowman = u"\u2603" + def b(s): + if isinstance(s, unicode): + return s.encode('utf8') + else: + return s + # unicode query with non-ascii data cur.execute(u"SELECT '%s';" % snowman) self.assertEqual(snowman.encode('utf8'), b(cur.fetchone()[0])) self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), - cur.mogrify(u"SELECT '%s';" % snowman).replace(b("E'"), b("'"))) + cur.mogrify(u"SELECT '%s';" % snowman).replace(b"E'", b"'")) # unicode args cur.execute("SELECT %s;", (snowman,)) self.assertEqual(snowman.encode("utf-8"), b(cur.fetchone()[0])) self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), - cur.mogrify("SELECT %s;", (snowman,)).replace(b("E'"), b("'"))) + cur.mogrify("SELECT %s;", (snowman,)).replace(b"E'", b"'")) # unicode query and args cur.execute(u"SELECT %s;", (snowman,)) self.assertEqual(snowman.encode("utf-8"), b(cur.fetchone()[0])) self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), - cur.mogrify(u"SELECT %s;", (snowman,)).replace(b("E'"), b("'"))) + cur.mogrify(u"SELECT %s;", (snowman,)).replace(b"E'", b"'")) def test_mogrify_decimal_explodes(self): # issue #7: explodes on windows with python 2.5 and psycopg 2.2.2 @@ -95,7 +103,7 @@ class CursorTests(ConnectingTestCase): conn = self.conn cur = conn.cursor() - self.assertEqual(b('SELECT 10.3;'), + self.assertEqual(b'SELECT 10.3;', cur.mogrify("SELECT %s;", (Decimal("10.3"),))) @skip_if_no_getrefcount @@ -103,8 +111,7 @@ class CursorTests(ConnectingTestCase): # issue #81: reference leak when a parameter value is referenced # more than once from a dict. cur = self.conn.cursor() - i = lambda x: x - foo = i('foo') * 10 + foo = (lambda x: x)('foo') * 10 import sys nref1 = sys.getrefcount(foo) cur.mogrify("select %(foo)s, %(foo)s, %(foo)s", {'foo': foo}) @@ -136,7 +143,7 @@ class CursorTests(ConnectingTestCase): self.assertEqual(Decimal('123.45'), curs.cast(1700, '123.45')) from datetime import date - self.assertEqual(date(2011,1,2), curs.cast(1082, '2011-01-02')) + self.assertEqual(date(2011, 1, 2), curs.cast(1082, '2011-01-02')) self.assertEqual("who am i?", curs.cast(705, 'who am i?')) # unknown def test_cast_specificity(self): @@ -159,7 +166,8 @@ class CursorTests(ConnectingTestCase): curs = self.conn.cursor() w = ref(curs) del curs - import gc; gc.collect() + import gc + gc.collect() self.assert_(w() is None) def test_null_name(self): @@ -169,7 +177,7 @@ class CursorTests(ConnectingTestCase): def test_invalid_name(self): curs = self.conn.cursor() curs.execute("create temp table invname (data int);") - for i in (10,20,30): + for i in (10, 20, 30): curs.execute("insert into invname values (%s)", (i,)) curs.close() @@ -194,16 +202,16 @@ class CursorTests(ConnectingTestCase): self._create_withhold_table() curs = self.conn.cursor("W") - self.assertEqual(curs.withhold, False); + self.assertEqual(curs.withhold, False) curs.withhold = True - self.assertEqual(curs.withhold, True); + self.assertEqual(curs.withhold, True) curs.execute("select data from withhold order by data") self.conn.commit() self.assertEqual(curs.fetchall(), [(10,), (20,), (30,)]) curs.close() curs = self.conn.cursor("W", withhold=True) - self.assertEqual(curs.withhold, True); + self.assertEqual(curs.withhold, True) curs.execute("select data from withhold order by data") self.conn.commit() self.assertEqual(curs.fetchall(), [(10,), (20,), (30,)]) @@ -265,18 +273,18 @@ class CursorTests(ConnectingTestCase): curs = self.conn.cursor() curs.execute("create table scrollable (data int)") curs.executemany("insert into scrollable values (%s)", - [ (i,) for i in range(100) ]) + [(i,) for i in range(100)]) curs.close() for t in range(2): if not t: curs = self.conn.cursor("S") - self.assertEqual(curs.scrollable, None); + self.assertEqual(curs.scrollable, None) curs.scrollable = True else: curs = self.conn.cursor("S", scrollable=True) - self.assertEqual(curs.scrollable, True); + self.assertEqual(curs.scrollable, True) curs.itersize = 10 # complex enough to make postgres cursors declare without @@ -304,7 +312,7 @@ class CursorTests(ConnectingTestCase): curs = self.conn.cursor() curs.execute("create table scrollable (data int)") curs.executemany("insert into scrollable values (%s)", - [ (i,) for i in range(100) ]) + [(i,) for i in range(100)]) curs.close() curs = self.conn.cursor("S") # default scrollability @@ -341,18 +349,18 @@ class CursorTests(ConnectingTestCase): def test_iter_named_cursor_default_itersize(self): curs = self.conn.cursor('tmp') curs.execute('select generate_series(1,50)') - rv = [ (r[0], curs.rownumber) for r in curs ] + rv = [(r[0], curs.rownumber) for r in curs] # everything swallowed in one gulp - self.assertEqual(rv, [(i,i) for i in range(1,51)]) + self.assertEqual(rv, [(i, i) for i in range(1, 51)]) @skip_before_postgres(8, 0) def test_iter_named_cursor_itersize(self): curs = self.conn.cursor('tmp') curs.itersize = 30 curs.execute('select generate_series(1,50)') - rv = [ (r[0], curs.rownumber) for r in curs ] + rv = [(r[0], curs.rownumber) for r in curs] # everything swallowed in two gulps - self.assertEqual(rv, [(i,((i - 1) % 30) + 1) for i in range(1,51)]) + self.assertEqual(rv, [(i, ((i - 1) % 30) + 1) for i in range(1, 51)]) @skip_before_postgres(8, 0) def test_iter_named_cursor_rownumber(self): diff --git a/tests/test_dates.py b/tests/test_dates.py index d6ce3482..3463d001 100755 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -27,6 +27,7 @@ import psycopg2 from psycopg2.tz import FixedOffsetTimezone, ZERO from testutils import unittest, ConnectingTestCase, skip_before_postgres + class CommonDatetimeTestsMixin: def execute(self, *args): @@ -144,10 +145,10 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): # The Python datetime module does not support time zone # offsets that are not a whole number of minutes. # We round the offset to the nearest minute. - self.check_time_tz("+01:15:00", 60 * (60 + 15)) - self.check_time_tz("+01:15:29", 60 * (60 + 15)) - self.check_time_tz("+01:15:30", 60 * (60 + 16)) - self.check_time_tz("+01:15:59", 60 * (60 + 16)) + self.check_time_tz("+01:15:00", 60 * (60 + 15)) + self.check_time_tz("+01:15:29", 60 * (60 + 15)) + self.check_time_tz("+01:15:30", 60 * (60 + 16)) + self.check_time_tz("+01:15:59", 60 * (60 + 16)) self.check_time_tz("-01:15:00", -60 * (60 + 15)) self.check_time_tz("-01:15:29", -60 * (60 + 15)) self.check_time_tz("-01:15:30", -60 * (60 + 16)) @@ -180,10 +181,10 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): # The Python datetime module does not support time zone # offsets that are not a whole number of minutes. # We round the offset to the nearest minute. - self.check_datetime_tz("+01:15:00", 60 * (60 + 15)) - self.check_datetime_tz("+01:15:29", 60 * (60 + 15)) - self.check_datetime_tz("+01:15:30", 60 * (60 + 16)) - self.check_datetime_tz("+01:15:59", 60 * (60 + 16)) + self.check_datetime_tz("+01:15:00", 60 * (60 + 15)) + self.check_datetime_tz("+01:15:29", 60 * (60 + 15)) + self.check_datetime_tz("+01:15:30", 60 * (60 + 16)) + self.check_datetime_tz("+01:15:59", 60 * (60 + 16)) self.check_datetime_tz("-01:15:00", -60 * (60 + 15)) self.check_datetime_tz("-01:15:29", -60 * (60 + 15)) self.check_datetime_tz("-01:15:30", -60 * (60 + 16)) @@ -269,32 +270,32 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date(self): from datetime import date - self._test_type_roundtrip(date(2010,5,3)) + self._test_type_roundtrip(date(2010, 5, 3)) def test_type_roundtrip_datetime(self): from datetime import datetime - dt = self._test_type_roundtrip(datetime(2010,5,3,10,20,30)) + dt = self._test_type_roundtrip(datetime(2010, 5, 3, 10, 20, 30)) self.assertEqual(None, dt.tzinfo) def test_type_roundtrip_datetimetz(self): from datetime import datetime import psycopg2.tz - tz = psycopg2.tz.FixedOffsetTimezone(8*60) - dt1 = datetime(2010,5,3,10,20,30, tzinfo=tz) + tz = psycopg2.tz.FixedOffsetTimezone(8 * 60) + dt1 = datetime(2010, 5, 3, 10, 20, 30, tzinfo=tz) dt2 = self._test_type_roundtrip(dt1) self.assertNotEqual(None, dt2.tzinfo) self.assertEqual(dt1, dt2) def test_type_roundtrip_time(self): from datetime import time - tm = self._test_type_roundtrip(time(10,20,30)) + tm = self._test_type_roundtrip(time(10, 20, 30)) self.assertEqual(None, tm.tzinfo) def test_type_roundtrip_timetz(self): from datetime import time import psycopg2.tz - tz = psycopg2.tz.FixedOffsetTimezone(8*60) - tm1 = time(10,20,30, tzinfo=tz) + tz = psycopg2.tz.FixedOffsetTimezone(8 * 60) + tm1 = time(10, 20, 30, tzinfo=tz) tm2 = self._test_type_roundtrip(tm1) self.assertNotEqual(None, tm2.tzinfo) self.assertEqual(tm1, tm2) @@ -305,15 +306,15 @@ class DatetimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date_array(self): from datetime import date - self._test_type_roundtrip_array(date(2010,5,3)) + self._test_type_roundtrip_array(date(2010, 5, 3)) def test_type_roundtrip_datetime_array(self): from datetime import datetime - self._test_type_roundtrip_array(datetime(2010,5,3,10,20,30)) + self._test_type_roundtrip_array(datetime(2010, 5, 3, 10, 20, 30)) def test_type_roundtrip_time_array(self): from datetime import time - self._test_type_roundtrip_array(time(10,20,30)) + self._test_type_roundtrip_array(time(10, 20, 30)) def test_type_roundtrip_interval_array(self): from datetime import timedelta @@ -355,8 +356,10 @@ class mxDateTimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): psycopg2.extensions.register_type(self.INTERVAL, self.conn) psycopg2.extensions.register_type(psycopg2.extensions.MXDATEARRAY, self.conn) psycopg2.extensions.register_type(psycopg2.extensions.MXTIMEARRAY, self.conn) - psycopg2.extensions.register_type(psycopg2.extensions.MXDATETIMEARRAY, self.conn) - psycopg2.extensions.register_type(psycopg2.extensions.MXINTERVALARRAY, self.conn) + psycopg2.extensions.register_type( + psycopg2.extensions.MXDATETIMEARRAY, self.conn) + psycopg2.extensions.register_type( + psycopg2.extensions.MXINTERVALARRAY, self.conn) def tearDown(self): self.conn.close() @@ -479,15 +482,15 @@ class mxDateTimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date(self): from mx.DateTime import Date - self._test_type_roundtrip(Date(2010,5,3)) + self._test_type_roundtrip(Date(2010, 5, 3)) def test_type_roundtrip_datetime(self): from mx.DateTime import DateTime - self._test_type_roundtrip(DateTime(2010,5,3,10,20,30)) + self._test_type_roundtrip(DateTime(2010, 5, 3, 10, 20, 30)) def test_type_roundtrip_time(self): from mx.DateTime import Time - self._test_type_roundtrip(Time(10,20,30)) + self._test_type_roundtrip(Time(10, 20, 30)) def test_type_roundtrip_interval(self): from mx.DateTime import DateTimeDeltaFrom @@ -495,15 +498,15 @@ class mxDateTimeTests(ConnectingTestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date_array(self): from mx.DateTime import Date - self._test_type_roundtrip_array(Date(2010,5,3)) + self._test_type_roundtrip_array(Date(2010, 5, 3)) def test_type_roundtrip_datetime_array(self): from mx.DateTime import DateTime - self._test_type_roundtrip_array(DateTime(2010,5,3,10,20,30)) + self._test_type_roundtrip_array(DateTime(2010, 5, 3, 10, 20, 30)) def test_type_roundtrip_time_array(self): from mx.DateTime import Time - self._test_type_roundtrip_array(Time(10,20,30)) + self._test_type_roundtrip_array(Time(10, 20, 30)) def test_type_roundtrip_interval_array(self): from mx.DateTime import DateTimeDeltaFrom @@ -549,22 +552,30 @@ class FixedOffsetTimezoneTests(unittest.TestCase): def test_repr_with_positive_offset(self): tzinfo = FixedOffsetTimezone(5 * 60) - self.assertEqual(repr(tzinfo), "psycopg2.tz.FixedOffsetTimezone(offset=300, name=None)") + self.assertEqual(repr(tzinfo), + "psycopg2.tz.FixedOffsetTimezone(offset=300, name=None)") def test_repr_with_negative_offset(self): tzinfo = FixedOffsetTimezone(-5 * 60) - self.assertEqual(repr(tzinfo), "psycopg2.tz.FixedOffsetTimezone(offset=-300, name=None)") + self.assertEqual(repr(tzinfo), + "psycopg2.tz.FixedOffsetTimezone(offset=-300, name=None)") def test_repr_with_name(self): tzinfo = FixedOffsetTimezone(name="FOO") - self.assertEqual(repr(tzinfo), "psycopg2.tz.FixedOffsetTimezone(offset=0, name='FOO')") + self.assertEqual(repr(tzinfo), + "psycopg2.tz.FixedOffsetTimezone(offset=0, name='FOO')") def test_instance_caching(self): - self.assert_(FixedOffsetTimezone(name="FOO") is FixedOffsetTimezone(name="FOO")) - self.assert_(FixedOffsetTimezone(7 * 60) is FixedOffsetTimezone(7 * 60)) - self.assert_(FixedOffsetTimezone(-9 * 60, 'FOO') is FixedOffsetTimezone(-9 * 60, 'FOO')) - self.assert_(FixedOffsetTimezone(9 * 60) is not FixedOffsetTimezone(9 * 60, 'FOO')) - self.assert_(FixedOffsetTimezone(name='FOO') is not FixedOffsetTimezone(9 * 60, 'FOO')) + self.assert_(FixedOffsetTimezone(name="FOO") + is FixedOffsetTimezone(name="FOO")) + self.assert_(FixedOffsetTimezone(7 * 60) + is FixedOffsetTimezone(7 * 60)) + self.assert_(FixedOffsetTimezone(-9 * 60, 'FOO') + is FixedOffsetTimezone(-9 * 60, 'FOO')) + self.assert_(FixedOffsetTimezone(9 * 60) + is not FixedOffsetTimezone(9 * 60, 'FOO')) + self.assert_(FixedOffsetTimezone(name='FOO') + is not FixedOffsetTimezone(9 * 60, 'FOO')) def test_pickle(self): # ticket #135 diff --git a/tests/test_errcodes.py b/tests/test_errcodes.py new file mode 100755 index 00000000..6865194f --- /dev/null +++ b/tests/test_errcodes.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +# test_errcodes.py - unit test for psycopg2.errcodes module +# +# Copyright (C) 2015 Daniele Varrazzo +# +# psycopg2 is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# In addition, as a special exception, the copyright holders give +# permission to link this program with the OpenSSL library (or with +# modified versions of OpenSSL that use the same license as OpenSSL), +# and distribute linked combinations including the two. +# +# You must obey the GNU Lesser General Public License in all respects for +# all of the code used other than OpenSSL. +# +# psycopg2 is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +from testutils import unittest, ConnectingTestCase + +try: + reload +except NameError: + from imp import reload + +from threading import Thread +from psycopg2 import errorcodes + + +class ErrocodeTests(ConnectingTestCase): + def test_lookup_threadsafe(self): + + # Increase if it does not fail with KeyError + MAX_CYCLES = 2000 + + errs = [] + + def f(pg_code='40001'): + try: + errorcodes.lookup(pg_code) + except Exception, e: + errs.append(e) + + for __ in xrange(MAX_CYCLES): + reload(errorcodes) + (t1, t2) = (Thread(target=f), Thread(target=f)) + (t1.start(), t2.start()) + (t1.join(), t2.join()) + + if errs: + self.fail( + "raised %s errors in %s cycles (first is %s %s)" % ( + len(errs), MAX_CYCLES, + errs[0].__class__.__name__, errs[0])) + + +def test_suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_extras_dictcursor.py b/tests/test_extras_dictcursor.py index f2feffff..20393c66 100755 --- a/tests/test_extras_dictcursor.py +++ b/tests/test_extras_dictcursor.py @@ -39,7 +39,8 @@ class ExtrasDictCursorTests(ConnectingTestCase): self.assert_(isinstance(cur, psycopg2.extras.DictCursor)) self.assertEqual(cur.name, None) # overridable - cur = self.conn.cursor('foo', cursor_factory=psycopg2.extras.NamedTupleCursor) + cur = self.conn.cursor('foo', + cursor_factory=psycopg2.extras.NamedTupleCursor) self.assertEqual(cur.name, 'foo') self.assert_(isinstance(cur, psycopg2.extras.NamedTupleCursor)) @@ -80,7 +81,6 @@ class ExtrasDictCursorTests(ConnectingTestCase): self.failUnless(row[0] == 'bar') return row - def testDictCursorWithPlainCursorRealFetchOne(self): self._testWithPlainCursorReal(lambda curs: curs.fetchone()) @@ -110,7 +110,6 @@ class ExtrasDictCursorTests(ConnectingTestCase): row = getter(curs) self.failUnless(row['foo'] == 'bar') - def testDictCursorWithNamedCursorFetchOne(self): self._testWithNamedCursor(lambda curs: curs.fetchone()) @@ -146,7 +145,6 @@ class ExtrasDictCursorTests(ConnectingTestCase): self.failUnless(row['foo'] == 'bar') self.failUnless(row[0] == 'bar') - def testDictCursorRealWithNamedCursorFetchOne(self): self._testWithNamedCursorReal(lambda curs: curs.fetchone()) @@ -176,12 +174,12 @@ class ExtrasDictCursorTests(ConnectingTestCase): self._testIterRowNumber(curs) def _testWithNamedCursorReal(self, getter): - curs = self.conn.cursor('aname', cursor_factory=psycopg2.extras.RealDictCursor) + curs = self.conn.cursor('aname', + cursor_factory=psycopg2.extras.RealDictCursor) curs.execute("SELECT * FROM ExtrasDictCursorTests") row = getter(curs) self.failUnless(row['foo'] == 'bar') - def _testNamedCursorNotGreedy(self, curs): curs.itersize = 2 curs.execute("""select clock_timestamp() as ts from generate_series(1,3)""") @@ -235,7 +233,7 @@ class NamedTupleCursorTest(ConnectingTestCase): from psycopg2.extras import NamedTupleConnection try: - from collections import namedtuple + from collections import namedtuple # noqa except ImportError: return @@ -346,7 +344,7 @@ class NamedTupleCursorTest(ConnectingTestCase): def test_error_message(self): try: - from collections import namedtuple + from collections import namedtuple # noqa except ImportError: # an import error somewhere from psycopg2.extras import NamedTupleConnection @@ -390,6 +388,7 @@ class NamedTupleCursorTest(ConnectingTestCase): from psycopg2.extras import NamedTupleCursor f_orig = NamedTupleCursor._make_nt calls = [0] + def f_patched(self_): calls[0] += 1 return f_orig(self_) diff --git a/tests/test_green.py b/tests/test_green.py index 506b38fe..0424a2cc 100755 --- a/tests/test_green.py +++ b/tests/test_green.py @@ -29,6 +29,7 @@ import psycopg2.extras from testutils import ConnectingTestCase + class ConnectionStub(object): """A `connection` wrapper allowing analysis of the `poll()` calls.""" def __init__(self, conn): @@ -43,6 +44,7 @@ class ConnectionStub(object): self.polls.append(rv) return rv + class GreenTestCase(ConnectingTestCase): def setUp(self): self._cb = psycopg2.extensions.get_wait_callback() @@ -89,7 +91,7 @@ class GreenTestCase(ConnectingTestCase): curs.fetchone() # now try to do something that will fail in the callback - psycopg2.extensions.set_wait_callback(lambda conn: 1//0) + psycopg2.extensions.set_wait_callback(lambda conn: 1 // 0) self.assertRaises(ZeroDivisionError, curs.execute, "select 2") self.assert_(conn.closed) diff --git a/tests/test_ipaddress.py b/tests/test_ipaddress.py new file mode 100755 index 00000000..97eabbaf --- /dev/null +++ b/tests/test_ipaddress.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# # test_ipaddress.py - tests for ipaddress support # +# Copyright (C) 2016 Daniele Varrazzo +# +# psycopg2 is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# psycopg2 is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +from __future__ import unicode_literals + +import sys +from functools import wraps + +from testutils import unittest, ConnectingTestCase, decorate_all_tests + +import psycopg2 +import psycopg2.extras + + +def skip_if_no_ipaddress(f): + @wraps(f) + def skip_if_no_ipaddress_(self): + if sys.version_info[:2] < (3, 3): + try: + import ipaddress # noqa + except ImportError: + return self.skipTest("'ipaddress' module not available") + + return f(self) + + return skip_if_no_ipaddress_ + + +class NetworkingTestCase(ConnectingTestCase): + def test_inet_cast(self): + import ipaddress as ip + cur = self.conn.cursor() + psycopg2.extras.register_ipaddress(cur) + + cur.execute("select null::inet") + self.assert_(cur.fetchone()[0] is None) + + cur.execute("select '127.0.0.1/24'::inet") + obj = cur.fetchone()[0] + self.assert_(isinstance(obj, ip.IPv4Interface), repr(obj)) + self.assertEquals(obj, ip.ip_interface('127.0.0.1/24')) + + cur.execute("select '::ffff:102:300/128'::inet") + obj = cur.fetchone()[0] + self.assert_(isinstance(obj, ip.IPv6Interface), repr(obj)) + self.assertEquals(obj, ip.ip_interface('::ffff:102:300/128')) + + def test_inet_array_cast(self): + import ipaddress as ip + cur = self.conn.cursor() + psycopg2.extras.register_ipaddress(cur) + cur.execute("select '{NULL,127.0.0.1,::ffff:102:300/128}'::inet[]") + l = cur.fetchone()[0] + self.assert_(l[0] is None) + self.assertEquals(l[1], ip.ip_interface('127.0.0.1')) + self.assertEquals(l[2], ip.ip_interface('::ffff:102:300/128')) + self.assert_(isinstance(l[1], ip.IPv4Interface), l) + self.assert_(isinstance(l[2], ip.IPv6Interface), l) + + def test_inet_adapt(self): + import ipaddress as ip + cur = self.conn.cursor() + psycopg2.extras.register_ipaddress(cur) + + cur.execute("select %s", [ip.ip_interface('127.0.0.1/24')]) + self.assertEquals(cur.fetchone()[0], '127.0.0.1/24') + + cur.execute("select %s", [ip.ip_interface('::ffff:102:300/128')]) + self.assertEquals(cur.fetchone()[0], '::ffff:102:300/128') + + def test_cidr_cast(self): + import ipaddress as ip + cur = self.conn.cursor() + psycopg2.extras.register_ipaddress(cur) + + cur.execute("select null::cidr") + self.assert_(cur.fetchone()[0] is None) + + cur.execute("select '127.0.0.0/24'::cidr") + obj = cur.fetchone()[0] + self.assert_(isinstance(obj, ip.IPv4Network), repr(obj)) + self.assertEquals(obj, ip.ip_network('127.0.0.0/24')) + + cur.execute("select '::ffff:102:300/128'::cidr") + obj = cur.fetchone()[0] + self.assert_(isinstance(obj, ip.IPv6Network), repr(obj)) + self.assertEquals(obj, ip.ip_network('::ffff:102:300/128')) + + def test_cidr_array_cast(self): + import ipaddress as ip + cur = self.conn.cursor() + psycopg2.extras.register_ipaddress(cur) + cur.execute("select '{NULL,127.0.0.1,::ffff:102:300/128}'::cidr[]") + l = cur.fetchone()[0] + self.assert_(l[0] is None) + self.assertEquals(l[1], ip.ip_network('127.0.0.1')) + self.assertEquals(l[2], ip.ip_network('::ffff:102:300/128')) + self.assert_(isinstance(l[1], ip.IPv4Network), l) + self.assert_(isinstance(l[2], ip.IPv6Network), l) + + def test_cidr_adapt(self): + import ipaddress as ip + cur = self.conn.cursor() + psycopg2.extras.register_ipaddress(cur) + + cur.execute("select %s", [ip.ip_network('127.0.0.0/24')]) + self.assertEquals(cur.fetchone()[0], '127.0.0.0/24') + + cur.execute("select %s", [ip.ip_network('::ffff:102:300/128')]) + self.assertEquals(cur.fetchone()[0], '::ffff:102:300/128') + +decorate_all_tests(NetworkingTestCase, skip_if_no_ipaddress) + + +def test_suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lobject.py b/tests/test_lobject.py index fb2297fa..4da20e95 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -29,10 +29,10 @@ from functools import wraps import psycopg2 import psycopg2.extensions -from psycopg2.extensions import b from testutils import unittest, decorate_all_tests, skip_if_tpc_disabled from testutils import ConnectingTestCase, skip_if_green + def skip_if_no_lo(f): @wraps(f) def skip_if_no_lo_(self): @@ -99,7 +99,7 @@ class LargeObjectTests(LargeObjectTestCase): lo = self.conn.lobject() lo2 = self.conn.lobject(lo.oid, "w") self.assertEqual(lo2.mode[0], "w") - lo2.write(b("some data")) + lo2.write(b"some data") def test_open_mode_n(self): # Openning an object in mode "n" gives us a closed lobject. @@ -136,7 +136,7 @@ class LargeObjectTests(LargeObjectTestCase): self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") fp = open(filename, "wb") - fp.write(b("some data")) + fp.write(b"some data") fp.close() lo = self.conn.lobject(0, "r", 0, filename) @@ -150,7 +150,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_write(self): lo = self.conn.lobject() - self.assertEqual(lo.write(b("some data")), len("some data")) + self.assertEqual(lo.write(b"some data"), len("some data")) def test_write_large(self): lo = self.conn.lobject() @@ -159,7 +159,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_read(self): lo = self.conn.lobject() - length = lo.write(b("some data")) + lo.write(b"some data") lo.close() lo = self.conn.lobject(lo.oid) @@ -170,19 +170,19 @@ class LargeObjectTests(LargeObjectTestCase): def test_read_binary(self): lo = self.conn.lobject() - length = lo.write(b("some data")) + lo.write(b"some data") lo.close() lo = self.conn.lobject(lo.oid, "rb") x = lo.read(4) - self.assertEqual(type(x), type(b(''))) - self.assertEqual(x, b("some")) - self.assertEqual(lo.read(), b(" data")) + self.assertEqual(type(x), type(b'')) + self.assertEqual(x, b"some") + self.assertEqual(lo.read(), b" data") def test_read_text(self): lo = self.conn.lobject() snowman = u"\u2603" - length = lo.write(u"some data " + snowman) + lo.write(u"some data " + snowman) lo.close() lo = self.conn.lobject(lo.oid, "rt") @@ -194,7 +194,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_read_large(self): lo = self.conn.lobject() data = "data" * 1000000 - length = lo.write("some" + data) + lo.write("some" + data) lo.close() lo = self.conn.lobject(lo.oid) @@ -206,7 +206,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_seek_tell(self): lo = self.conn.lobject() - length = lo.write(b("some data")) + length = lo.write(b"some data") self.assertEqual(lo.tell(), length) lo.close() lo = self.conn.lobject(lo.oid) @@ -236,7 +236,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_export(self): lo = self.conn.lobject() - lo.write(b("some data")) + lo.write(b"some data") self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") @@ -244,7 +244,7 @@ class LargeObjectTests(LargeObjectTestCase): self.assertTrue(os.path.exists(filename)) f = open(filename, "rb") try: - self.assertEqual(f.read(), b("some data")) + self.assertEqual(f.read(), b"some data") finally: f.close() @@ -256,7 +256,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_write_after_close(self): lo = self.conn.lobject() lo.close() - self.assertRaises(psycopg2.InterfaceError, lo.write, b("some data")) + self.assertRaises(psycopg2.InterfaceError, lo.write, b"some data") def test_read_after_close(self): lo = self.conn.lobject() @@ -281,7 +281,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_export_after_close(self): lo = self.conn.lobject() - lo.write(b("some data")) + lo.write(b"some data") lo.close() self.tmpdir = tempfile.mkdtemp() @@ -290,7 +290,7 @@ class LargeObjectTests(LargeObjectTestCase): self.assertTrue(os.path.exists(filename)) f = open(filename, "rb") try: - self.assertEqual(f.read(), b("some data")) + self.assertEqual(f.read(), b"some data") finally: f.close() @@ -307,7 +307,7 @@ class LargeObjectTests(LargeObjectTestCase): self.lo_oid = lo.oid self.conn.commit() - self.assertRaises(psycopg2.ProgrammingError, lo.write, b("some data")) + self.assertRaises(psycopg2.ProgrammingError, lo.write, b"some data") def test_read_after_commit(self): lo = self.conn.lobject() @@ -340,7 +340,7 @@ class LargeObjectTests(LargeObjectTestCase): def test_export_after_commit(self): lo = self.conn.lobject() - lo.write(b("some data")) + lo.write(b"some data") self.conn.commit() self.tmpdir = tempfile.mkdtemp() @@ -349,7 +349,7 @@ class LargeObjectTests(LargeObjectTestCase): self.assertTrue(os.path.exists(filename)) f = open(filename, "rb") try: - self.assertEqual(f.read(), b("some data")) + self.assertEqual(f.read(), b"some data") finally: f.close() @@ -400,6 +400,7 @@ def skip_if_no_truncate(f): return skip_if_no_truncate_ + class LargeObjectTruncateTests(LargeObjectTestCase): def test_truncate(self): lo = self.conn.lobject() @@ -451,15 +452,19 @@ def _has_lo64(conn): return (True, "this server and build support the lo64 API") + def skip_if_no_lo64(f): @wraps(f) def skip_if_no_lo64_(self): lo64, msg = _has_lo64(self.conn) - if not lo64: return self.skipTest(msg) - else: return f(self) + if not lo64: + return self.skipTest(msg) + else: + return f(self) return skip_if_no_lo64_ + class LargeObject64Tests(LargeObjectTestCase): def test_seek_tell_truncate_greater_than_2gb(self): lo = self.conn.lobject() @@ -478,11 +483,14 @@ def skip_if_lo64(f): @wraps(f) def skip_if_lo64_(self): lo64, msg = _has_lo64(self.conn) - if lo64: return self.skipTest(msg) - else: return f(self) + if lo64: + return self.skipTest(msg) + else: + return f(self) return skip_if_lo64_ + class LargeObjectNot64Tests(LargeObjectTestCase): def test_seek_larger_than_2gb(self): lo = self.conn.lobject() diff --git a/tests/test_module.py b/tests/test_module.py index 62b85ee2..6a1606d6 100755 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -31,9 +31,11 @@ from testutils import ConnectingTestCase, skip_copy_if_green, script_to_py3 import psycopg2 + class ConnectTestCase(unittest.TestCase): def setUp(self): self.args = None + def conect_stub(dsn, connection_factory=None, async=False): self.args = (dsn, connection_factory, async) @@ -43,6 +45,9 @@ class ConnectTestCase(unittest.TestCase): def tearDown(self): psycopg2._connect = self._connect_orig + def assertDsnEqual(self, dsn1, dsn2): + self.assertEqual(set(dsn1.split()), set(dsn2.split())) + def test_there_has_to_be_something(self): self.assertRaises(TypeError, psycopg2.connect) self.assertRaises(TypeError, psycopg2.connect, @@ -57,8 +62,8 @@ class ConnectTestCase(unittest.TestCase): self.assertEqual(self.args[2], False) def test_dsn(self): - psycopg2.connect('dbname=blah x=y') - self.assertEqual(self.args[0], 'dbname=blah x=y') + psycopg2.connect('dbname=blah host=y') + self.assertEqual(self.args[0], 'dbname=blah host=y') self.assertEqual(self.args[1], None) self.assertEqual(self.args[2], False) @@ -83,37 +88,43 @@ class ConnectTestCase(unittest.TestCase): self.assertEqual(len(self.args[0].split()), 4) def test_generic_keywords(self): - psycopg2.connect(foo='bar') - self.assertEqual(self.args[0], 'foo=bar') + psycopg2.connect(options='stuff') + self.assertEqual(self.args[0], 'options=stuff') def test_factory(self): def f(dsn, async=False): pass - psycopg2.connect(database='foo', bar='baz', connection_factory=f) - self.assertEqual(self.args[0], 'dbname=foo bar=baz') + psycopg2.connect(database='foo', host='baz', connection_factory=f) + self.assertDsnEqual(self.args[0], 'dbname=foo host=baz') self.assertEqual(self.args[1], f) self.assertEqual(self.args[2], False) - psycopg2.connect("dbname=foo bar=baz", connection_factory=f) - self.assertEqual(self.args[0], 'dbname=foo bar=baz') + psycopg2.connect("dbname=foo host=baz", connection_factory=f) + self.assertDsnEqual(self.args[0], 'dbname=foo host=baz') self.assertEqual(self.args[1], f) self.assertEqual(self.args[2], False) def test_async(self): - psycopg2.connect(database='foo', bar='baz', async=1) - self.assertEqual(self.args[0], 'dbname=foo bar=baz') + psycopg2.connect(database='foo', host='baz', async=1) + self.assertDsnEqual(self.args[0], 'dbname=foo host=baz') self.assertEqual(self.args[1], None) self.assert_(self.args[2]) - psycopg2.connect("dbname=foo bar=baz", async=True) - self.assertEqual(self.args[0], 'dbname=foo bar=baz') + psycopg2.connect("dbname=foo host=baz", async=True) + self.assertDsnEqual(self.args[0], 'dbname=foo host=baz') self.assertEqual(self.args[1], None) self.assert_(self.args[2]) + def test_int_port_param(self): + psycopg2.connect(database='sony', port=6543) + dsn = " %s " % self.args[0] + self.assert_(" dbname=sony " in dsn, dsn) + self.assert_(" port=6543 " in dsn, dsn) + def test_empty_param(self): psycopg2.connect(database='sony', password='') - self.assertEqual(self.args[0], "dbname=sony password=''") + self.assertDsnEqual(self.args[0], "dbname=sony password=''") def test_escape(self): psycopg2.connect(database='hello world') @@ -131,13 +142,12 @@ class ConnectTestCase(unittest.TestCase): psycopg2.connect(database=r"\every thing'") self.assertEqual(self.args[0], r"dbname='\\every thing\''") - def test_no_kwargs_swallow(self): - self.assertRaises(TypeError, - psycopg2.connect, 'dbname=foo', database='foo') - self.assertRaises(TypeError, - psycopg2.connect, 'dbname=foo', user='postgres') - self.assertRaises(TypeError, - psycopg2.connect, 'dbname=foo', no_such_param='meh') + def test_params_merging(self): + psycopg2.connect('dbname=foo', database='bar') + self.assertEqual(self.args[0], 'dbname=bar') + + psycopg2.connect('dbname=foo', user='postgres') + self.assertDsnEqual(self.args[0], 'dbname=foo user=postgres') class ExceptionsTestCase(ConnectingTestCase): @@ -203,7 +213,8 @@ class ExceptionsTestCase(ConnectingTestCase): self.assertEqual(diag.sqlstate, '42P01') del diag - gc.collect(); gc.collect() + gc.collect() + gc.collect() assert(w() is None) @skip_copy_if_green @@ -325,7 +336,7 @@ class TestVersionDiscovery(unittest.TestCase): self.assertTrue(type(psycopg2.__libpq_version__) is int) try: self.assertTrue(type(psycopg2.extensions.libpq_version()) is int) - except NotSupportedError: + except psycopg2.NotSupportedError: self.assertTrue(psycopg2.__libpq_version__ < 90100) diff --git a/tests/test_notify.py b/tests/test_notify.py index fc6224d7..1a0ac457 100755 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -67,8 +67,8 @@ curs.execute("NOTIFY " %(name)r %(payload)r) curs.close() conn.close() """ % { - 'module': psycopg2.__name__, - 'dsn': dsn, 'sec': sec, 'name': name, 'payload': payload}) + 'module': psycopg2.__name__, + 'dsn': dsn, 'sec': sec, 'name': name, 'payload': payload}) return Popen([sys.executable, '-c', script_to_py3(script)], stdout=PIPE) @@ -79,7 +79,7 @@ conn.close() proc = self.notify('foo', 1) t0 = time.time() - ready = select.select([self.conn], [], [], 5) + select.select([self.conn], [], [], 5) t1 = time.time() self.assert_(0.99 < t1 - t0 < 4, t1 - t0) @@ -107,7 +107,7 @@ conn.close() names = dict.fromkeys(['foo', 'bar', 'baz']) for (pid, name) in self.conn.notifies: self.assertEqual(pids[name], pid) - names.pop(name) # raise if name found twice + names.pop(name) # raise if name found twice def test_notifies_received_on_execute(self): self.autocommit(self.conn) @@ -217,6 +217,6 @@ conn.close() def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) + if __name__ == "__main__": unittest.main() - diff --git a/tests/test_psycopg2_dbapi20.py b/tests/test_psycopg2_dbapi20.py index 28ea6690..c780d506 100755 --- a/tests/test_psycopg2_dbapi20.py +++ b/tests/test_psycopg2_dbapi20.py @@ -30,12 +30,14 @@ import psycopg2 from testconfig import dsn + class Psycopg2Tests(dbapi20.DatabaseAPI20Test): driver = psycopg2 connect_args = () connect_kw_args = {'dsn': dsn} lower_func = 'lower' # For stored procedure test + def test_callproc(self): # Until DBAPI 2.0 compliance, callproc should return None or it's just # misleading. Therefore, we will skip the return value test for diff --git a/tests/test_quote.py b/tests/test_quote.py index 6e945624..72c9c1e4 100755 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -23,11 +23,12 @@ # License for more details. import sys -from testutils import unittest, ConnectingTestCase, skip_before_libpq +import testutils +from testutils import unittest, ConnectingTestCase import psycopg2 import psycopg2.extensions -from psycopg2.extensions import b + class QuotingTestCase(ConnectingTestCase): r"""Checks the correct quoting of strings and binary objects. @@ -51,7 +52,7 @@ class QuotingTestCase(ConnectingTestCase): data = """some data with \t chars to escape into, 'quotes' and \\ a backslash too. """ - data += "".join(map(chr, range(1,127))) + data += "".join(map(chr, range(1, 127))) curs = self.conn.cursor() curs.execute("SELECT %s;", (data,)) @@ -60,10 +61,22 @@ class QuotingTestCase(ConnectingTestCase): self.assertEqual(res, data) self.assert_(not self.conn.notices) + def test_string_null_terminator(self): + curs = self.conn.cursor() + data = 'abcd\x01\x00cdefg' + + try: + curs.execute("SELECT %s", (data,)) + except ValueError as e: + self.assertEquals(str(e), + 'A string literal cannot contain NUL (0x00) characters.') + else: + self.fail("ValueError not raised") + def test_binary(self): - data = b("""some data with \000\013 binary + data = b"""some data with \000\013 binary stuff into, 'quotes' and \\ a backslash too. - """) + """ if sys.version_info[0] < 3: data += "".join(map(chr, range(256))) else: @@ -76,7 +89,7 @@ class QuotingTestCase(ConnectingTestCase): else: res = curs.fetchone()[0].tobytes() - if res[0] in (b('x'), ord(b('x'))) and self.conn.server_version >= 90000: + if res[0] in (b'x', ord(b'x')) and self.conn.server_version >= 90000: return self.skipTest( "bytea broken with server >= 9.0, libpq < 9") @@ -90,13 +103,13 @@ class QuotingTestCase(ConnectingTestCase): if server_encoding != "UTF8": return self.skipTest( "Unicode test skipped since server encoding is %s" - % server_encoding) + % server_encoding) data = u"""some data with \t chars to escape into, 'quotes', \u20ac euro sign and \\ a backslash too. """ - data += u"".join(map(unichr, [ u for u in range(1,65536) - if not 0xD800 <= u <= 0xDFFF ])) # surrogate area + data += u"".join(map(unichr, [u for u in range(1, 65536) + if not 0xD800 <= u <= 0xDFFF])) # surrogate area self.conn.set_client_encoding('UNICODE') psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, self.conn) @@ -156,7 +169,7 @@ class QuotingTestCase(ConnectingTestCase): class TestQuotedString(ConnectingTestCase): - def test_encoding(self): + def test_encoding_from_conn(self): q = psycopg2.extensions.QuotedString('hi') self.assertEqual(q.encoding, 'latin1') @@ -166,13 +179,13 @@ class TestQuotedString(ConnectingTestCase): class TestQuotedIdentifier(ConnectingTestCase): - @skip_before_libpq(9, 0) + @testutils.skip_before_libpq(9, 0) def test_identifier(self): from psycopg2.extensions import quote_ident self.assertEqual(quote_ident('blah-blah', self.conn), '"blah-blah"') self.assertEqual(quote_ident('quote"inside', self.conn), '"quote""inside"') - @skip_before_libpq(9, 0) + @testutils.skip_before_libpq(9, 0) def test_unicode_ident(self): from psycopg2.extensions import quote_ident snowman = u"\u2603" @@ -183,9 +196,59 @@ class TestQuotedIdentifier(ConnectingTestCase): self.assertEqual(quote_ident(snowman, self.conn), quoted) +class TestStringAdapter(ConnectingTestCase): + def test_encoding_default(self): + from psycopg2.extensions import adapt + a = adapt("hello") + self.assertEqual(a.encoding, 'latin1') + self.assertEqual(a.getquoted(), b"'hello'") + + # NOTE: we can't really test an encoding different from utf8, because + # when encoding without connection the libpq will use parameters from + # a previous one, so what would happens depends jn the tests run order. + # egrave = u'\xe8' + # self.assertEqual(adapt(egrave).getquoted(), "'\xe8'") + + def test_encoding_error(self): + from psycopg2.extensions import adapt + snowman = u"\u2603" + a = adapt(snowman) + self.assertRaises(UnicodeEncodeError, a.getquoted) + + def test_set_encoding(self): + # Note: this works-ish mostly in case when the standard db connection + # we test with is utf8, otherwise the encoding chosen by PQescapeString + # may give bad results. + from psycopg2.extensions import adapt + snowman = u"\u2603" + a = adapt(snowman) + a.encoding = 'utf8' + self.assertEqual(a.encoding, 'utf8') + self.assertEqual(a.getquoted(), b"'\xe2\x98\x83'") + + def test_connection_wins_anyway(self): + from psycopg2.extensions import adapt + snowman = u"\u2603" + a = adapt(snowman) + a.encoding = 'latin9' + + self.conn.set_client_encoding('utf8') + a.prepare(self.conn) + + self.assertEqual(a.encoding, 'utf_8') + self.assertEqual(a.getquoted(), b"'\xe2\x98\x83'") + + @testutils.skip_before_python(3) + def test_adapt_bytes(self): + snowman = u"\u2603" + self.conn.set_client_encoding('utf8') + a = psycopg2.extensions.QuotedString(snowman.encode('utf8')) + a.prepare(self.conn) + self.assertEqual(a.getquoted(), b"'\xe2\x98\x83'") + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) if __name__ == "__main__": unittest.main() - diff --git a/tests/test_replication.py b/tests/test_replication.py new file mode 100755 index 00000000..79d1295d --- /dev/null +++ b/tests/test_replication.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python + +# test_replication.py - unit test for replication protocol +# +# Copyright (C) 2015 Daniele Varrazzo +# +# psycopg2 is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# In addition, as a special exception, the copyright holders give +# permission to link this program with the OpenSSL library (or with +# modified versions of OpenSSL that use the same license as OpenSSL), +# and distribute linked combinations including the two. +# +# You must obey the GNU Lesser General Public License in all respects for +# all of the code used other than OpenSSL. +# +# psycopg2 is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +import psycopg2 +from psycopg2.extras import ( + PhysicalReplicationConnection, LogicalReplicationConnection, StopReplication) + +import testconfig +from testutils import unittest, ConnectingTestCase +from testutils import skip_before_postgres, skip_if_green + +skip_repl_if_green = skip_if_green("replication not supported in green mode") + + +class ReplicationTestCase(ConnectingTestCase): + def setUp(self): + super(ReplicationTestCase, self).setUp() + self.slot = testconfig.repl_slot + self._slots = [] + + def tearDown(self): + # first close all connections, as they might keep the slot(s) active + super(ReplicationTestCase, self).tearDown() + + import time + time.sleep(0.025) # sometimes the slot is still active, wait a little + + if self._slots: + kill_conn = self.connect() + if kill_conn: + kill_cur = kill_conn.cursor() + for slot in self._slots: + kill_cur.execute("SELECT pg_drop_replication_slot(%s)", (slot,)) + kill_conn.commit() + kill_conn.close() + + def create_replication_slot(self, cur, slot_name=testconfig.repl_slot, **kwargs): + cur.create_replication_slot(slot_name, **kwargs) + self._slots.append(slot_name) + + def drop_replication_slot(self, cur, slot_name=testconfig.repl_slot): + cur.drop_replication_slot(slot_name) + self._slots.remove(slot_name) + + # generate some events for our replication stream + def make_replication_events(self): + conn = self.connect() + if conn is None: + return + cur = conn.cursor() + + try: + cur.execute("DROP TABLE dummy1") + except psycopg2.ProgrammingError: + conn.rollback() + cur.execute( + "CREATE TABLE dummy1 AS SELECT * FROM generate_series(1, 5) AS id") + conn.commit() + + +class ReplicationTest(ReplicationTestCase): + @skip_before_postgres(9, 0) + def test_physical_replication_connection(self): + conn = self.repl_connect(connection_factory=PhysicalReplicationConnection) + if conn is None: + return + cur = conn.cursor() + cur.execute("IDENTIFY_SYSTEM") + cur.fetchall() + + @skip_before_postgres(9, 0) + def test_datestyle(self): + if testconfig.repl_dsn is None: + return self.skipTest("replication tests disabled by default") + + conn = self.repl_connect( + dsn=testconfig.repl_dsn, options='-cdatestyle=german', + connection_factory=PhysicalReplicationConnection) + if conn is None: + return + cur = conn.cursor() + cur.execute("IDENTIFY_SYSTEM") + cur.fetchall() + + @skip_before_postgres(9, 4) + def test_logical_replication_connection(self): + conn = self.repl_connect(connection_factory=LogicalReplicationConnection) + if conn is None: + return + cur = conn.cursor() + cur.execute("IDENTIFY_SYSTEM") + cur.fetchall() + + @skip_before_postgres(9, 4) # slots require 9.4 + def test_create_replication_slot(self): + conn = self.repl_connect(connection_factory=PhysicalReplicationConnection) + if conn is None: + return + cur = conn.cursor() + + self.create_replication_slot(cur) + self.assertRaises( + psycopg2.ProgrammingError, self.create_replication_slot, cur) + + @skip_before_postgres(9, 4) # slots require 9.4 + @skip_repl_if_green + def test_start_on_missing_replication_slot(self): + conn = self.repl_connect(connection_factory=PhysicalReplicationConnection) + if conn is None: + return + cur = conn.cursor() + + self.assertRaises(psycopg2.ProgrammingError, + cur.start_replication, self.slot) + + self.create_replication_slot(cur) + cur.start_replication(self.slot) + + @skip_before_postgres(9, 4) # slots require 9.4 + @skip_repl_if_green + def test_start_and_recover_from_error(self): + conn = self.repl_connect(connection_factory=LogicalReplicationConnection) + if conn is None: + return + cur = conn.cursor() + + self.create_replication_slot(cur, output_plugin='test_decoding') + + # try with invalid options + cur.start_replication( + slot_name=self.slot, options={'invalid_param': 'value'}) + + def consume(msg): + pass + # we don't see the error from the server before we try to read the data + self.assertRaises(psycopg2.DataError, cur.consume_stream, consume) + + # try with correct command + cur.start_replication(slot_name=self.slot) + + @skip_before_postgres(9, 4) # slots require 9.4 + @skip_repl_if_green + def test_stop_replication(self): + conn = self.repl_connect(connection_factory=LogicalReplicationConnection) + if conn is None: + return + cur = conn.cursor() + + self.create_replication_slot(cur, output_plugin='test_decoding') + + self.make_replication_events() + + cur.start_replication(self.slot) + + def consume(msg): + raise StopReplication() + self.assertRaises(StopReplication, cur.consume_stream, consume) + + +class AsyncReplicationTest(ReplicationTestCase): + @skip_before_postgres(9, 4) # slots require 9.4 + @skip_repl_if_green + def test_async_replication(self): + conn = self.repl_connect( + connection_factory=LogicalReplicationConnection, async=1) + if conn is None: + return + + cur = conn.cursor() + + self.create_replication_slot(cur, output_plugin='test_decoding') + self.wait(cur) + + cur.start_replication(self.slot) + self.wait(cur) + + self.make_replication_events() + + self.msg_count = 0 + + def consume(msg): + # just check the methods + "%s: %s" % (cur.io_timestamp, repr(msg)) + + self.msg_count += 1 + if self.msg_count > 3: + cur.send_feedback(reply=True) + raise StopReplication() + + cur.send_feedback(flush_lsn=msg.data_start) + + # cannot be used in asynchronous mode + self.assertRaises(psycopg2.ProgrammingError, cur.consume_stream, consume) + + def process_stream(): + from select import select + while True: + msg = cur.read_message() + if msg: + consume(msg) + else: + select([cur], [], []) + self.assertRaises(StopReplication, process_stream) + + +def test_suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 724d0d80..2dc44ec5 100755 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -29,6 +29,7 @@ import psycopg2 from psycopg2.extensions import ( ISOLATION_LEVEL_SERIALIZABLE, STATUS_BEGIN, STATUS_READY) + class TransactionTests(ConnectingTestCase): def setUp(self): @@ -147,6 +148,7 @@ class DeadlockSerializationTests(ConnectingTestCase): self.thread1_error = exc step1.set() conn.close() + def task2(): try: conn = self.connect() @@ -174,7 +176,7 @@ class DeadlockSerializationTests(ConnectingTestCase): self.assertFalse(self.thread1_error and self.thread2_error) error = self.thread1_error or self.thread2_error self.assertTrue(isinstance( - error, psycopg2.extensions.TransactionRollbackError)) + error, psycopg2.extensions.TransactionRollbackError)) def test_serialisation_failure(self): self.thread1_error = self.thread2_error = None @@ -195,6 +197,7 @@ class DeadlockSerializationTests(ConnectingTestCase): self.thread1_error = exc step1.set() conn.close() + def task2(): try: conn = self.connect() @@ -221,7 +224,7 @@ class DeadlockSerializationTests(ConnectingTestCase): self.assertFalse(self.thread1_error and self.thread2_error) error = self.thread1_error or self.thread2_error self.assertTrue(isinstance( - error, psycopg2.extensions.TransactionRollbackError)) + error, psycopg2.extensions.TransactionRollbackError)) class QueryCancellationTests(ConnectingTestCase): diff --git a/tests/test_types_basic.py b/tests/test_types_basic.py index 199dc1b6..bee23d53 100755 --- a/tests/test_types_basic.py +++ b/tests/test_types_basic.py @@ -30,7 +30,6 @@ import testutils from testutils import unittest, ConnectingTestCase, decorate_all_tests import psycopg2 -from psycopg2.extensions import b class TypesBasicTests(ConnectingTestCase): @@ -69,13 +68,16 @@ class TypesBasicTests(ConnectingTestCase): "wrong decimal quoting: " + str(s)) s = self.execute("SELECT %s AS foo", (decimal.Decimal("NaN"),)) self.failUnless(str(s) == "NaN", "wrong decimal quoting: " + str(s)) - self.failUnless(type(s) == decimal.Decimal, "wrong decimal conversion: " + repr(s)) + self.failUnless(type(s) == decimal.Decimal, + "wrong decimal conversion: " + repr(s)) s = self.execute("SELECT %s AS foo", (decimal.Decimal("infinity"),)) self.failUnless(str(s) == "NaN", "wrong decimal quoting: " + str(s)) - self.failUnless(type(s) == decimal.Decimal, "wrong decimal conversion: " + repr(s)) + self.failUnless(type(s) == decimal.Decimal, + "wrong decimal conversion: " + repr(s)) s = self.execute("SELECT %s AS foo", (decimal.Decimal("-infinity"),)) self.failUnless(str(s) == "NaN", "wrong decimal quoting: " + str(s)) - self.failUnless(type(s) == decimal.Decimal, "wrong decimal conversion: " + repr(s)) + self.failUnless(type(s) == decimal.Decimal, + "wrong decimal conversion: " + repr(s)) def testFloatNan(self): try: @@ -95,11 +97,11 @@ class TypesBasicTests(ConnectingTestCase): except ValueError: return self.skipTest("inf not available on this platform") s = self.execute("SELECT %s AS foo", (float("inf"),)) - self.failUnless(str(s) == "inf", "wrong float quoting: " + str(s)) + self.failUnless(str(s) == "inf", "wrong float quoting: " + str(s)) self.failUnless(type(s) == float, "wrong float conversion: " + repr(s)) s = self.execute("SELECT %s AS foo", (float("-inf"),)) - self.failUnless(str(s) == "-inf", "wrong float quoting: " + str(s)) + self.failUnless(str(s) == "-inf", "wrong float quoting: " + str(s)) def testBinary(self): if sys.version_info[0] < 3: @@ -142,8 +144,8 @@ class TypesBasicTests(ConnectingTestCase): self.assertEqual(s, buf2.tobytes()) def testArray(self): - s = self.execute("SELECT %s AS foo", ([[1,2],[3,4]],)) - self.failUnlessEqual(s, [[1,2],[3,4]]) + s = self.execute("SELECT %s AS foo", ([[1, 2], [3, 4]],)) + self.failUnlessEqual(s, [[1, 2], [3, 4]]) s = self.execute("SELECT %s AS foo", (['one', 'two', 'three'],)) self.failUnlessEqual(s, ['one', 'two', 'three']) @@ -151,9 +153,12 @@ class TypesBasicTests(ConnectingTestCase): # ticket #42 import datetime curs = self.conn.cursor() - curs.execute("create table array_test (id integer, col timestamp without time zone[])") + curs.execute( + "create table array_test " + "(id integer, col timestamp without time zone[])") - curs.execute("insert into array_test values (%s, %s)", (1, [datetime.date(2011,2,14)])) + curs.execute("insert into array_test values (%s, %s)", + (1, [datetime.date(2011, 2, 14)])) curs.execute("select col from array_test where id = 1") self.assertEqual(curs.fetchone()[0], [datetime.datetime(2011, 2, 14, 0, 0)]) @@ -190,8 +195,9 @@ class TypesBasicTests(ConnectingTestCase): ss = ['', '{', '{}}', '{' * 20 + '}' * 20] for s in ss: self.assertRaises(psycopg2.DataError, - psycopg2.extensions.STRINGARRAY, b(s), curs) + psycopg2.extensions.STRINGARRAY, s.encode('utf8'), curs) + @testutils.skip_before_postgres(8, 2) def testArrayOfNulls(self): curs = self.conn.cursor() curs.execute(""" @@ -208,9 +214,9 @@ class TypesBasicTests(ConnectingTestCase): curs.execute("insert into na (texta) values (%s)", ([None],)) curs.execute("insert into na (texta) values (%s)", (['a', None],)) curs.execute("insert into na (texta) values (%s)", ([None, None],)) - curs.execute("insert into na (inta) values (%s)", ([None],)) - curs.execute("insert into na (inta) values (%s)", ([42, None],)) - curs.execute("insert into na (inta) values (%s)", ([None, None],)) + curs.execute("insert into na (inta) values (%s)", ([None],)) + curs.execute("insert into na (inta) values (%s)", ([42, None],)) + curs.execute("insert into na (inta) values (%s)", ([None, None],)) curs.execute("insert into na (boola) values (%s)", ([None],)) curs.execute("insert into na (boola) values (%s)", ([True, None],)) curs.execute("insert into na (boola) values (%s)", ([None, None],)) @@ -220,7 +226,7 @@ class TypesBasicTests(ConnectingTestCase): curs.execute("insert into na (textaa) values (%s)", ([['a', None]],)) # curs.execute("insert into na (textaa) values (%s)", ([[None, None]],)) # curs.execute("insert into na (intaa) values (%s)", ([[None]],)) - curs.execute("insert into na (intaa) values (%s)", ([[42, None]],)) + curs.execute("insert into na (intaa) values (%s)", ([[42, None]],)) # curs.execute("insert into na (intaa) values (%s)", ([[None, None]],)) # curs.execute("insert into na (boolaa) values (%s)", ([[None]],)) curs.execute("insert into na (boolaa) values (%s)", ([[True, None]],)) @@ -308,9 +314,9 @@ class TypesBasicTests(ConnectingTestCase): def testByteaHexCheckFalsePositive(self): # the check \x -> x to detect bad bytea decode # may be fooled if the first char is really an 'x' - o1 = psycopg2.Binary(b('x')) + o1 = psycopg2.Binary(b'x') o2 = self.execute("SELECT %s::bytea AS foo", (o1,)) - self.assertEqual(b('x'), o2[0]) + self.assertEqual(b'x', o2[0]) def testNegNumber(self): d1 = self.execute("select -%s;", (decimal.Decimal('-1.0'),)) @@ -323,30 +329,43 @@ class TypesBasicTests(ConnectingTestCase): self.assertEqual(1, l1) def testGenericArray(self): - a = self.execute("select '{1,2,3}'::int4[]") - self.assertEqual(a, [1,2,3]) - a = self.execute("select array['a','b','''']::text[]") - self.assertEqual(a, ['a','b',"'"]) + a = self.execute("select '{1, 2, 3}'::int4[]") + self.assertEqual(a, [1, 2, 3]) + a = self.execute("select array['a', 'b', '''']::text[]") + self.assertEqual(a, ['a', 'b', "'"]) @testutils.skip_before_postgres(8, 2) def testGenericArrayNull(self): def caster(s, cur): - if s is None: return "nada" + if s is None: + return "nada" return int(s) * 2 base = psycopg2.extensions.new_type((23,), "INT4", caster) array = psycopg2.extensions.new_array_type((1007,), "INT4ARRAY", base) psycopg2.extensions.register_type(array, self.conn) - a = self.execute("select '{1,2,3}'::int4[]") - self.assertEqual(a, [2,4,6]) - a = self.execute("select '{1,2,NULL}'::int4[]") - self.assertEqual(a, [2,4,'nada']) + a = self.execute("select '{1, 2, 3}'::int4[]") + self.assertEqual(a, [2, 4, 6]) + a = self.execute("select '{1, 2, NULL}'::int4[]") + self.assertEqual(a, [2, 4, 'nada']) + + @testutils.skip_before_postgres(8, 2) + def testNetworkArray(self): + # we don't know these types, but we know their arrays + a = self.execute("select '{192.168.0.1/24}'::inet[]") + self.assertEqual(a, ['192.168.0.1/24']) + a = self.execute("select '{192.168.0.0/24}'::cidr[]") + self.assertEqual(a, ['192.168.0.0/24']) + a = self.execute("select '{10:20:30:40:50:60}'::macaddr[]") + self.assertEqual(a, ['10:20:30:40:50:60']) class AdaptSubclassTest(unittest.TestCase): def test_adapt_subtype(self): from psycopg2.extensions import adapt - class Sub(str): pass + + class Sub(str): + pass s1 = "hel'lo" s2 = Sub(s1) self.assertEqual(adapt(s1).getquoted(), adapt(s2).getquoted()) @@ -354,44 +373,54 @@ class AdaptSubclassTest(unittest.TestCase): def test_adapt_most_specific(self): from psycopg2.extensions import adapt, register_adapter, AsIs - class A(object): pass - class B(A): pass - class C(B): pass + class A(object): + pass + + class B(A): + pass + + class C(B): + pass register_adapter(A, lambda a: AsIs("a")) register_adapter(B, lambda b: AsIs("b")) try: - self.assertEqual(b('b'), adapt(C()).getquoted()) + self.assertEqual(b'b', adapt(C()).getquoted()) finally: - del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] - del psycopg2.extensions.adapters[B, psycopg2.extensions.ISQLQuote] + del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] + del psycopg2.extensions.adapters[B, psycopg2.extensions.ISQLQuote] @testutils.skip_from_python(3) def test_no_mro_no_joy(self): from psycopg2.extensions import adapt, register_adapter, AsIs - class A: pass - class B(A): pass + class A: + pass + + class B(A): + pass register_adapter(A, lambda a: AsIs("a")) try: self.assertRaises(psycopg2.ProgrammingError, adapt, B()) finally: - del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] - + del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] @testutils.skip_before_python(3) def test_adapt_subtype_3(self): from psycopg2.extensions import adapt, register_adapter, AsIs - class A: pass - class B(A): pass + class A: + pass + + class B(A): + pass register_adapter(A, lambda a: AsIs("a")) try: - self.assertEqual(b("a"), adapt(B()).getquoted()) + self.assertEqual(b"a", adapt(B()).getquoted()) finally: - del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] + del psycopg2.extensions.adapters[A, psycopg2.extensions.ISQLQuote] class ByteaParserTest(unittest.TestCase): @@ -434,19 +463,20 @@ class ByteaParserTest(unittest.TestCase): self.assertEqual(rv, None) def test_blank(self): - rv = self.cast(b('')) - self.assertEqual(rv, b('')) + rv = self.cast(b'') + self.assertEqual(rv, b'') def test_blank_hex(self): # Reported as problematic in ticket #48 - rv = self.cast(b('\\x')) - self.assertEqual(rv, b('')) + rv = self.cast(b'\\x') + self.assertEqual(rv, b'') def test_full_hex(self, upper=False): buf = ''.join(("%02x" % i) for i in range(256)) - if upper: buf = buf.upper() + if upper: + buf = buf.upper() buf = '\\x' + buf - rv = self.cast(b(buf)) + rv = self.cast(buf.encode('utf8')) if sys.version_info[0] < 3: self.assertEqual(rv, ''.join(map(chr, range(256)))) else: @@ -457,7 +487,7 @@ class ByteaParserTest(unittest.TestCase): def test_full_escaped_octal(self): buf = ''.join(("\\%03o" % i) for i in range(256)) - rv = self.cast(b(buf)) + rv = self.cast(buf.encode('utf8')) if sys.version_info[0] < 3: self.assertEqual(rv, ''.join(map(chr, range(256)))) else: @@ -469,7 +499,7 @@ class ByteaParserTest(unittest.TestCase): buf += string.ascii_letters buf += ''.join('\\' + c for c in string.ascii_letters) buf += '\\\\' - rv = self.cast(b(buf)) + rv = self.cast(buf.encode('utf8')) if sys.version_info[0] < 3: tgt = ''.join(map(chr, range(32))) \ + string.ascii_letters * 2 + '\\' @@ -479,6 +509,7 @@ class ByteaParserTest(unittest.TestCase): self.assertEqual(rv, tgt) + def skip_if_cant_cast(f): @wraps(f) def skip_if_cant_cast_(self, *args, **kwargs): @@ -498,4 +529,3 @@ def test_suite(): if __name__ == "__main__": unittest.main() - diff --git a/tests/test_types_extras.py b/tests/test_types_extras.py index b81cecab..8e615616 100755 --- a/tests/test_types_extras.py +++ b/tests/test_types_extras.py @@ -20,6 +20,7 @@ import sys from decimal import Decimal from datetime import date, datetime from functools import wraps +from pickle import dumps, loads from testutils import unittest, skip_if_no_uuid, skip_before_postgres from testutils import ConnectingTestCase, decorate_all_tests @@ -28,14 +29,14 @@ from testutils import py3_raises_typeerror import psycopg2 import psycopg2.extras import psycopg2.extensions as ext -from psycopg2.extensions import b def filter_scs(conn, s): if conn.get_parameter_status("standard_conforming_strings") == 'off': return s else: - return s.replace(b("E'"), b("'")) + return s.replace(b"E'", b"'") + class TypesExtrasTests(ConnectingTestCase): """Test that all type conversions are working.""" @@ -60,7 +61,8 @@ class TypesExtrasTests(ConnectingTestCase): def testUUIDARRAY(self): import uuid psycopg2.extras.register_uuid() - u = [uuid.UUID('9c6d5a77-7256-457e-9461-347b4358e350'), uuid.UUID('9c6d5a77-7256-457e-9461-347b4358e352')] + u = [uuid.UUID('9c6d5a77-7256-457e-9461-347b4358e350'), + uuid.UUID('9c6d5a77-7256-457e-9461-347b4358e352')] s = self.execute("SELECT %s AS foo", (u,)) self.failUnless(u == s) # array with a NULL element @@ -98,7 +100,7 @@ class TypesExtrasTests(ConnectingTestCase): a = psycopg2.extensions.adapt(i) a.prepare(self.conn) self.assertEqual( - filter_scs(self.conn, b("E'192.168.1.0/24'::inet")), + filter_scs(self.conn, b"E'192.168.1.0/24'::inet"), a.getquoted()) # adapts ok with unicode too @@ -106,11 +108,12 @@ class TypesExtrasTests(ConnectingTestCase): a = psycopg2.extensions.adapt(i) a.prepare(self.conn) self.assertEqual( - filter_scs(self.conn, b("E'192.168.1.0/24'::inet")), + filter_scs(self.conn, b"E'192.168.1.0/24'::inet"), a.getquoted()) def test_adapt_fail(self): - class Foo(object): pass + class Foo(object): + pass self.assertRaises(psycopg2.ProgrammingError, psycopg2.extensions.adapt, Foo(), ext.ISQLQuote, None) try: @@ -130,6 +133,7 @@ def skip_if_no_hstore(f): return skip_if_no_hstore_ + class HstoreTestCase(ConnectingTestCase): def test_adapt_8(self): if self.conn.server_version >= 90000: @@ -145,17 +149,18 @@ class HstoreTestCase(ConnectingTestCase): a.prepare(self.conn) q = a.getquoted() - self.assert_(q.startswith(b("((")), q) - ii = q[1:-1].split(b("||")) + self.assert_(q.startswith(b"(("), q) + ii = q[1:-1].split(b"||") ii.sort() self.assertEqual(len(ii), len(o)) - self.assertEqual(ii[0], filter_scs(self.conn, b("(E'a' => E'1')"))) - self.assertEqual(ii[1], filter_scs(self.conn, b("(E'b' => E'''')"))) - self.assertEqual(ii[2], filter_scs(self.conn, b("(E'c' => NULL)"))) + self.assertEqual(ii[0], filter_scs(self.conn, b"(E'a' => E'1')")) + self.assertEqual(ii[1], filter_scs(self.conn, b"(E'b' => E'''')")) + self.assertEqual(ii[2], filter_scs(self.conn, b"(E'c' => NULL)")) if 'd' in o: encc = u'\xe0'.encode(psycopg2.extensions.encodings[self.conn.encoding]) - self.assertEqual(ii[3], filter_scs(self.conn, b("(E'd' => E'") + encc + b("')"))) + self.assertEqual(ii[3], + filter_scs(self.conn, b"(E'd' => E'" + encc + b"')")) def test_adapt_9(self): if self.conn.server_version < 90000: @@ -171,11 +176,11 @@ class HstoreTestCase(ConnectingTestCase): a.prepare(self.conn) q = a.getquoted() - m = re.match(b(r'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)'), q) + m = re.match(br'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)', q) self.assert_(m, repr(q)) - kk = m.group(1).split(b(", ")) - vv = m.group(2).split(b(", ")) + kk = m.group(1).split(b", ") + vv = m.group(2).split(b", ") ii = zip(kk, vv) ii.sort() @@ -183,12 +188,12 @@ class HstoreTestCase(ConnectingTestCase): return tuple([filter_scs(self.conn, s) for s in args]) self.assertEqual(len(ii), len(o)) - self.assertEqual(ii[0], f(b("E'a'"), b("E'1'"))) - self.assertEqual(ii[1], f(b("E'b'"), b("E''''"))) - self.assertEqual(ii[2], f(b("E'c'"), b("NULL"))) + self.assertEqual(ii[0], f(b"E'a'", b"E'1'")) + self.assertEqual(ii[1], f(b"E'b'", b"E''''")) + self.assertEqual(ii[2], f(b"E'c'", b"NULL")) if 'd' in o: encc = u'\xe0'.encode(psycopg2.extensions.encodings[self.conn.encoding]) - self.assertEqual(ii[3], f(b("E'd'"), b("E'") + encc + b("'"))) + self.assertEqual(ii[3], f(b"E'd'", b"E'" + encc + b"'")) def test_parse(self): from psycopg2.extras import HstoreAdapter @@ -199,7 +204,7 @@ class HstoreTestCase(ConnectingTestCase): ok(None, None) ok('', {}) ok('"a"=>"1", "b"=>"2"', {'a': '1', 'b': '2'}) - ok('"a" => "1" ,"b" => "2"', {'a': '1', 'b': '2'}) + ok('"a" => "1" , "b" => "2"', {'a': '1', 'b': '2'}) ok('"a"=>NULL, "b"=>"2"', {'a': None, 'b': '2'}) ok(r'"a"=>"\"", "\""=>"2"', {'a': '"', '"': '2'}) ok('"a"=>"\'", "\'"=>"2"', {'a': "'", "'": '2'}) @@ -402,7 +407,9 @@ class HstoreTestCase(ConnectingTestCase): from psycopg2.extras import register_hstore register_hstore(None, globally=True, oid=oid, array_oid=aoid) try: - cur.execute("select null::hstore, ''::hstore, 'a => b'::hstore, '{a=>b}'::hstore[]") + cur.execute(""" + select null::hstore, ''::hstore, + 'a => b'::hstore, '{a=>b}'::hstore[]""") t = cur.fetchone() self.assert_(t[0] is None) self.assertEqual(t[1], {}) @@ -449,12 +456,13 @@ def skip_if_no_composite(f): return skip_if_no_composite_ + class AdaptTypeTestCase(ConnectingTestCase): @skip_if_no_composite def test_none_in_record(self): curs = self.conn.cursor() s = curs.mogrify("SELECT %s;", [(42, None)]) - self.assertEqual(b("SELECT (42, NULL);"), s) + self.assertEqual(b"SELECT (42, NULL);", s) curs.execute("SELECT %s;", [(42, None)]) d = curs.fetchone()[0] self.assertEqual("(42,)", d) @@ -463,8 +471,11 @@ class AdaptTypeTestCase(ConnectingTestCase): # the None adapter is not actually invoked in regular adaptation class WonkyAdapter(object): - def __init__(self, obj): pass - def getquoted(self): return "NOPE!" + def __init__(self, obj): + pass + + def getquoted(self): + return "NOPE!" curs = self.conn.cursor() @@ -474,13 +485,14 @@ class AdaptTypeTestCase(ConnectingTestCase): self.assertEqual(ext.adapt(None).getquoted(), "NOPE!") s = curs.mogrify("SELECT %s;", (None,)) - self.assertEqual(b("SELECT NULL;"), s) + self.assertEqual(b"SELECT NULL;", s) finally: ext.register_adapter(type(None), orig_adapter) def test_tokenization(self): from psycopg2.extras import CompositeCaster + def ok(s, v): self.assertEqual(CompositeCaster.tokenize(s), v) @@ -519,26 +531,26 @@ class AdaptTypeTestCase(ConnectingTestCase): self.assertEqual(t.oid, oid) self.assert_(issubclass(t.type, tuple)) self.assertEqual(t.attnames, ['anint', 'astring', 'adate']) - self.assertEqual(t.atttypes, [23,25,1082]) + self.assertEqual(t.atttypes, [23, 25, 1082]) curs = self.conn.cursor() - r = (10, 'hello', date(2011,1,2)) + r = (10, 'hello', date(2011, 1, 2)) curs.execute("select %s::type_isd;", (r,)) v = curs.fetchone()[0] self.assert_(isinstance(v, t.type)) self.assertEqual(v[0], 10) self.assertEqual(v[1], "hello") - self.assertEqual(v[2], date(2011,1,2)) + self.assertEqual(v[2], date(2011, 1, 2)) try: - from collections import namedtuple + from collections import namedtuple # noqa except ImportError: pass else: self.assert_(t.type is not tuple) self.assertEqual(v.anint, 10) self.assertEqual(v.astring, "hello") - self.assertEqual(v.adate, date(2011,1,2)) + self.assertEqual(v.adate, date(2011, 1, 2)) @skip_if_no_composite def test_empty_string(self): @@ -574,14 +586,14 @@ class AdaptTypeTestCase(ConnectingTestCase): psycopg2.extras.register_composite("type_r_ft", self.conn) curs = self.conn.cursor() - r = (0.25, (date(2011,1,2), (42, "hello"))) + r = (0.25, (date(2011, 1, 2), (42, "hello"))) curs.execute("select %s::type_r_ft;", (r,)) v = curs.fetchone()[0] self.assertEqual(r, v) try: - from collections import namedtuple + from collections import namedtuple # noqa except ImportError: pass else: @@ -595,7 +607,7 @@ class AdaptTypeTestCase(ConnectingTestCase): curs2 = self.conn.cursor() psycopg2.extras.register_composite("type_ii", curs1) curs1.execute("select (1,2)::type_ii") - self.assertEqual(curs1.fetchone()[0], (1,2)) + self.assertEqual(curs1.fetchone()[0], (1, 2)) curs2.execute("select (1,2)::type_ii") self.assertEqual(curs2.fetchone()[0], "(1,2)") @@ -610,7 +622,7 @@ class AdaptTypeTestCase(ConnectingTestCase): curs1 = conn1.cursor() curs2 = conn2.cursor() curs1.execute("select (1,2)::type_ii") - self.assertEqual(curs1.fetchone()[0], (1,2)) + self.assertEqual(curs1.fetchone()[0], (1, 2)) curs2.execute("select (1,2)::type_ii") self.assertEqual(curs2.fetchone()[0], "(1,2)") finally: @@ -629,9 +641,9 @@ class AdaptTypeTestCase(ConnectingTestCase): curs1 = conn1.cursor() curs2 = conn2.cursor() curs1.execute("select (1,2)::type_ii") - self.assertEqual(curs1.fetchone()[0], (1,2)) + self.assertEqual(curs1.fetchone()[0], (1, 2)) curs2.execute("select (1,2)::type_ii") - self.assertEqual(curs2.fetchone()[0], (1,2)) + self.assertEqual(curs2.fetchone()[0], (1, 2)) finally: # drop the registered typecasters to help the refcounting # script to return precise values. @@ -661,30 +673,30 @@ class AdaptTypeTestCase(ConnectingTestCase): "typens.typens_ii", self.conn) self.assertEqual(t.schema, 'typens') curs.execute("select (4,8)::typens.typens_ii") - self.assertEqual(curs.fetchone()[0], (4,8)) + self.assertEqual(curs.fetchone()[0], (4, 8)) @skip_if_no_composite @skip_before_postgres(8, 4) def test_composite_array(self): - oid = self._create_type("type_isd", + self._create_type("type_isd", [('anint', 'integer'), ('astring', 'text'), ('adate', 'date')]) t = psycopg2.extras.register_composite("type_isd", self.conn) curs = self.conn.cursor() - r1 = (10, 'hello', date(2011,1,2)) - r2 = (20, 'world', date(2011,1,3)) + r1 = (10, 'hello', date(2011, 1, 2)) + r2 = (20, 'world', date(2011, 1, 3)) curs.execute("select %s::type_isd[];", ([r1, r2],)) v = curs.fetchone()[0] self.assertEqual(len(v), 2) self.assert_(isinstance(v[0], t.type)) self.assertEqual(v[0][0], 10) self.assertEqual(v[0][1], "hello") - self.assertEqual(v[0][2], date(2011,1,2)) + self.assertEqual(v[0][2], date(2011, 1, 2)) self.assert_(isinstance(v[1], t.type)) self.assertEqual(v[1][0], 20) self.assertEqual(v[1][1], "world") - self.assertEqual(v[1][2], date(2011,1,3)) + self.assertEqual(v[1][2], date(2011, 1, 3)) @skip_if_no_composite def test_wrong_schema(self): @@ -752,7 +764,7 @@ class AdaptTypeTestCase(ConnectingTestCase): register_composite('type_ii', conn) curs = conn.cursor() curs.execute("select '(1,2)'::type_ii as x") - self.assertEqual(curs.fetchone()['x'], (1,2)) + self.assertEqual(curs.fetchone()['x'], (1, 2)) finally: conn.close() @@ -761,7 +773,7 @@ class AdaptTypeTestCase(ConnectingTestCase): curs = conn.cursor() register_composite('type_ii', conn) curs.execute("select '(1,2)'::type_ii as x") - self.assertEqual(curs.fetchone()['x'], (1,2)) + self.assertEqual(curs.fetchone()['x'], (1, 2)) finally: conn.close() @@ -782,13 +794,13 @@ class AdaptTypeTestCase(ConnectingTestCase): self.assertEqual(t.oid, oid) curs = self.conn.cursor() - r = (10, 'hello', date(2011,1,2)) + r = (10, 'hello', date(2011, 1, 2)) curs.execute("select %s::type_isd;", (r,)) v = curs.fetchone()[0] self.assert_(isinstance(v, dict)) self.assertEqual(v['anint'], 10) self.assertEqual(v['astring'], "hello") - self.assertEqual(v['adate'], date(2011,1,2)) + self.assertEqual(v['adate'], date(2011, 1, 2)) def _create_type(self, name, fields): curs = self.conn.cursor() @@ -825,6 +837,7 @@ def skip_if_json_module(f): return skip_if_json_module_ + def skip_if_no_json_module(f): """Skip a test if no Python json module is available""" @wraps(f) @@ -836,6 +849,7 @@ def skip_if_no_json_module(f): return skip_if_no_json_module_ + def skip_if_no_json_type(f): """Skip a test if PostgreSQL json type is not available""" @wraps(f) @@ -849,6 +863,7 @@ def skip_if_no_json_type(f): return skip_if_no_json_type_ + class JsonTestCase(ConnectingTestCase): @skip_if_json_module def test_module_not_available(self): @@ -858,6 +873,7 @@ class JsonTestCase(ConnectingTestCase): @skip_if_json_module def test_customizable_with_module_not_available(self): from psycopg2.extras import Json + class MyJson(Json): def dumps(self, obj): assert obj is None @@ -870,7 +886,7 @@ class JsonTestCase(ConnectingTestCase): from psycopg2.extras import json, Json objs = [None, "te'xt", 123, 123.45, - u'\xe0\u20ac', ['a', 100], {'a': 100} ] + u'\xe0\u20ac', ['a', 100], {'a': 100}] curs = self.conn.cursor() for obj in enumerate(objs): @@ -889,9 +905,11 @@ class JsonTestCase(ConnectingTestCase): curs = self.conn.cursor() obj = Decimal('123.45') - dumps = lambda obj: json.dumps(obj, cls=DecimalEncoder) + + def dumps(obj): + return json.dumps(obj, cls=DecimalEncoder) self.assertEqual(curs.mogrify("%s", (Json(obj, dumps=dumps),)), - b("'123.45'")) + b"'123.45'") @skip_if_no_json_module def test_adapt_subclass(self): @@ -909,8 +927,7 @@ class JsonTestCase(ConnectingTestCase): curs = self.conn.cursor() obj = Decimal('123.45') - self.assertEqual(curs.mogrify("%s", (MyJson(obj),)), - b("'123.45'")) + self.assertEqual(curs.mogrify("%s", (MyJson(obj),)), b"'123.45'") @skip_if_no_json_module def test_register_on_dict(self): @@ -920,11 +937,9 @@ class JsonTestCase(ConnectingTestCase): try: curs = self.conn.cursor() obj = {'a': 123} - self.assertEqual(curs.mogrify("%s", (obj,)), - b("""'{"a": 123}'""")) + self.assertEqual(curs.mogrify("%s", (obj,)), b"""'{"a": 123}'""") finally: - del psycopg2.extensions.adapters[dict, ext.ISQLQuote] - + del psycopg2.extensions.adapters[dict, ext.ISQLQuote] def test_type_not_available(self): curs = self.conn.cursor() @@ -984,7 +999,9 @@ class JsonTestCase(ConnectingTestCase): @skip_if_no_json_type def test_loads(self): json = psycopg2.extras.json - loads = lambda x: json.loads(x, parse_float=Decimal) + + def loads(s): + return json.loads(s, parse_float=Decimal) psycopg2.extras.register_json(self.conn, loads=loads) curs = self.conn.cursor() curs.execute("""select '{"a": 100.0, "b": null}'::json""") @@ -1000,7 +1017,9 @@ class JsonTestCase(ConnectingTestCase): old = psycopg2.extensions.string_types.get(114) olda = psycopg2.extensions.string_types.get(199) - loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) + + def loads(s): + return psycopg2.extras.json.loads(s, parse_float=Decimal) try: new, newa = psycopg2.extras.register_json( loads=loads, oid=oid, array_oid=array_oid) @@ -1022,7 +1041,8 @@ class JsonTestCase(ConnectingTestCase): def test_register_default(self): curs = self.conn.cursor() - loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) + def loads(s): + return psycopg2.extras.json.loads(s, parse_float=Decimal) psycopg2.extras.register_default_json(curs, loads=loads) curs.execute("""select '{"a": 100.0, "b": null}'::json""") @@ -1072,6 +1092,7 @@ class JsonTestCase(ConnectingTestCase): def skip_if_no_jsonb_type(f): return skip_before_postgres(9, 4)(f) + class JsonbTestCase(ConnectingTestCase): @staticmethod def myloads(s): @@ -1120,7 +1141,10 @@ class JsonbTestCase(ConnectingTestCase): def test_loads(self): json = psycopg2.extras.json - loads = lambda x: json.loads(x, parse_float=Decimal) + + def loads(s): + return json.loads(s, parse_float=Decimal) + psycopg2.extras.register_json(self.conn, loads=loads, name='jsonb') curs = self.conn.cursor() curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") @@ -1136,7 +1160,9 @@ class JsonbTestCase(ConnectingTestCase): def test_register_default(self): curs = self.conn.cursor() - loads = lambda x: psycopg2.extras.json.loads(x, parse_float=Decimal) + def loads(s): + return psycopg2.extras.json.loads(s, parse_float=Decimal) + psycopg2.extras.register_default_jsonb(curs, loads=loads) curs.execute("""select '{"a": 100.0, "b": null}'::jsonb""") @@ -1202,7 +1228,7 @@ class RangeTestCase(unittest.TestCase): ('[)', True, False), ('(]', False, True), ('()', False, False), - ('[]', True, True),]: + ('[]', True, True)]: r = Range(10, 20, bounds) self.assertEqual(r.lower, 10) self.assertEqual(r.upper, 20) @@ -1296,11 +1322,11 @@ class RangeTestCase(unittest.TestCase): self.assert_(not Range(empty=True)) def test_eq_hash(self): - from psycopg2.extras import Range def assert_equal(r1, r2): self.assert_(r1 == r2) self.assert_(hash(r1) == hash(r2)) + from psycopg2.extras import Range assert_equal(Range(empty=True), Range(empty=True)) assert_equal(Range(), Range()) assert_equal(Range(10, None), Range(10, None)) @@ -1323,8 +1349,11 @@ class RangeTestCase(unittest.TestCase): def test_eq_subclass(self): from psycopg2.extras import Range, NumericRange - class IntRange(NumericRange): pass - class PositiveIntRange(IntRange): pass + class IntRange(NumericRange): + pass + + class PositiveIntRange(IntRange): + pass self.assertEqual(Range(10, 20), IntRange(10, 20)) self.assertEqual(PositiveIntRange(10, 20), IntRange(10, 20)) @@ -1397,6 +1426,12 @@ class RangeTestCase(unittest.TestCase): with py3_raises_typeerror(): self.assert_(Range(1, 2) >= 1) + def test_pickling(self): + from psycopg2.extras import Range + + r = Range(0, 4) + self.assertEqual(loads(dumps(r)), r) + def skip_if_no_range(f): @wraps(f) @@ -1476,8 +1511,8 @@ class RangeCasterTestCase(ConnectingTestCase): r = cur.fetchone()[0] self.assert_(isinstance(r, DateRange)) self.assert_(not r.isempty) - self.assertEqual(r.lower, date(2000,1,2)) - self.assertEqual(r.upper, date(2012,12,31)) + self.assertEqual(r.lower, date(2000, 1, 2)) + self.assertEqual(r.upper, date(2012, 12, 31)) self.assert_(not r.lower_inf) self.assert_(not r.upper_inf) self.assert_(r.lower_inc) @@ -1486,8 +1521,8 @@ class RangeCasterTestCase(ConnectingTestCase): def test_cast_timestamp(self): from psycopg2.extras import DateTimeRange cur = self.conn.cursor() - ts1 = datetime(2000,1,1) - ts2 = datetime(2000,12,31,23,59,59,999) + ts1 = datetime(2000, 1, 1) + ts2 = datetime(2000, 12, 31, 23, 59, 59, 999) cur.execute("select tsrange(%s, %s, '()')", (ts1, ts2)) r = cur.fetchone()[0] self.assert_(isinstance(r, DateTimeRange)) @@ -1503,8 +1538,9 @@ class RangeCasterTestCase(ConnectingTestCase): from psycopg2.extras import DateTimeTZRange from psycopg2.tz import FixedOffsetTimezone cur = self.conn.cursor() - ts1 = datetime(2000,1,1, tzinfo=FixedOffsetTimezone(600)) - ts2 = datetime(2000,12,31,23,59,59,999, tzinfo=FixedOffsetTimezone(600)) + ts1 = datetime(2000, 1, 1, tzinfo=FixedOffsetTimezone(600)) + ts2 = datetime(2000, 12, 31, 23, 59, 59, 999, + tzinfo=FixedOffsetTimezone(600)) cur.execute("select tstzrange(%s, %s, '[]')", (ts1, ts2)) r = cur.fetchone()[0] self.assert_(isinstance(r, DateTimeTZRange)) @@ -1594,8 +1630,9 @@ class RangeCasterTestCase(ConnectingTestCase): self.assert_(isinstance(r1, DateTimeRange)) self.assert_(r1.isempty) - ts1 = datetime(2000,1,1, tzinfo=FixedOffsetTimezone(600)) - ts2 = datetime(2000,12,31,23,59,59,999, tzinfo=FixedOffsetTimezone(600)) + ts1 = datetime(2000, 1, 1, tzinfo=FixedOffsetTimezone(600)) + ts2 = datetime(2000, 12, 31, 23, 59, 59, 999, + tzinfo=FixedOffsetTimezone(600)) r = DateTimeTZRange(ts1, ts2, '(]') cur.execute("select %s", (r,)) r1 = cur.fetchone()[0] @@ -1623,7 +1660,7 @@ class RangeCasterTestCase(ConnectingTestCase): self.assert_(not r1.lower_inc) self.assert_(r1.upper_inc) - cur.execute("select %s", ([r,r,r],)) + cur.execute("select %s", ([r, r, r],)) rs = cur.fetchone()[0] self.assertEqual(len(rs), 3) for r1 in rs: @@ -1647,12 +1684,12 @@ class RangeCasterTestCase(ConnectingTestCase): id integer primary key, range textrange)""") - bounds = [ '[)', '(]', '()', '[]' ] - ranges = [ TextRange(low, up, bounds[i % 4]) + bounds = ['[)', '(]', '()', '[]'] + ranges = [TextRange(low, up, bounds[i % 4]) for i, (low, up) in enumerate(zip( [None] + map(chr, range(1, 128)), - map(chr, range(1,128)) + [None], - ))] + map(chr, range(1, 128)) + [None], + ))] ranges.append(TextRange()) ranges.append(TextRange(empty=True)) @@ -1732,6 +1769,6 @@ decorate_all_tests(RangeCasterTestCase, skip_if_no_range) def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) + if __name__ == "__main__": unittest.main() - diff --git a/tests/test_with.py b/tests/test_with.py index 2f018fc8..9d91b51e 100755 --- a/tests/test_with.py +++ b/tests/test_with.py @@ -30,6 +30,7 @@ import psycopg2.extensions as ext from testutils import unittest, ConnectingTestCase + class WithTestCase(ConnectingTestCase): def setUp(self): ConnectingTestCase.setUp(self) @@ -93,7 +94,7 @@ class WithConnectionTestCase(WithTestCase): with self.conn as conn: curs = conn.cursor() curs.execute("insert into test_with values (3)") - 1/0 + 1 / 0 self.assertRaises(ZeroDivisionError, f) self.assertEqual(self.conn.status, ext.STATUS_READY) @@ -113,6 +114,7 @@ class WithConnectionTestCase(WithTestCase): def test_subclass_commit(self): commits = [] + class MyConn(ext.connection): def commit(self): commits.append(None) @@ -131,6 +133,7 @@ class WithConnectionTestCase(WithTestCase): def test_subclass_rollback(self): rollbacks = [] + class MyConn(ext.connection): def rollback(self): rollbacks.append(None) @@ -140,7 +143,7 @@ class WithConnectionTestCase(WithTestCase): with self.connect(connection_factory=MyConn) as conn: curs = conn.cursor() curs.execute("insert into test_with values (11)") - 1/0 + 1 / 0 except ZeroDivisionError: pass else: @@ -175,7 +178,7 @@ class WithCursorTestCase(WithTestCase): with self.conn as conn: with conn.cursor() as curs: curs.execute("insert into test_with values (5)") - 1/0 + 1 / 0 except ZeroDivisionError: pass @@ -189,6 +192,7 @@ class WithCursorTestCase(WithTestCase): def test_subclass(self): closes = [] + class MyCurs(ext.cursor): def close(self): closes.append(None) diff --git a/tests/testconfig.py b/tests/testconfig.py index 0f995fbf..82b48a39 100644 --- a/tests/testconfig.py +++ b/tests/testconfig.py @@ -7,8 +7,6 @@ dbhost = os.environ.get('PSYCOPG2_TESTDB_HOST', None) dbport = os.environ.get('PSYCOPG2_TESTDB_PORT', None) dbuser = os.environ.get('PSYCOPG2_TESTDB_USER', None) dbpass = os.environ.get('PSYCOPG2_TESTDB_PASSWORD', None) -repl_dsn = os.environ.get('PSYCOPG2_TEST_REPL_DSN', - "dbname=psycopg2_test replication=1") # Check if we want to test psycopg's green path. green = os.environ.get('PSYCOPG2_TEST_GREEN', None) @@ -35,4 +33,10 @@ if dbuser is not None: if dbpass is not None: dsn += ' password=%s' % dbpass +# Don't run replication tests if REPL_DSN is not set, default to normal DSN if +# set to empty string. +repl_dsn = os.environ.get('PSYCOPG2_TEST_REPL_DSN', None) +if repl_dsn == '': + repl_dsn = dsn +repl_slot = os.environ.get('PSYCOPG2_TEST_REPL_SLOT', 'psycopg2_test_slot') diff --git a/tests/testutils.py b/tests/testutils.py index 76671d99..93477357 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -27,6 +27,7 @@ import os import platform import sys +import select from functools import wraps from testconfig import dsn, repl_dsn @@ -68,8 +69,8 @@ else: # Silence warnings caused by the stubbornness of the Python unittest # maintainers # http://bugs.python.org/issue9424 -if not hasattr(unittest.TestCase, 'assert_') \ -or unittest.TestCase.assert_ is not unittest.TestCase.assertTrue: +if (not hasattr(unittest.TestCase, 'assert_') + or unittest.TestCase.assert_ is not unittest.TestCase.assertTrue): # mavaff... unittest.TestCase.assert_ = unittest.TestCase.assertTrue unittest.TestCase.failUnless = unittest.TestCase.assertTrue @@ -100,7 +101,7 @@ class ConnectingTestCase(unittest.TestCase): self._conns except AttributeError, e: raise AttributeError( - "%s (did you remember calling ConnectingTestCase.setUp()?)" + "%s (did you forget to call ConnectingTestCase.setUp()?)" % e) if 'dsn' in kwargs: @@ -121,15 +122,25 @@ class ConnectingTestCase(unittest.TestCase): Should raise a skip test if not available, but guard for None on old Python versions. """ + if repl_dsn is None: + return self.skipTest("replication tests disabled by default") + if 'dsn' not in kwargs: kwargs['dsn'] = repl_dsn import psycopg2 try: conn = self.connect(**kwargs) + if conn.async == 1: + self.wait(conn) except psycopg2.OperationalError, e: - return self.skipTest("replication db not configured: %s" % e) + # If pgcode is not set it is a genuine connection error + # Otherwise we tried to run some bad operation in the connection + # (e.g. bug #482) and we'd rather know that. + if e.pgcode is None: + return self.skipTest("replication db not configured: %s" % e) + else: + raise - conn.autocommit = True return conn def _get_conn(self): @@ -143,6 +154,23 @@ class ConnectingTestCase(unittest.TestCase): conn = property(_get_conn, _set_conn) + # for use with async connections only + def wait(self, cur_or_conn): + import psycopg2.extensions + pollable = cur_or_conn + if not hasattr(pollable, 'poll'): + pollable = cur_or_conn.connection + while True: + state = pollable.poll() + if state == psycopg2.extensions.POLL_OK: + break + elif state == psycopg2.extensions.POLL_READ: + select.select([pollable], [], [], 10) + elif state == psycopg2.extensions.POLL_WRITE: + select.select([], [pollable], [], 10) + else: + raise Exception("Unexpected result from poll: %r", state) + def decorate_all_tests(cls, *decorators): """ @@ -159,7 +187,7 @@ def skip_if_no_uuid(f): @wraps(f) def skip_if_no_uuid_(self): try: - import uuid + import uuid # noqa except ImportError: return self.skipTest("uuid not available in this Python version") @@ -207,7 +235,7 @@ def skip_if_no_namedtuple(f): @wraps(f) def skip_if_no_namedtuple_(self): try: - from collections import namedtuple + from collections import namedtuple # noqa except ImportError: return self.skipTest("collections.namedtuple not available") else: @@ -221,7 +249,7 @@ def skip_if_no_iobase(f): @wraps(f) def skip_if_no_iobase_(self): try: - from io import TextIOBase + from io import TextIOBase # noqa except ImportError: return self.skipTest("io.TextIOBase not found.") else: @@ -233,6 +261,7 @@ def skip_if_no_iobase(f): def skip_before_postgres(*ver): """Skip a test on PostgreSQL before a certain version.""" ver = ver + (0,) * (3 - len(ver)) + def skip_before_postgres_(f): @wraps(f) def skip_before_postgres__(self): @@ -245,9 +274,11 @@ def skip_before_postgres(*ver): return skip_before_postgres__ return skip_before_postgres_ + def skip_after_postgres(*ver): """Skip a test on PostgreSQL after (including) a certain version.""" ver = ver + (0,) * (3 - len(ver)) + def skip_after_postgres_(f): @wraps(f) def skip_after_postgres__(self): @@ -260,6 +291,7 @@ def skip_after_postgres(*ver): return skip_after_postgres__ return skip_after_postgres_ + def libpq_version(): import psycopg2 v = psycopg2.__libpq_version__ @@ -267,9 +299,11 @@ def libpq_version(): v = psycopg2.extensions.libpq_version() return v + def skip_before_libpq(*ver): """Skip a test if libpq we're linked to is older than a certain version.""" ver = ver + (0,) * (3 - len(ver)) + def skip_before_libpq_(f): @wraps(f) def skip_before_libpq__(self): @@ -282,9 +316,11 @@ def skip_before_libpq(*ver): return skip_before_libpq__ return skip_before_libpq_ + def skip_after_libpq(*ver): """Skip a test if libpq we're linked to is newer than a certain version.""" ver = ver + (0,) * (3 - len(ver)) + def skip_after_libpq_(f): @wraps(f) def skip_after_libpq__(self): @@ -297,6 +333,7 @@ def skip_after_libpq(*ver): return skip_after_libpq__ return skip_after_libpq_ + def skip_before_python(*ver): """Skip a test on Python before a certain version.""" def skip_before_python_(f): @@ -311,6 +348,7 @@ def skip_before_python(*ver): return skip_before_python__ return skip_before_python_ + def skip_from_python(*ver): """Skip a test on Python after (including) a certain version.""" def skip_from_python_(f): @@ -325,6 +363,7 @@ def skip_from_python(*ver): return skip_from_python__ return skip_from_python_ + def skip_if_no_superuser(f): """Skip a test if the database user running the test is not a superuser""" @wraps(f) @@ -341,6 +380,7 @@ def skip_if_no_superuser(f): return skip_if_no_superuser_ + def skip_if_green(reason): def skip_if_green_(f): @wraps(f) @@ -356,6 +396,7 @@ def skip_if_green(reason): skip_copy_if_green = skip_if_green("copy in async mode currently not supported") + def skip_if_no_getrefcount(f): @wraps(f) def skip_if_no_getrefcount_(self): @@ -365,6 +406,7 @@ def skip_if_no_getrefcount(f): return f(self) return skip_if_no_getrefcount_ + def skip_if_windows(f): """Skip a test if run on windows""" @wraps(f) @@ -403,6 +445,7 @@ def script_to_py3(script): f2.close() os.remove(filename) + class py3_raises_typeerror(object): def __enter__(self): diff --git a/tox.ini b/tox.ini index f27f3f15..4a1129d5 100644 --- a/tox.ini +++ b/tox.ini @@ -8,3 +8,8 @@ envlist = py26, py27 [testenv] commands = make check + +[flake8] +max-line-length = 85 +ignore = E128, W503 +exclude = build, doc, sandbox, examples, tests/dbapi20.py