From 0fbbd1cc91ec5060e319219ccc548b0b76fb1648 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 13:39:02 +0000 Subject: [PATCH 01/80] Bump to next dev version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a77907a7..42a17f4b 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ from distutils.ccompiler import get_default_compiler # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. -PSYCOPG_VERSION = '2.3.1' +PSYCOPG_VERSION = '2.3.2.dev0' version_flags = ['dt', 'dec'] From 9fa1eac2b49a799eda0e4d00c903e5e9a683931a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 10:29:49 +0000 Subject: [PATCH 02/80] Dropped unused include file. --- psycopg/lobject_type.c | 1 - psycopg/pgversion.h | 2 -- psycopg/pqpath.c | 1 - psycopg/utils.c | 1 - 4 files changed, 5 deletions(-) delete mode 100644 psycopg/pgversion.h diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 8aefa74e..74901123 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -36,7 +36,6 @@ #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" #include "psycopg/pqpath.h" -#include "pgversion.h" #ifdef PSYCOPG_EXTENSIONS diff --git a/psycopg/pgversion.h b/psycopg/pgversion.h deleted file mode 100644 index f874a9ae..00000000 --- a/psycopg/pgversion.h +++ /dev/null @@ -1,2 +0,0 @@ -#define PG_VERSION_MAJOR 7 -#define PG_VERSION_MINOR 4 diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index ca975658..c6fb6cd3 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -43,7 +43,6 @@ #include "psycopg/green.h" #include "psycopg/typecast.h" #include "psycopg/pgtypes.h" -#include "psycopg/pgversion.h" /* Strip off the severity from a Postgres error message. */ diff --git a/psycopg/utils.c b/psycopg/utils.c index c6028052..d8d33075 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -31,7 +31,6 @@ #include "psycopg/psycopg.h" #include "psycopg/connection.h" #include "psycopg/pgtypes.h" -#include "psycopg/pgversion.h" #include char * From 6d7916cfe12657408103ed20100436b64fb2b163 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 10:55:19 +0000 Subject: [PATCH 03/80] Internal imports simplified. .c files only need to import psycopg.h: it will in turn import dependencies from Python and libpq and configure.h. psycopg.h should be the first to be imported, so the basic imports are not required in the .h's As a guideline I'm trying to import from the most specific to the most generic to detect missing imports in the .h's. --- psycopg/adapter_asis.c | 16 +++++++--------- psycopg/adapter_asis.h | 5 ----- psycopg/adapter_binary.c | 18 +++++++----------- psycopg/adapter_binary.h | 6 ------ psycopg/adapter_datetime.c | 14 ++++++-------- psycopg/adapter_datetime.h | 5 ----- psycopg/adapter_list.c | 11 ++++------- psycopg/adapter_list.h | 5 ----- psycopg/adapter_mxdatetime.c | 15 +++++++-------- psycopg/adapter_mxdatetime.h | 5 ----- psycopg/adapter_pboolean.c | 15 ++++++--------- psycopg/adapter_pboolean.h | 5 ----- psycopg/adapter_pdecimal.c | 15 ++++++--------- psycopg/adapter_pdecimal.h | 5 ----- psycopg/adapter_pfloat.c | 15 ++++++--------- psycopg/adapter_pfloat.h | 5 ----- psycopg/adapter_qstring.c | 15 +++++---------- psycopg/adapter_qstring.h | 5 ----- psycopg/connection.h | 5 ----- psycopg/connection_int.c | 9 ++++----- psycopg/connection_type.c | 18 ++++++++---------- psycopg/cursor.h | 5 ----- psycopg/cursor_int.c | 9 ++++----- psycopg/cursor_type.c | 14 ++++++-------- psycopg/green.c | 4 ++-- psycopg/lobject.h | 3 --- psycopg/lobject_int.c | 10 ++++------ psycopg/lobject_type.c | 10 ++++------ psycopg/microprotocols.c | 13 +++++-------- psycopg/microprotocols.h | 3 --- psycopg/microprotocols_proto.c | 13 +++++-------- psycopg/microprotocols_proto.h | 6 ------ psycopg/notify.h | 4 ---- psycopg/notify_type.c | 12 +++++------- psycopg/pqpath.c | 9 +++------ psycopg/pqpath.h | 1 - psycopg/psycopgmodule.c | 6 +----- psycopg/python.h | 2 -- psycopg/typecast.c | 9 +++------ psycopg/typecast.h | 5 ----- psycopg/typecast_binary.c | 1 - psycopg/typecast_binary.h | 5 ----- psycopg/utils.c | 7 +++---- psycopg/xid.h | 4 ---- psycopg/xid_type.c | 12 +++++------- 45 files changed, 116 insertions(+), 263 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index 31cfabbe..5f3da8bf 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -22,19 +22,17 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public * License for more details. */ - -#define PY_SSIZE_T_CLEAN -#include + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/adapter_asis.h" +#include "psycopg/microprotocols_proto.h" + #include #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_asis.h" -#include "psycopg/microprotocols_proto.h" /** the AsIs object **/ diff --git a/psycopg/adapter_asis.h b/psycopg/adapter_asis.h index 28992a6e..04ba08bc 100644 --- a/psycopg/adapter_asis.h +++ b/psycopg/adapter_asis.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_ASIS_H #define PSYCOPG_ASIS_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 9b4a1c38..63d705e4 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -23,21 +23,17 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" -#include "psycopg/connection.h" + #include "psycopg/adapter_binary.h" #include "psycopg/microprotocols_proto.h" +#include "psycopg/connection.h" + +#include +#include +#include + /** the quoting code */ diff --git a/psycopg/adapter_binary.h b/psycopg/adapter_binary.h index 37aec702..899f5ce5 100644 --- a/psycopg/adapter_binary.h +++ b/psycopg/adapter_binary.h @@ -26,12 +26,6 @@ #ifndef PSYCOPG_BINARY_H #define PSYCOPG_BINARY_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 08b0cd64..9fd08575 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -23,8 +23,12 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/adapter_datetime.h" +#include "psycopg/microprotocols_proto.h" + #include #include #include @@ -32,12 +36,6 @@ #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_datetime.h" -#include "psycopg/microprotocols_proto.h" extern HIDDEN PyObject *pyPsycopgTzModule; extern HIDDEN PyObject *pyPsycopgTzLOCAL; diff --git a/psycopg/adapter_datetime.h b/psycopg/adapter_datetime.h index a2c36fb9..dd59e9b5 100644 --- a/psycopg/adapter_datetime.h +++ b/psycopg/adapter_datetime.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_DATETIME_H #define PSYCOPG_DATETIME_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index f9ab509a..d4436b0a 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -23,19 +23,16 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/adapter_list.h" #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" +#include +#include + /* list_str, list_getquoted - return result of quoting */ diff --git a/psycopg/adapter_list.h b/psycopg/adapter_list.h index a368e1a0..d7136103 100644 --- a/psycopg/adapter_list.h +++ b/psycopg/adapter_list.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_LIST_H #define PSYCOPG_LIST_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index c87d8a66..3ce0f031 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -23,19 +23,18 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +/* TODO: check if still compiles ok: I have no mx on this box */ +#include "psycopg/adapter_mxdatetime.h" +#include "psycopg/microprotocols_proto.h" + #include #include #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_mxdatetime.h" -#include "psycopg/microprotocols_proto.h" int psyco_adapter_mxdatetime_init(void) diff --git a/psycopg/adapter_mxdatetime.h b/psycopg/adapter_mxdatetime.h index 5fdeb960..bd9d4c9d 100644 --- a/psycopg/adapter_mxdatetime.h +++ b/psycopg/adapter_mxdatetime.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_MXDATETIME_H #define PSYCOPG_MXDATETIME_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index edffe8f5..8425805b 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -23,19 +23,16 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/adapter_pboolean.h" +#include "psycopg/microprotocols_proto.h" + #include #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_pboolean.h" -#include "psycopg/microprotocols_proto.h" - /** the Boolean object **/ diff --git a/psycopg/adapter_pboolean.h b/psycopg/adapter_pboolean.h index de5cd9f3..b1bb18ae 100644 --- a/psycopg/adapter_pboolean.h +++ b/psycopg/adapter_pboolean.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_PBOOLEAN_H #define PSYCOPG_PBOOLEAN_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index 347461af..b9850b46 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -23,19 +23,16 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/adapter_pdecimal.h" +#include "psycopg/microprotocols_proto.h" + #include #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_pdecimal.h" -#include "psycopg/microprotocols_proto.h" - /** the Decimal object **/ diff --git a/psycopg/adapter_pdecimal.h b/psycopg/adapter_pdecimal.h index 82825a23..4f89fad5 100644 --- a/psycopg/adapter_pdecimal.h +++ b/psycopg/adapter_pdecimal.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_PDECIMAL_H #define PSYCOPG_PDECIMAL_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 3e9f341b..3ae34a4f 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -23,19 +23,16 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/adapter_pfloat.h" +#include "psycopg/microprotocols_proto.h" + #include #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/adapter_pfloat.h" -#include "psycopg/microprotocols_proto.h" - /** the Float object **/ diff --git a/psycopg/adapter_pfloat.h b/psycopg/adapter_pfloat.h index 12b40e04..7439c04f 100644 --- a/psycopg/adapter_pfloat.h +++ b/psycopg/adapter_pfloat.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_PFLOAT_H #define PSYCOPG_PFLOAT_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 59c1d7b2..1b147f59 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -23,22 +23,17 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/adapter_qstring.h" #include "psycopg/microprotocols_proto.h" +#include +#include +#include + /* qstring_quote - do the quote process on plain and unicode strings */ diff --git a/psycopg/adapter_qstring.h b/psycopg/adapter_qstring.h index 83a8a7d4..db986be3 100644 --- a/psycopg/adapter_qstring.h +++ b/psycopg/adapter_qstring.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_QSTRING_H #define PSYCOPG_QSTRING_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/connection.h b/psycopg/connection.h index 76a6a093..477c6901 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_CONNECTION_H #define PSYCOPG_CONNECTION_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" #include "psycopg/xid.h" #ifdef __cplusplus diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 73292b88..e04d5104 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -23,19 +23,18 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/cursor.h" #include "psycopg/pqpath.h" #include "psycopg/green.h" #include "psycopg/notify.h" +#include + + /* conn_notice_callback - process notices */ static void diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 3469dc2f..d884b004 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -23,18 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/cursor.h" #include "psycopg/pqpath.h" @@ -42,6 +33,13 @@ #include "psycopg/green.h" #include "psycopg/xid.h" +#include +#include + +#include +#include + + /** DBAPI methods **/ /* cursor method - allocate a new cursor */ diff --git a/psycopg/cursor.h b/psycopg/cursor.h index 96ca2b7c..b9c0e3af 100644 --- a/psycopg/cursor.h +++ b/psycopg/cursor.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_CURSOR_H #define PSYCOPG_CURSOR_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" #include "psycopg/connection.h" #ifdef __cplusplus diff --git a/psycopg/cursor_int.c b/psycopg/cursor_int.c index 004ba82a..f53db30f 100644 --- a/psycopg/cursor_int.c +++ b/psycopg/cursor_int.c @@ -23,16 +23,15 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" + #include "psycopg/cursor.h" #include "psycopg/pqpath.h" +#include + + /* curs_reset - reset the cursor to a clean state */ void diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 488a5cd3..064b985e 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -23,15 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/cursor.h" #include "psycopg/connection.h" #include "psycopg/green.h" @@ -39,9 +33,13 @@ #include "psycopg/typecast.h" #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" -#include "pgversion.h" + +#include +#include + #include + extern PyObject *pyPsycopgTzFixedOffsetTimezone; diff --git a/psycopg/green.c b/psycopg/green.c index 4c7cc409..c9b6e07f 100644 --- a/psycopg/green.c +++ b/psycopg/green.c @@ -24,13 +24,13 @@ */ #define PSYCOPG_MODULE -#include "psycopg/python.h" #include "psycopg/psycopg.h" -#include "psycopg/config.h" + #include "psycopg/green.h" #include "psycopg/connection.h" #include "psycopg/pqpath.h" + HIDDEN PyObject *wait_callback = NULL; static PyObject *have_wait_callback(void); diff --git a/psycopg/lobject.h b/psycopg/lobject.h index 2d6715cc..cddfa6e9 100644 --- a/psycopg/lobject.h +++ b/psycopg/lobject.h @@ -26,11 +26,8 @@ #ifndef PSYCOPG_LOBJECT_H #define PSYCOPG_LOBJECT_H 1 -#include -#include #include -#include "psycopg/config.h" #include "psycopg/connection.h" #ifdef __cplusplus diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index bac2afb6..d6ebd44e 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -23,17 +23,15 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" -#include "psycopg/connection.h" + #include "psycopg/lobject.h" +#include "psycopg/connection.h" #include "psycopg/pqpath.h" +#include + #ifdef PSYCOPG_EXTENSIONS static void diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 74901123..98c11642 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -23,20 +23,18 @@ * License for more details. */ -#include -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/lobject.h" #include "psycopg/connection.h" #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" #include "psycopg/pqpath.h" +#include +#include + #ifdef PSYCOPG_EXTENSIONS diff --git a/psycopg/microprotocols.c b/psycopg/microprotocols.c index 12ec7fb0..a523d7d2 100644 --- a/psycopg/microprotocols.c +++ b/psycopg/microprotocols.c @@ -23,18 +23,15 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" -#include "psycopg/cursor.h" -#include "psycopg/connection.h" + #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" +#include "psycopg/cursor.h" +#include "psycopg/connection.h" + +#include /** the adapters registry **/ diff --git a/psycopg/microprotocols.h b/psycopg/microprotocols.h index 08e86104..df084c58 100644 --- a/psycopg/microprotocols.h +++ b/psycopg/microprotocols.h @@ -26,9 +26,6 @@ #ifndef PSYCOPG_MICROPROTOCOLS_H #define PSYCOPG_MICROPROTOCOLS_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include "psycopg/config.h" #include "psycopg/connection.h" #include "psycopg/cursor.h" diff --git a/psycopg/microprotocols_proto.c b/psycopg/microprotocols_proto.c index 1c17a85a..620ca3ff 100644 --- a/psycopg/microprotocols_proto.c +++ b/psycopg/microprotocols_proto.c @@ -23,19 +23,16 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/microprotocols_proto.h" + #include #include #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/microprotocols_proto.h" - /** void protocol implementation **/ diff --git a/psycopg/microprotocols_proto.h b/psycopg/microprotocols_proto.h index d7a67f24..614a2637 100644 --- a/psycopg/microprotocols_proto.h +++ b/psycopg/microprotocols_proto.h @@ -26,12 +26,6 @@ #ifndef PSYCOPG_ISQLQUOTE_H #define PSYCOPG_ISQLQUOTE_H 1 -#define PY_SSIZE_T_CLEAN -#include -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/notify.h b/psycopg/notify.h index 1490fb41..645fdd73 100644 --- a/psycopg/notify.h +++ b/psycopg/notify.h @@ -26,10 +26,6 @@ #ifndef PSYCOPG_NOTIFY_H #define PSYCOPG_NOTIFY_H 1 -#include - -#include "psycopg/config.h" - extern HIDDEN PyTypeObject NotifyType; typedef struct { diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index 53969709..963ff839 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -23,15 +23,13 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/notify.h" + #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/notify.h" static const char notify_doc[] = "A notification received from the backend.\n\n" diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index c6fb6cd3..e15fc7ae 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -29,14 +29,9 @@ connection. */ -#define PY_SSIZE_T_CLEAN -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/pqpath.h" #include "psycopg/connection.h" #include "psycopg/cursor.h" @@ -44,6 +39,8 @@ #include "psycopg/typecast.h" #include "psycopg/pgtypes.h" +#include + /* Strip off the severity from a Postgres error message. */ static const char * diff --git a/psycopg/pqpath.h b/psycopg/pqpath.h index 51a640dc..8e9e20a2 100644 --- a/psycopg/pqpath.h +++ b/psycopg/pqpath.h @@ -26,7 +26,6 @@ #ifndef PSYCOPG_PQPATH_H #define PSYCOPG_PQPATH_H 1 -#include "psycopg/config.h" #include "psycopg/cursor.h" #include "psycopg/connection.h" diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index db44807f..f68e2ca5 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -23,13 +23,9 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/cursor.h" #include "psycopg/green.h" diff --git a/psycopg/python.h b/psycopg/python.h index f4256d8d..27d4bb4f 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -26,8 +26,6 @@ #ifndef PSYCOPG_PYTHON_H #define PSYCOPG_PYTHON_H 1 -#define PY_SSIZE_T_CLEAN -#include #include #if PY_VERSION_HEX < 0x02040000 diff --git a/psycopg/typecast.c b/psycopg/typecast.c index 6da1effc..f12178b0 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -23,14 +23,11 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/psycopg.h" -#include "psycopg/python.h" #include "psycopg/typecast.h" #include "psycopg/cursor.h" diff --git a/psycopg/typecast.h b/psycopg/typecast.h index 06232207..cbae10a7 100644 --- a/psycopg/typecast.h +++ b/psycopg/typecast.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_TYPECAST_H #define PSYCOPG_TYPECAST_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/typecast_binary.c b/psycopg/typecast_binary.c index 6a707eae..6336c6f7 100644 --- a/psycopg/typecast_binary.c +++ b/psycopg/typecast_binary.c @@ -25,7 +25,6 @@ #include "typecast_binary.h" -#include #include diff --git a/psycopg/typecast_binary.h b/psycopg/typecast_binary.h index 14064568..bc70992a 100644 --- a/psycopg/typecast_binary.h +++ b/psycopg/typecast_binary.h @@ -26,11 +26,6 @@ #ifndef PSYCOPG_TYPECAST_BINARY_H #define PSYCOPG_TYPECAST_BINARY_H 1 -#define PY_SSIZE_T_CLEAN -#include - -#include "psycopg/config.h" - #ifdef __cplusplus extern "C" { #endif diff --git a/psycopg/utils.c b/psycopg/utils.c index d8d33075..22464e60 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -23,14 +23,13 @@ * License for more details. */ -#include -#include - #define PSYCOPG_MODULE -#include "psycopg/config.h" #include "psycopg/psycopg.h" + #include "psycopg/connection.h" #include "psycopg/pgtypes.h" + +#include #include char * diff --git a/psycopg/xid.h b/psycopg/xid.h index 017f3b4c..e63db619 100644 --- a/psycopg/xid.h +++ b/psycopg/xid.h @@ -27,10 +27,6 @@ #ifndef PSYCOPG_XID_H #define PSYCOPG_XID_H 1 -#include - -#include "psycopg/config.h" - extern HIDDEN PyTypeObject XidType; typedef struct { diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index 606cb64e..54f8553b 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -24,15 +24,13 @@ * License for more details. */ -#define PY_SSIZE_T_CLEAN -#include +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#include "psycopg/xid.h" + #include -#define PSYCOPG_MODULE -#include "psycopg/config.h" -#include "psycopg/python.h" -#include "psycopg/psycopg.h" -#include "psycopg/xid.h" static const char xid_doc[] = "A transaction identifier used for two-phase commit.\n\n" From 89f70bdb3c9827e01f9a26a3944eccdafb5b212d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 13:36:30 +0000 Subject: [PATCH 04/80] Changed Python const RO -> READONLY. --- psycopg/adapter_asis.c | 2 +- psycopg/adapter_binary.c | 4 ++-- psycopg/adapter_datetime.c | 4 ++-- psycopg/adapter_list.c | 2 +- psycopg/adapter_mxdatetime.c | 4 ++-- psycopg/adapter_pboolean.c | 2 +- psycopg/adapter_pdecimal.c | 2 +- psycopg/adapter_pfloat.c | 2 +- psycopg/adapter_qstring.c | 6 +++--- psycopg/connection_type.c | 24 ++++++++++++------------ psycopg/cursor_type.c | 18 +++++++++--------- psycopg/lobject_type.c | 4 ++-- psycopg/microprotocols_proto.c | 2 +- psycopg/notify_type.c | 6 +++--- psycopg/typecast.c | 4 ++-- psycopg/xid_type.c | 12 ++++++------ 16 files changed, 49 insertions(+), 49 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index 5f3da8bf..ccf38bae 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -74,7 +74,7 @@ asis_conform(asisObject *self, PyObject *args) /* object member list */ static struct PyMemberDef asisObject_members[] = { - {"adapted", T_OBJECT, offsetof(asisObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(asisObject, wrapped), READONLY}, {NULL} }; diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 63d705e4..28bf8e7a 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -149,8 +149,8 @@ binary_conform(binaryObject *self, PyObject *args) /* object member list */ static struct PyMemberDef binaryObject_members[] = { - {"adapted", T_OBJECT, offsetof(binaryObject, wrapped), RO}, - {"buffer", T_OBJECT, offsetof(binaryObject, buffer), RO}, + {"adapted", T_OBJECT, offsetof(binaryObject, wrapped), READONLY}, + {"buffer", T_OBJECT, offsetof(binaryObject, buffer), READONLY}, {NULL} }; diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 9fd08575..7cfcda0f 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -133,8 +133,8 @@ pydatetime_conform(pydatetimeObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pydatetimeObject_members[] = { - {"adapted", T_OBJECT, offsetof(pydatetimeObject, wrapped), RO}, - {"type", T_INT, offsetof(pydatetimeObject, type), RO}, + {"adapted", T_OBJECT, offsetof(pydatetimeObject, wrapped), READONLY}, + {"type", T_INT, offsetof(pydatetimeObject, type), READONLY}, {NULL} }; diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index d4436b0a..40598be7 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -136,7 +136,7 @@ list_conform(listObject *self, PyObject *args) /* object member list */ static struct PyMemberDef listObject_members[] = { - {"adapted", T_OBJECT, offsetof(listObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(listObject, wrapped), READONLY}, {NULL} }; diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index 3ce0f031..29012203 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -137,8 +137,8 @@ mxdatetime_conform(mxdatetimeObject *self, PyObject *args) /* object member list */ static struct PyMemberDef mxdatetimeObject_members[] = { - {"adapted", T_OBJECT, offsetof(mxdatetimeObject, wrapped), RO}, - {"type", T_INT, offsetof(mxdatetimeObject, type), RO}, + {"adapted", T_OBJECT, offsetof(mxdatetimeObject, wrapped), READONLY}, + {"type", T_INT, offsetof(mxdatetimeObject, type), READONLY}, {NULL} }; diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index 8425805b..f802832c 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -83,7 +83,7 @@ pboolean_conform(pbooleanObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pbooleanObject_members[] = { - {"adapted", T_OBJECT, offsetof(pbooleanObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(pbooleanObject, wrapped), READONLY}, {NULL} }; diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index b9850b46..d5953e88 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -105,7 +105,7 @@ pdecimal_conform(pdecimalObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pdecimalObject_members[] = { - {"adapted", T_OBJECT, offsetof(pdecimalObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(pdecimalObject, wrapped), READONLY}, {NULL} }; diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 3ae34a4f..18c2e510 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -75,7 +75,7 @@ pfloat_conform(pfloatObject *self, PyObject *args) /* object member list */ static struct PyMemberDef pfloatObject_members[] = { - {"adapted", T_OBJECT, offsetof(pfloatObject, wrapped), RO}, + {"adapted", T_OBJECT, offsetof(pfloatObject, wrapped), READONLY}, {NULL} }; diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 1b147f59..8b65fbaa 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -181,9 +181,9 @@ qstring_conform(qstringObject *self, PyObject *args) /* object member list */ static struct PyMemberDef qstringObject_members[] = { - {"adapted", T_OBJECT, offsetof(qstringObject, wrapped), RO}, - {"buffer", T_OBJECT, offsetof(qstringObject, buffer), RO}, - {"encoding", T_STRING, offsetof(qstringObject, encoding), RO}, + {"adapted", T_OBJECT, offsetof(qstringObject, wrapped), READONLY}, + {"buffer", T_OBJECT, offsetof(qstringObject, buffer), READONLY}, + {"encoding", T_STRING, offsetof(qstringObject, encoding), READONLY}, {NULL} }; diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index d884b004..f663b1fa 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -787,31 +787,31 @@ static struct PyMethodDef connectionObject_methods[] = { static struct PyMemberDef connectionObject_members[] = { #ifdef PSYCOPG_EXTENSIONS - {"closed", T_LONG, offsetof(connectionObject, closed), RO, + {"closed", T_LONG, offsetof(connectionObject, closed), READONLY, "True if the connection is closed."}, {"isolation_level", T_LONG, - offsetof(connectionObject, isolation_level), RO, + offsetof(connectionObject, isolation_level), READONLY, "The current isolation level."}, - {"encoding", T_STRING, offsetof(connectionObject, encoding), RO, + {"encoding", T_STRING, offsetof(connectionObject, encoding), READONLY, "The current client encoding."}, - {"notices", T_OBJECT, offsetof(connectionObject, notice_list), RO}, - {"notifies", T_OBJECT, offsetof(connectionObject, notifies), RO}, - {"dsn", T_STRING, offsetof(connectionObject, dsn), RO, + {"notices", T_OBJECT, offsetof(connectionObject, notice_list), READONLY}, + {"notifies", T_OBJECT, offsetof(connectionObject, notifies), READONLY}, + {"dsn", T_STRING, offsetof(connectionObject, dsn), READONLY, "The current connection string."}, - {"async", T_LONG, offsetof(connectionObject, async), RO, + {"async", T_LONG, offsetof(connectionObject, async), READONLY, "True if the connection is asynchronous."}, {"status", T_INT, - offsetof(connectionObject, status), RO, + offsetof(connectionObject, status), READONLY, "The current transaction status."}, - {"string_types", T_OBJECT, offsetof(connectionObject, string_types), RO, + {"string_types", T_OBJECT, offsetof(connectionObject, string_types), READONLY, "A set of typecasters to convert textual values."}, - {"binary_types", T_OBJECT, offsetof(connectionObject, binary_types), RO, + {"binary_types", T_OBJECT, offsetof(connectionObject, binary_types), READONLY, "A set of typecasters to convert binary values."}, {"protocol_version", T_INT, - offsetof(connectionObject, protocol), RO, + offsetof(connectionObject, protocol), READONLY, "Protocol version used for this connection. Currently always 3."}, {"server_version", T_INT, - offsetof(connectionObject, server_version), RO, + offsetof(connectionObject, server_version), READONLY, "Server version."}, #endif {NULL} diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 064b985e..d7f848a8 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1540,29 +1540,29 @@ static struct PyMethodDef cursorObject_methods[] = { static struct PyMemberDef cursorObject_members[] = { /* DBAPI-2.0 basics */ - {"rowcount", T_LONG, OFFSETOF(rowcount), RO, + {"rowcount", T_LONG, OFFSETOF(rowcount), READONLY, "Number of rows read from the backend in the last command."}, {"arraysize", T_LONG, OFFSETOF(arraysize), 0, "Number of records `fetchmany()` must fetch if not explicitly " \ "specified."}, - {"description", T_OBJECT, OFFSETOF(description), RO, + {"description", T_OBJECT, OFFSETOF(description), READONLY, "Cursor description as defined in DBAPI-2.0."}, - {"lastrowid", T_LONG, OFFSETOF(lastoid), RO, + {"lastrowid", T_LONG, OFFSETOF(lastoid), READONLY, "The ``oid`` of the last row inserted by the cursor."}, /* DBAPI-2.0 extensions */ - {"rownumber", T_LONG, OFFSETOF(row), RO, + {"rownumber", T_LONG, OFFSETOF(row), READONLY, "The current row position."}, - {"connection", T_OBJECT, OFFSETOF(conn), RO, + {"connection", T_OBJECT, OFFSETOF(conn), READONLY, "The connection where the cursor comes from."}, #ifdef PSYCOPG_EXTENSIONS - {"name", T_STRING, OFFSETOF(name), RO}, - {"statusmessage", T_OBJECT, OFFSETOF(pgstatus), RO, + {"name", T_STRING, OFFSETOF(name), READONLY}, + {"statusmessage", T_OBJECT, OFFSETOF(pgstatus), READONLY, "The return message of the last command."}, - {"query", T_OBJECT, OFFSETOF(query), RO, + {"query", T_OBJECT, OFFSETOF(query), READONLY, "The last query text sent to the backend."}, {"row_factory", T_OBJECT, OFFSETOF(tuple_factory), 0}, {"tzinfo_factory", T_OBJECT, OFFSETOF(tzinfo_factory), 0}, - {"typecaster", T_OBJECT, OFFSETOF(caster), RO}, + {"typecaster", T_OBJECT, OFFSETOF(caster), READONLY}, {"string_types", T_OBJECT, OFFSETOF(string_types), 0}, {"binary_types", T_OBJECT, OFFSETOF(binary_types), 0}, #endif diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 98c11642..2cc65064 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -274,9 +274,9 @@ static struct PyMethodDef lobjectObject_methods[] = { /* object member list */ static struct PyMemberDef lobjectObject_members[] = { - {"oid", T_UINT, offsetof(lobjectObject, oid), RO, + {"oid", T_UINT, offsetof(lobjectObject, oid), READONLY, "The backend OID associated to this lobject."}, - {"mode", T_STRING, offsetof(lobjectObject, smode), RO, + {"mode", T_STRING, offsetof(lobjectObject, smode), READONLY, "Open mode ('r', 'w', 'rw' or 'n')."}, {NULL} }; diff --git a/psycopg/microprotocols_proto.c b/psycopg/microprotocols_proto.c index 620ca3ff..3214dadd 100644 --- a/psycopg/microprotocols_proto.c +++ b/psycopg/microprotocols_proto.c @@ -96,7 +96,7 @@ static struct PyMethodDef isqlquoteObject_methods[] = { static struct PyMemberDef isqlquoteObject_members[] = { /* DBAPI-2.0 extensions (exception objects) */ - {"_wrapped", T_OBJECT, offsetof(isqlquoteObject, wrapped), RO}, + {"_wrapped", T_OBJECT, offsetof(isqlquoteObject, wrapped), READONLY}, {NULL} }; diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index 963ff839..83dee9fb 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -54,9 +54,9 @@ static const char payload_doc[] = "of the server this member is always the empty string."; static PyMemberDef notify_members[] = { - { "pid", T_OBJECT, offsetof(NotifyObject, pid), RO, (char *)pid_doc }, - { "channel", T_OBJECT, offsetof(NotifyObject, channel), RO, (char *)channel_doc }, - { "payload", T_OBJECT, offsetof(NotifyObject, payload), RO, (char *)payload_doc }, + { "pid", T_OBJECT, offsetof(NotifyObject, pid), READONLY, (char *)pid_doc }, + { "channel", T_OBJECT, offsetof(NotifyObject, channel), READONLY, (char *)channel_doc }, + { "payload", T_OBJECT, offsetof(NotifyObject, payload), READONLY, (char *)payload_doc }, { NULL } }; diff --git a/psycopg/typecast.c b/psycopg/typecast.c index f12178b0..e0a9b99d 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -390,8 +390,8 @@ typecast_richcompare(PyObject *obj1, PyObject* obj2, int opid) } static struct PyMemberDef typecastObject_members[] = { - {"name", T_OBJECT, OFFSETOF(name), RO}, - {"values", T_OBJECT, OFFSETOF(values), RO}, + {"name", T_OBJECT, OFFSETOF(name), READONLY}, + {"values", T_OBJECT, OFFSETOF(values), READONLY}, {NULL} }; diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index 54f8553b..bd26e1d7 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -68,12 +68,12 @@ static const char database_doc[] = "Database the recovered transaction belongs to."; static PyMemberDef xid_members[] = { - { "format_id", T_OBJECT, offsetof(XidObject, format_id), RO, (char *)format_id_doc }, - { "gtrid", T_OBJECT, offsetof(XidObject, gtrid), RO, (char *)gtrid_doc }, - { "bqual", T_OBJECT, offsetof(XidObject, bqual), RO, (char *)bqual_doc }, - { "prepared", T_OBJECT, offsetof(XidObject, prepared), RO, (char *)prepared_doc }, - { "owner", T_OBJECT, offsetof(XidObject, owner), RO, (char *)owner_doc }, - { "database", T_OBJECT, offsetof(XidObject, database), RO, (char *)database_doc }, + { "format_id", T_OBJECT, offsetof(XidObject, format_id), READONLY, (char *)format_id_doc }, + { "gtrid", T_OBJECT, offsetof(XidObject, gtrid), READONLY, (char *)gtrid_doc }, + { "bqual", T_OBJECT, offsetof(XidObject, bqual), READONLY, (char *)bqual_doc }, + { "prepared", T_OBJECT, offsetof(XidObject, prepared), READONLY, (char *)prepared_doc }, + { "owner", T_OBJECT, offsetof(XidObject, owner), READONLY, (char *)owner_doc }, + { "database", T_OBJECT, offsetof(XidObject, database), READONLY, (char *)database_doc }, { NULL } }; From ec182e818e43e7cd917c3e197b4cfa740ce687b9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 14:38:25 +0000 Subject: [PATCH 05/80] Added list of files the extension depends on. --- setup.py | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 42a17f4b..dd96979c 100644 --- a/setup.py +++ b/setup.py @@ -345,14 +345,37 @@ ext = [] ; data_files = [] # sources sources = [ - 'psycopgmodule.c', 'pqpath.c', 'typecast.c', + 'psycopgmodule.c', + 'green.c', 'pqpath.c', 'utils.c', + + 'connection_int.c', 'connection_type.c', + 'cursor_int.c', 'cursor_type.c', + 'lobject_int.c', 'lobject_type.c', + 'notify_type.c', 'xid_type.c', + + 'adapter_asis.c', 'adapter_binary.c', 'adapter_datetime.c', + 'adapter_list.c', 'adapter_pboolean.c', 'adapter_pdecimal.c', + 'adapter_pfloat.c', 'adapter_qstring.c', 'microprotocols.c', 'microprotocols_proto.c', - 'connection_type.c', 'connection_int.c', 'cursor_type.c', 'cursor_int.c', - 'lobject_type.c', 'lobject_int.c', 'notify_type.c', 'xid_type.c', - 'adapter_qstring.c', 'adapter_pboolean.c', 'adapter_binary.c', - 'adapter_asis.c', 'adapter_list.c', 'adapter_datetime.c', - 'adapter_pfloat.c', 'adapter_pdecimal.c', - 'green.c', 'utils.c'] + 'typecast.c', +] + +depends = [ + # headers + 'config.h', 'pgtypes.h', 'psycopg.h', 'python.h', + 'connection.h', 'cursor.h', 'green.h', 'lobject.h', + 'notify.h', 'pqpath.h', 'xid.h', + + 'adapter_asis.h', 'adapter_binary.h', 'adapter_datetime.h', + 'adapter_list.h', 'adapter_pboolean.h', 'adapter_pdecimal.h', + 'adapter_pfloat.h', 'adapter_qstring.h', + 'microprotocols.h', 'microprotocols_proto.h', + 'typecast.h', 'typecast_binary.h', + + # included sources + 'typecast_array.c', 'typecast_basic.c', 'typecast_binary.c', + 'typecast_builtins.c', 'typecast_datetime.c', +] parser = ConfigParser.ConfigParser() parser.read('setup.cfg') @@ -371,6 +394,7 @@ if os.path.exists(mxincludedir): include_dirs.append(mxincludedir) define_macros.append(('HAVE_MXDATETIME','1')) sources.append('adapter_mxdatetime.c') + depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) have_mxdatetime = True version_flags.append('mx') @@ -418,10 +442,12 @@ else: # build the extension sources = map(lambda x: os.path.join('psycopg', x), sources) +depends = map(lambda x: os.path.join('psycopg', x), depends) ext.append(Extension("psycopg2._psycopg", sources, define_macros=define_macros, include_dirs=include_dirs, + depends=depends, undef_macros=[])) setup(name="psycopg2", version=PSYCOPG_VERSION, From 8a1fa9d3a0c3fe87f3594b5cd75adb6f729ee868 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 14:58:53 +0000 Subject: [PATCH 06/80] setup.py compatible with both python 2 and 3. --- setup.py | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index dd96979c..36cdc97d 100644 --- a/setup.py +++ b/setup.py @@ -41,17 +41,27 @@ Operating System :: Microsoft :: Windows Operating System :: Unix """ +# Note: The setup.py must be compatible with both Python 2 and 3 + import os import os.path import sys import re import subprocess -import ConfigParser from distutils.core import setup, Extension from distutils.errors import DistutilsFileError from distutils.command.build_ext import build_ext from distutils.sysconfig import get_python_inc from distutils.ccompiler import get_default_compiler +try: + from distutils.command.build_py import build_py_2to3 as build_py +except ImportError: + from distutils.command.build_py import build_py + +try: + import configparser +except ImportError: + import ConfigParser as configparser # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. @@ -74,6 +84,8 @@ def get_pg_config(kind, pg_config="pg_config"): r = p.stdout.readline().strip() if not r: raise Warning(p.stderr.readline()) + if not isinstance(r, str): + r = r.decode('ascii') return r class psycopg_build_ext(build_ext): @@ -244,7 +256,8 @@ class psycopg_build_ext(build_ext): define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % (int(pgmajor), int(pgminor), int(pgpatch)))) - except Warning, w: + except Warning: + w = sys.exc_info() # work around py 2/3 different syntax if self.pg_config == self.DEFAULT_PG_CONFIG: sys.stderr.write("Warning: %s" % str(w)) else: @@ -280,21 +293,24 @@ class psycopg_build_ext(build_ext): for settingName in ('pg_config', 'include_dirs', 'library_dirs'): try: val = parser.get('build_ext', settingName) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass else: if val.strip() != '': return None # end of guard conditions - import _winreg + try: + import winreg + except ImportError: + import _winreg as winreg pg_inst_base_dir = None pg_config_path = None - reg = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE) + reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) try: - pg_inst_list_key = _winreg.OpenKey(reg, + pg_inst_list_key = winreg.OpenKey(reg, 'SOFTWARE\\PostgreSQL\\Installations' ) except EnvironmentError: @@ -304,23 +320,23 @@ class psycopg_build_ext(build_ext): try: # Determine the name of the first subkey, if any: try: - first_sub_key_name = _winreg.EnumKey(pg_inst_list_key, 0) + first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) except EnvironmentError: first_sub_key_name = None if first_sub_key_name is not None: - pg_first_inst_key = _winreg.OpenKey(reg, + pg_first_inst_key = winreg.OpenKey(reg, 'SOFTWARE\\PostgreSQL\\Installations\\' + first_sub_key_name ) try: - pg_inst_base_dir = _winreg.QueryValueEx( + pg_inst_base_dir = winreg.QueryValueEx( pg_first_inst_key, 'Base Directory' )[0] finally: - _winreg.CloseKey(pg_first_inst_key) + winreg.CloseKey(pg_first_inst_key) finally: - _winreg.CloseKey(pg_inst_list_key) + winreg.CloseKey(pg_inst_list_key) if pg_inst_base_dir and os.path.exists(pg_inst_base_dir): pg_config_path = os.path.join(pg_inst_base_dir, 'bin', @@ -377,7 +393,7 @@ depends = [ 'typecast_builtins.c', 'typecast_datetime.c', ] -parser = ConfigParser.ConfigParser() +parser = configparser.ConfigParser() parser.read('setup.cfg') # Choose a datetime module @@ -441,8 +457,8 @@ else: # build the extension -sources = map(lambda x: os.path.join('psycopg', x), sources) -depends = map(lambda x: os.path.join('psycopg', x), depends) +sources = [ os.path.join('psycopg', x) for x in sources] +depends = [ os.path.join('psycopg', x) for x in depends] ext.append(Extension("psycopg2._psycopg", sources, define_macros=define_macros, @@ -461,10 +477,12 @@ setup(name="psycopg2", platforms = ["any"], description=__doc__.split("\n")[0], long_description="\n".join(__doc__.split("\n")[2:]), - classifiers=filter(None, classifiers.split("\n")), + classifiers=[x for x in classifiers.split("\n") if x], data_files=data_files, package_dir={'psycopg2':'lib', 'psycopg2.tests': 'tests'}, packages=['psycopg2', 'psycopg2.tests'], - cmdclass={ 'build_ext': psycopg_build_ext }, + cmdclass={ + 'build_ext': psycopg_build_ext, + 'build_py': build_py, }, ext_modules=ext) From a30e461038dba6439980175ca6bc546c96551814 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 16:20:02 +0000 Subject: [PATCH 07/80] The Makefile can run with both Python 2 and 3. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e73b5f68..9ecf15fa 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ # make check # this requires setting up a test database with the correct user PYTHON := python$(PYTHON_VERSION) -PYTHON_VERSION ?= $(shell $(PYTHON) -c 'import sys; print "%d.%d" % sys.version_info[:2]') +PYTHON_VERSION ?= $(shell $(PYTHON) -c 'import sys; print ("%d.%d" % sys.version_info[:2])') BUILD_DIR = $(shell pwd)/build/lib.$(PYTHON_VERSION) ENV_DIR = $(shell pwd)/env/py-$(PYTHON_VERSION) ENV_BIN = $(ENV_DIR)/bin From 31093a7a58462617bd4646c7a6613754d761b566 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 16:45:21 +0000 Subject: [PATCH 08/80] Some light cleanup for Py3 conversion. Either flagged as warning by python2.6 -3 or converted by 2to3. --- lib/extras.py | 33 +++++++++++++++++++++------------ lib/pool.py | 2 +- tests/test_cursor.py | 2 +- tests/test_dates.py | 18 +++++++++--------- tests/test_green.py | 2 +- 5 files changed, 33 insertions(+), 24 deletions(-) diff --git a/lib/extras.py b/lib/extras.py index 09142ed6..b4528289 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -26,6 +26,7 @@ and classes untill a better place in the distribution is found. # License for more details. import os +import sys import time import codecs import warnings @@ -47,7 +48,7 @@ class DictCursorBase(_cursor): """Base class for all dict-like cursors.""" def __init__(self, *args, **kwargs): - if kwargs.has_key('row_factory'): + if 'row_factory' in kwargs: row_factory = kwargs['row_factory'] del kwargs['row_factory'] else: @@ -140,20 +141,17 @@ class DictRow(list): self[:] = [None] * len(cursor.description) def __getitem__(self, x): - if type(x) != int: + if not isinstance(x, int): x = self._index[x] return list.__getitem__(self, x) def __setitem__(self, x, v): - if type(x) != int: + if not isinstance(x, int): x = self._index[x] list.__setitem__(self, x, v) def items(self): - res = [] - for n, v in self._index.items(): - res.append((n, list.__getitem__(self, v))) - return res + return list(self.iteritems()) def keys(self): return self._index.keys() @@ -162,7 +160,7 @@ class DictRow(list): return tuple(self[:]) def has_key(self, x): - return self._index.has_key(x) + return x in self._index def get(self, x, default=None): try: @@ -171,7 +169,7 @@ class DictRow(list): return default def iteritems(self): - for n, v in self._index.items(): + for n, v in self._index.iteritems(): yield n, list.__getitem__(self, v) def iterkeys(self): @@ -181,10 +179,18 @@ class DictRow(list): return list.__iter__(self) def copy(self): - return dict(self.items()) + return dict(self.iteritems()) def __contains__(self, x): - return self._index.__contains__(x) + return x in self._index + + # grop the crusty Py2 methods + if sys.version_info[0] > 2: + items = iteritems; del iteritems + keys = iterkeys; del iterkeys + values = itervalues; del itervalues + del has_key + class RealDictConnection(_connection): """A connection that uses `RealDictCursor` automatically.""" @@ -615,7 +621,10 @@ class HstoreAdapter(object): """, regex.VERBOSE) # backslash decoder - _bsdec = codecs.getdecoder("string_escape") + if sys.version_info[0] < 3: + _bsdec = codecs.getdecoder("string_escape") + else: + _bsdec = codecs.getdecoder("unicode_escape") def parse(self, s, cur, _decoder=_bsdec): """Parse an hstore representation in a Python string. diff --git a/lib/pool.py b/lib/pool.py index ab517b6a..8a8fa539 100644 --- a/lib/pool.py +++ b/lib/pool.py @@ -100,7 +100,7 @@ class AbstractConnectionPool(object): if self.closed: raise PoolError("connection pool is closed") if key is None: key = self._getkey() - if self._used.has_key(key): + if key in self._used: return self._used[key] if self._pool: diff --git a/tests/test_cursor.py b/tests/test_cursor.py index 90b7cf2e..aedb5f5f 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -18,7 +18,7 @@ class CursorTests(unittest.TestCase): cur = conn.cursor() cur.execute("create temp table test_exc (data int);") def buggygen(): - yield 1/0 + yield 1//0 self.assertRaises(ZeroDivisionError, cur.executemany, "insert into test_exc values (%s)", buggygen()) cur.close() diff --git a/tests/test_dates.py b/tests/test_dates.py index 5c468337..9958913d 100644 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -243,18 +243,18 @@ class DatetimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date(self): from datetime import date - self._test_type_roundtrip(date(2010,05,03)) + self._test_type_roundtrip(date(2010,5,3)) def test_type_roundtrip_datetime(self): from datetime import datetime - dt = self._test_type_roundtrip(datetime(2010,05,03,10,20,30)) + dt = self._test_type_roundtrip(datetime(2010,5,3,10,20,30)) self.assertEqual(None, dt.tzinfo) def test_type_roundtrip_datetimetz(self): from datetime import datetime import psycopg2.tz tz = psycopg2.tz.FixedOffsetTimezone(8*60) - dt1 = datetime(2010,05,03,10,20,30, tzinfo=tz) + dt1 = datetime(2010,5,3,10,20,30, tzinfo=tz) dt2 = self._test_type_roundtrip(dt1) self.assertNotEqual(None, dt2.tzinfo) self.assertEqual(dt1, dt2) @@ -269,11 +269,11 @@ class DatetimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date_array(self): from datetime import date - self._test_type_roundtrip_array(date(2010,05,03)) + self._test_type_roundtrip_array(date(2010,5,3)) def test_type_roundtrip_datetime_array(self): from datetime import datetime - self._test_type_roundtrip_array(datetime(2010,05,03,10,20,30)) + self._test_type_roundtrip_array(datetime(2010,5,3,10,20,30)) def test_type_roundtrip_time_array(self): from datetime import time @@ -426,11 +426,11 @@ class mxDateTimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date(self): from mx.DateTime import Date - self._test_type_roundtrip(Date(2010,05,03)) + self._test_type_roundtrip(Date(2010,5,3)) def test_type_roundtrip_datetime(self): from mx.DateTime import DateTime - self._test_type_roundtrip(DateTime(2010,05,03,10,20,30)) + self._test_type_roundtrip(DateTime(2010,5,3,10,20,30)) def test_type_roundtrip_time(self): from mx.DateTime import Time @@ -442,11 +442,11 @@ class mxDateTimeTests(unittest.TestCase, CommonDatetimeTestsMixin): def test_type_roundtrip_date_array(self): from mx.DateTime import Date - self._test_type_roundtrip_array(Date(2010,05,03)) + self._test_type_roundtrip_array(Date(2010,5,3)) def test_type_roundtrip_datetime_array(self): from mx.DateTime import DateTime - self._test_type_roundtrip_array(DateTime(2010,05,03,10,20,30)) + self._test_type_roundtrip_array(DateTime(2010,5,3,10,20,30)) def test_type_roundtrip_time_array(self): from mx.DateTime import Time diff --git a/tests/test_green.py b/tests/test_green.py index 04fbc1ae..07ecdb7b 100644 --- a/tests/test_green.py +++ b/tests/test_green.py @@ -63,7 +63,7 @@ class GreenTests(unittest.TestCase): curs.fetchone() # now try to do something that will fail in the callback - psycopg2.extensions.set_wait_callback(lambda conn: 1/0) + psycopg2.extensions.set_wait_callback(lambda conn: 1//0) self.assertRaises(ZeroDivisionError, curs.execute, "select 2") # check that the connection is left in an usable state From 9b29282ee477609052545aef0492b3f0461e7879 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 16:50:29 +0000 Subject: [PATCH 09/80] 'make check' runs the test in the build directory. This way tests can be run win Py3 too, as the setup 2to3s them. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ecf15fa..12617f06 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ ez_setup: wget -O $(EZ_SETUP) http://peak.telecommunity.com/dist/ez_setup.py check: - PYTHONPATH=$(BUILD_DIR):.:$(PYTHONPATH) $(PYTHON) tests/__init__.py --verbose + PYTHONPATH=$(BUILD_DIR):$(BUILD_DIR)/tests:$(PYTHONPATH) $(PYTHON) $(BUILD_DIR)/psycopg2/tests/__init__.py --verbose testdb: @echo "* Creating $(TESTDB)" From 2196ff5488ee5951b57849f674a83c3aca656f1c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 21:30:00 +0000 Subject: [PATCH 10/80] Added a few compatibility macros defined in Py 2.6. --- psycopg/python.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/psycopg/python.h b/psycopg/python.h index 27d4bb4f..6faa5b96 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -51,8 +51,12 @@ #define CONV_CODE_PY_SSIZE_T "n" #endif -#ifndef Py_TYPE - #define Py_TYPE(o) (((PyObject*)(o))->ob_type) +/* Macros defined in Python 2.6 */ +#ifndef Py_REFCNT +#define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) +#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) +#define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) +#define PyVarObject_HEAD_INIT(x,n) PyObject_HEAD_INIT(x) n, #endif /* FORMAT_CODE_PY_SSIZE_T is for Py_ssize_t: */ From 8dfa9915ebf04a6bcea4e7b02d8999d6c3ab0c0d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 21:31:10 +0000 Subject: [PATCH 11/80] Using Py_TYPE and Py_REFCNT macros. --- psycopg/adapter_asis.c | 8 ++++---- psycopg/adapter_binary.c | 8 ++++---- psycopg/adapter_datetime.c | 8 ++++---- psycopg/adapter_list.c | 8 ++++---- psycopg/adapter_mxdatetime.c | 8 ++++---- psycopg/adapter_pboolean.c | 8 ++++---- psycopg/adapter_pdecimal.c | 8 ++++---- psycopg/adapter_pfloat.c | 8 ++++---- psycopg/adapter_qstring.c | 8 ++++---- psycopg/connection_type.c | 12 ++++++------ psycopg/cursor_type.c | 14 +++++++------- psycopg/lobject_type.c | 6 +++--- psycopg/microprotocols.c | 10 ++++++---- psycopg/microprotocols_proto.c | 2 +- psycopg/notify_type.c | 2 +- psycopg/psycopgmodule.c | 34 +++++++++++++++++----------------- psycopg/typecast.c | 6 +++--- psycopg/typecast_binary.c | 2 +- psycopg/typecast_datetime.c | 2 +- psycopg/xid_type.c | 2 +- 20 files changed, 83 insertions(+), 81 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index ccf38bae..f779c165 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -94,7 +94,7 @@ asis_setup(asisObject *self, PyObject *obj) { Dprintf("asis_setup: init asis object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -102,7 +102,7 @@ asis_setup(asisObject *self, PyObject *obj) Dprintf("asis_setup: good asis object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -123,10 +123,10 @@ asis_dealloc(PyObject* obj) Dprintf("asis_dealloc: deleted asis object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 28bf8e7a..fd469180 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -172,7 +172,7 @@ binary_setup(binaryObject *self, PyObject *str) { Dprintf("binary_setup: init binary object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); self->buffer = NULL; @@ -182,7 +182,7 @@ binary_setup(binaryObject *self, PyObject *str) Dprintf("binary_setup: good binary object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt); + self, Py_REFCNT(self)); return 0; } @@ -208,10 +208,10 @@ binary_dealloc(PyObject* obj) Dprintf("binary_dealloc: deleted binary object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 7cfcda0f..2dab61cd 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -154,7 +154,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type) { Dprintf("pydatetime_setup: init datetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt); + self, Py_REFCNT(self)); self->type = type; Py_INCREF(obj); @@ -162,7 +162,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type) Dprintf("pydatetime_setup: good pydatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt); + self, Py_REFCNT(self)); return 0; } @@ -183,9 +183,9 @@ pydatetime_dealloc(PyObject* obj) Py_CLEAR(self->wrapped); Dprintf("mpydatetime_dealloc: deleted pydatetime object at %p, " - "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, obj->ob_refcnt); + "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 40598be7..e98e72b9 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -158,7 +158,7 @@ list_setup(listObject *self, PyObject *obj, const char *enc) { Dprintf("list_setup: init list object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); if (!PyList_Check(obj)) @@ -173,7 +173,7 @@ list_setup(listObject *self, PyObject *obj, const char *enc) Dprintf("list_setup: good list object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -198,9 +198,9 @@ list_dealloc(PyObject* obj) if (self->encoding) free(self->encoding); Dprintf("list_dealloc: deleted list object at %p, " - "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, obj->ob_refcnt); + "refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index 29012203..0216250d 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -158,7 +158,7 @@ mxdatetime_setup(mxdatetimeObject *self, PyObject *obj, int type) { Dprintf("mxdatetime_setup: init mxdatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); self->type = type; @@ -167,7 +167,7 @@ mxdatetime_setup(mxdatetimeObject *self, PyObject *obj, int type) Dprintf("mxdatetime_setup: good mxdatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -190,10 +190,10 @@ mxdatetime_dealloc(PyObject* obj) Dprintf("mxdatetime_dealloc: deleted mxdatetime object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index f802832c..d1c8f708 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -103,7 +103,7 @@ pboolean_setup(pbooleanObject *self, PyObject *obj) { Dprintf("pboolean_setup: init pboolean object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -111,7 +111,7 @@ pboolean_setup(pbooleanObject *self, PyObject *obj) Dprintf("pboolean_setup: good pboolean object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -134,10 +134,10 @@ pboolean_dealloc(PyObject* obj) Dprintf("pboolean_dealloc: deleted pboolean object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index d5953e88..0b7b7d66 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -125,7 +125,7 @@ pdecimal_setup(pdecimalObject *self, PyObject *obj) { Dprintf("pdecimal_setup: init pdecimal object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -133,7 +133,7 @@ pdecimal_setup(pdecimalObject *self, PyObject *obj) Dprintf("pdecimal_setup: good pdecimal object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -156,10 +156,10 @@ pdecimal_dealloc(PyObject* obj) Dprintf("pdecimal_dealloc: deleted pdecimal object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 18c2e510..1aaee99f 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -95,7 +95,7 @@ pfloat_setup(pfloatObject *self, PyObject *obj) { Dprintf("pfloat_setup: init pfloat object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); Py_INCREF(obj); @@ -103,7 +103,7 @@ pfloat_setup(pfloatObject *self, PyObject *obj) Dprintf("pfloat_setup: good pfloat object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -126,10 +126,10 @@ pfloat_dealloc(PyObject* obj) Dprintf("pfloat_dealloc: deleted pfloat object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 8b65fbaa..6b6fe331 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -205,7 +205,7 @@ qstring_setup(qstringObject *self, PyObject *str, const char *enc) { Dprintf("qstring_setup: init qstring object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); self->buffer = NULL; @@ -219,7 +219,7 @@ qstring_setup(qstringObject *self, PyObject *str, const char *enc) Dprintf("qstring_setup: good qstring object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -248,10 +248,10 @@ qstring_dealloc(PyObject* obj) Dprintf("qstring_dealloc: deleted qstring object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index f663b1fa..5a46982f 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -101,7 +101,7 @@ psyco_conn_cursor(connectionObject *self, PyObject *args, PyObject *keywds) Dprintf("psyco_conn_cursor: new cursor at %p: refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); return obj; } @@ -578,7 +578,7 @@ psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) Dprintf("psyco_conn_lobject: new lobject at %p: refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt); + obj, Py_REFCNT(obj)); return obj; } @@ -846,7 +846,7 @@ connection_setup(connectionObject *self, const char *dsn, long int async) Dprintf("connection_setup: init connection object at %p, " "async %ld, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, async, ((PyObject *)self)->ob_refcnt + self, async, Py_REFCNT(self) ); self->dsn = strdup(dsn); @@ -875,7 +875,7 @@ connection_setup(connectionObject *self, const char *dsn, long int async) else { Dprintf("connection_setup: good connection object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); res = 0; } @@ -916,10 +916,10 @@ connection_dealloc(PyObject* obj) Dprintf("connection_dealloc: deleted connection object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt + obj, Py_REFCNT(obj) ); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index d7f848a8..6b40032b 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -130,7 +130,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) } Dprintf("_mogrify: value refcnt: " - FORMAT_CODE_PY_SSIZE_T " (+1)", value->ob_refcnt); + FORMAT_CODE_PY_SSIZE_T " (+1)", Py_REFCNT(value)); if (n == NULL) { n = PyDict_New(); @@ -183,7 +183,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) Py_DECREF(key); /* key has the original refcnt now */ Dprintf("_mogrify: after value refcnt: " FORMAT_CODE_PY_SSIZE_T, - value->ob_refcnt + Py_REFCNT(value) ); } c = d; @@ -594,7 +594,7 @@ _psyco_curs_mogrify(cursorObject *self, Dprintf("psyco_curs_mogrify: cvt->refcnt = " FORMAT_CODE_PY_SSIZE_T ", fquery->refcnt = " FORMAT_CODE_PY_SSIZE_T, - cvt->ob_refcnt, fquery->ob_refcnt); + Py_REFCNT(cvt), Py_REFCNT(fquery)); } else { fquery = operation; @@ -679,7 +679,7 @@ _psyco_curs_buildrow_fill(cursorObject *self, PyObject *res, if (val) { Dprintf("_psyco_curs_buildrow: val->refcnt = " FORMAT_CODE_PY_SSIZE_T, - val->ob_refcnt + Py_REFCNT(val) ); if (istuple) { PyTuple_SET_ITEM(res, i, val); @@ -1631,7 +1631,7 @@ cursor_setup(cursorObject *self, connectionObject *conn, const char *name) Dprintf("cursor_setup: good cursor object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - self, ((PyObject *)self)->ob_refcnt + self, Py_REFCNT(self) ); return 0; } @@ -1659,9 +1659,9 @@ cursor_dealloc(PyObject* obj) Dprintf("cursor_dealloc: deleted cursor object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt); + obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 2cc65064..9284bb8a 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -315,7 +315,7 @@ lobject_setup(lobjectObject *self, connectionObject *conn, return -1; Dprintf("lobject_setup: good lobject object at %p, refcnt = " - FORMAT_CODE_PY_SSIZE_T, self, ((PyObject *)self)->ob_refcnt); + FORMAT_CODE_PY_SSIZE_T, self, Py_REFCNT(self)); Dprintf("lobject_setup: oid = %d, fd = %d", self->oid, self->fd); return 0; } @@ -330,9 +330,9 @@ lobject_dealloc(PyObject* obj) Py_XDECREF((PyObject*)self->conn); Dprintf("lobject_dealloc: deleted lobject object at %p, refcnt = " - FORMAT_CODE_PY_SSIZE_T, obj, obj->ob_refcnt); + FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/microprotocols.c b/psycopg/microprotocols.c index a523d7d2..8fd3346c 100644 --- a/psycopg/microprotocols.c +++ b/psycopg/microprotocols.c @@ -84,7 +84,7 @@ _get_superclass_adapter(PyObject *obj, PyObject *proto) PyObject *key, *adapter; Py_ssize_t i, ii; - type = (PyTypeObject *)Py_TYPE(obj); + type = Py_TYPE(obj); if (!((Py_TPFLAGS_HAVE_CLASS & type->tp_flags) && type->tp_mro)) { /* has no mro */ return NULL; @@ -138,7 +138,8 @@ microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) if (obj == Py_None) return PyString_FromString("NULL"); - Dprintf("microprotocols_adapt: trying to adapt %s", obj->ob_type->tp_name); + Dprintf("microprotocols_adapt: trying to adapt %s", + Py_TYPE(obj)->tp_name); /* look for an adapter in the registry */ key = PyTuple_Pack(2, Py_TYPE(obj), proto); @@ -194,7 +195,8 @@ microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) } /* else set the right exception and return NULL */ - PyOS_snprintf(buffer, 255, "can't adapt type '%s'", obj->ob_type->tp_name); + PyOS_snprintf(buffer, 255, "can't adapt type '%s'", + Py_TYPE(obj)->tp_name); psyco_set_error(ProgrammingError, NULL, buffer, NULL, NULL); return NULL; } @@ -213,7 +215,7 @@ microprotocol_getquoted(PyObject *obj, connectionObject *conn) } Dprintf("microprotocol_getquoted: adapted to %s", - adapted->ob_type->tp_name); + Py_TYPE(adapted)->tp_name); /* if requested prepare the object passing it the connection */ if (conn) { diff --git a/psycopg/microprotocols_proto.c b/psycopg/microprotocols_proto.c index 3214dadd..f71eec14 100644 --- a/psycopg/microprotocols_proto.c +++ b/psycopg/microprotocols_proto.c @@ -118,7 +118,7 @@ isqlquote_dealloc(PyObject* obj) Py_XDECREF(self->wrapped); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index 83dee9fb..1439ab75 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -114,7 +114,7 @@ notify_dealloc(NotifyObject *self) Py_CLEAR(self->channel); Py_CLEAR(self->payload); - self->ob_type->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free((PyObject *)self); } static void diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index f68e2ca5..8f9f2d1a 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -713,20 +713,20 @@ init_psycopg(void) Dprintf("initpsycopg: initializing psycopg %s", PSYCOPG_VERSION); /* initialize all the new types and then the module */ - connectionType.ob_type = &PyType_Type; - cursorType.ob_type = &PyType_Type; - typecastType.ob_type = &PyType_Type; - qstringType.ob_type = &PyType_Type; - binaryType.ob_type = &PyType_Type; - isqlquoteType.ob_type = &PyType_Type; - pbooleanType.ob_type = &PyType_Type; - pfloatType.ob_type = &PyType_Type; - pdecimalType.ob_type = &PyType_Type; - asisType.ob_type = &PyType_Type; - listType.ob_type = &PyType_Type; - chunkType.ob_type = &PyType_Type; - NotifyType.ob_type = &PyType_Type; - XidType.ob_type = &PyType_Type; + Py_TYPE(&connectionType) = &PyType_Type; + Py_TYPE(&cursorType) = &PyType_Type; + Py_TYPE(&typecastType) = &PyType_Type; + Py_TYPE(&qstringType) = &PyType_Type; + Py_TYPE(&binaryType) = &PyType_Type; + Py_TYPE(&isqlquoteType) = &PyType_Type; + Py_TYPE(&pbooleanType) = &PyType_Type; + Py_TYPE(&pfloatType) = &PyType_Type; + Py_TYPE(&pdecimalType) = &PyType_Type; + Py_TYPE(&asisType) = &PyType_Type; + Py_TYPE(&listType) = &PyType_Type; + Py_TYPE(&chunkType) = &PyType_Type; + Py_TYPE(&NotifyType) = &PyType_Type; + Py_TYPE(&XidType) = &PyType_Type; if (PyType_Ready(&connectionType) == -1) return; if (PyType_Ready(&cursorType) == -1) return; @@ -744,13 +744,13 @@ init_psycopg(void) if (PyType_Ready(&XidType) == -1) return; #ifdef PSYCOPG_EXTENSIONS - lobjectType.ob_type = &PyType_Type; + Py_TYPE(&lobjectType) = &PyType_Type; if (PyType_Ready(&lobjectType) == -1) return; #endif /* import mx.DateTime module, if necessary */ #ifdef HAVE_MXDATETIME - mxdatetimeType.ob_type = &PyType_Type; + Py_TYPE(&mxdatetimeType) = &PyType_Type; if (PyType_Ready(&mxdatetimeType) == -1) return; if (mxDateTime_ImportModuleAndAPI() != 0) { Dprintf("initpsycopg: why marc hide mx.DateTime again?!"); @@ -772,7 +772,7 @@ init_psycopg(void) PyDateTime_IMPORT; if (psyco_adapter_datetime_init()) { return; } - pydatetimeType.ob_type = &PyType_Type; + Py_TYPE(&pydatetimeType) = &PyType_Type; if (PyType_Ready(&pydatetimeType) == -1) return; /* import psycopg2.tz anyway (TODO: replace with C-level module?) */ diff --git a/psycopg/typecast.c b/psycopg/typecast.c index e0a9b99d..46882055 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -309,7 +309,7 @@ typecast_add(PyObject *obj, PyObject *dict, int binary) Dprintf("typecast_add: object at %p, values refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, type->values->ob_refcnt + obj, Py_REFCNT(type->values) ); if (dict == NULL) @@ -407,7 +407,7 @@ typecast_dealloc(PyObject *obj) Py_CLEAR(self->pcast); Py_CLEAR(self->bcast); - obj->ob_type->tp_free(obj); + Py_TYPE(obj)->tp_free(obj); } static int @@ -521,7 +521,7 @@ typecast_new(PyObject *name, PyObject *values, PyObject *cast, PyObject *base) if (obj == NULL) return NULL; Dprintf("typecast_new: new type at = %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - obj, obj->ob_refcnt); + obj, Py_REFCNT(obj)); Py_INCREF(values); obj->values = values; diff --git a/psycopg/typecast_binary.c b/psycopg/typecast_binary.c index 6336c6f7..f0d4f866 100644 --- a/psycopg/typecast_binary.c +++ b/psycopg/typecast_binary.c @@ -41,7 +41,7 @@ chunk_dealloc(chunkObject *self) self->base, self->len ); PQfreemem(self->base); - self->ob_type->tp_free((PyObject *) self); + Py_TYPE(self)->tp_free((PyObject *)self); } static PyObject * diff --git a/psycopg/typecast_datetime.c b/psycopg/typecast_datetime.c index 2bc3625e..29149c05 100644 --- a/psycopg/typecast_datetime.c +++ b/psycopg/typecast_datetime.c @@ -159,7 +159,7 @@ typecast_PYDATETIME_cast(const char *str, Py_ssize_t len, PyObject *curs) y, m, d, hh, mm, ss, us, tzinfo); Dprintf("typecast_PYDATETIME_cast: tzinfo: %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, - tzinfo, tzinfo->ob_refcnt + tzinfo, Py_REFCNT(tzinfo) ); Py_DECREF(tzinfo); } diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index bd26e1d7..68a9a210 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -184,7 +184,7 @@ xid_dealloc(XidObject *self) Py_CLEAR(self->owner); Py_CLEAR(self->database); - self->ob_type->tp_free((PyObject *)self); + Py_TYPE(self)->tp_free((PyObject *)self); } static void From 9b30147341323b912d207f329663791a816f96be Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 21:47:36 +0000 Subject: [PATCH 12/80] Using PyVarObject_HEAD_INIT macro. --- psycopg/adapter_asis.c | 3 +-- psycopg/adapter_binary.c | 3 +-- psycopg/adapter_datetime.c | 3 +-- psycopg/adapter_list.c | 3 +-- psycopg/adapter_mxdatetime.c | 3 +-- psycopg/adapter_pboolean.c | 3 +-- psycopg/adapter_pdecimal.c | 3 +-- psycopg/adapter_pfloat.c | 3 +-- psycopg/adapter_qstring.c | 3 +-- psycopg/connection_type.c | 3 +-- psycopg/cursor_type.c | 3 +-- psycopg/lobject_type.c | 3 +-- psycopg/microprotocols_proto.c | 3 +-- psycopg/notify_type.c | 3 +-- psycopg/typecast.c | 3 +-- psycopg/typecast_binary.c | 3 +-- psycopg/xid_type.c | 3 +-- 17 files changed, 17 insertions(+), 34 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index f779c165..b0e7bc89 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -165,8 +165,7 @@ asis_repr(asisObject *self) "AsIs(str) -> new AsIs adapter object" PyTypeObject asisType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.AsIs", sizeof(asisObject), 0, diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index fd469180..ed477b74 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -249,8 +249,7 @@ binary_repr(binaryObject *self) "Binary(buffer) -> new binary object" PyTypeObject binaryType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Binary", sizeof(binaryObject), 0, diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 2dab61cd..ec1f2d49 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -225,8 +225,7 @@ pydatetime_repr(pydatetimeObject *self) "datetime(datetime, type) -> new datetime wrapper object" PyTypeObject pydatetimeType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.datetime", sizeof(pydatetimeObject), 0, diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index e98e72b9..53846182 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -239,8 +239,7 @@ list_repr(listObject *self) "List(list) -> new list wrapper object" PyTypeObject listType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.List", sizeof(listObject), 0, diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index 0216250d..36ef48a9 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -233,8 +233,7 @@ mxdatetime_repr(mxdatetimeObject *self) "MxDateTime(mx, type) -> new mx.DateTime wrapper object" PyTypeObject mxdatetimeType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.MxDateTime", sizeof(mxdatetimeObject), 0, diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index d1c8f708..73284a83 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -177,8 +177,7 @@ pboolean_repr(pbooleanObject *self) "Boolean(str) -> new Boolean adapter object" PyTypeObject pbooleanType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Boolean", sizeof(pbooleanObject), 0, diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index 0b7b7d66..c877eaeb 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -199,8 +199,7 @@ pdecimal_repr(pdecimalObject *self) "Decimal(str) -> new Decimal adapter object" PyTypeObject pdecimalType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Decimal", sizeof(pdecimalObject), 0, diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 1aaee99f..f75424ef 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -169,8 +169,7 @@ pfloat_repr(pfloatObject *self) "Float(str) -> new Float adapter object" PyTypeObject pfloatType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.Float", sizeof(pfloatObject), 0, diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 6b6fe331..d4338be8 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -291,8 +291,7 @@ qstring_repr(qstringObject *self) "QuotedString(str, enc) -> new quoted object with 'enc' encoding" PyTypeObject qstringType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.QuotedString", sizeof(qstringObject), 0, diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 5a46982f..0f10276a 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -978,8 +978,7 @@ connection_traverse(connectionObject *self, visitproc visit, void *arg) " ProgrammingError, IntegrityError, DataError, NotSupportedError" PyTypeObject connectionType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.connection", sizeof(connectionObject), 0, diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 6b40032b..0a398963 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1719,8 +1719,7 @@ cursor_traverse(cursorObject *self, visitproc visit, void *arg) "A database cursor." PyTypeObject cursorType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.cursor", sizeof(cursorObject), 0, diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 9284bb8a..29dcddc6 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -377,8 +377,7 @@ lobject_repr(lobjectObject *self) "A database large object." PyTypeObject lobjectType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.lobject", sizeof(lobjectObject), 0, diff --git a/psycopg/microprotocols_proto.c b/psycopg/microprotocols_proto.c index f71eec14..d929c569 100644 --- a/psycopg/microprotocols_proto.c +++ b/psycopg/microprotocols_proto.c @@ -153,8 +153,7 @@ isqlquote_del(PyObject* self) "returning the SQL representation of the object.\n\n" PyTypeObject isqlquoteType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.ISQLQuote", sizeof(isqlquoteObject), 0, diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index 1439ab75..02873120 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -278,8 +278,7 @@ static PySequenceMethods notify_sequence = { PyTypeObject NotifyType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2.extensions.Notify", sizeof(NotifyObject), 0, diff --git a/psycopg/typecast.c b/psycopg/typecast.c index 46882055..46b9c9f4 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -452,8 +452,7 @@ typecast_call(PyObject *obj, PyObject *args, PyObject *kwargs) } PyTypeObject typecastType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.type", sizeof(typecastObject), 0, diff --git a/psycopg/typecast_binary.c b/psycopg/typecast_binary.c index f0d4f866..261fc6f2 100644 --- a/psycopg/typecast_binary.c +++ b/psycopg/typecast_binary.c @@ -85,8 +85,7 @@ static PyBufferProcs chunk_as_buffer = #define chunk_doc "memory chunk" PyTypeObject chunkType = { - PyObject_HEAD_INIT(NULL) - 0, /* ob_size */ + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2._psycopg.chunk", /* tp_name */ sizeof(chunkObject), /* tp_basicsize */ 0, /* tp_itemsize */ diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index 68a9a210..9aed22ac 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -304,8 +304,7 @@ static struct PyMethodDef xid_methods[] = { }; PyTypeObject XidType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "psycopg2.extensions.Xid", sizeof(XidObject), 0, From 4635c2aa4f41c1fe05ec4d5d562f89eb3c4f9d27 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 13 Dec 2010 02:24:33 +0000 Subject: [PATCH 13/80] Import structmember/stringobject headers from python.h. stringobject is not to be imported with Python 3. --- psycopg/adapter_asis.c | 2 -- psycopg/adapter_binary.c | 2 -- psycopg/adapter_datetime.c | 2 -- psycopg/adapter_list.c | 3 --- psycopg/adapter_mxdatetime.c | 2 -- psycopg/adapter_pboolean.c | 2 -- psycopg/adapter_pdecimal.c | 1 - psycopg/adapter_pfloat.c | 1 - psycopg/adapter_qstring.c | 2 -- psycopg/connection_type.c | 3 --- psycopg/cursor_type.c | 1 - psycopg/lobject_type.c | 1 - psycopg/microprotocols.c | 2 -- psycopg/microprotocols_proto.c | 3 --- psycopg/notify_type.c | 2 -- psycopg/python.h | 3 +++ psycopg/typecast.c | 2 -- psycopg/xid_type.c | 2 -- 18 files changed, 3 insertions(+), 33 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index b0e7bc89..0d0a7f67 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -29,8 +29,6 @@ #include "psycopg/adapter_asis.h" #include "psycopg/microprotocols_proto.h" -#include -#include #include diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index ed477b74..cf31ccbf 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -30,8 +30,6 @@ #include "psycopg/microprotocols_proto.h" #include "psycopg/connection.h" -#include -#include #include diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index ec1f2d49..f640684a 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -29,8 +29,6 @@ #include "psycopg/adapter_datetime.h" #include "psycopg/microprotocols_proto.h" -#include -#include #include #include diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 53846182..8bd66881 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -30,9 +30,6 @@ #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" -#include -#include - /* list_str, list_getquoted - return result of quoting */ diff --git a/psycopg/adapter_mxdatetime.c b/psycopg/adapter_mxdatetime.c index 36ef48a9..793dfba2 100644 --- a/psycopg/adapter_mxdatetime.c +++ b/psycopg/adapter_mxdatetime.c @@ -30,8 +30,6 @@ #include "psycopg/adapter_mxdatetime.h" #include "psycopg/microprotocols_proto.h" -#include -#include #include #include diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index 73284a83..a74d9f37 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -29,8 +29,6 @@ #include "psycopg/adapter_pboolean.h" #include "psycopg/microprotocols_proto.h" -#include -#include #include diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index c877eaeb..ed79ea1a 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -29,7 +29,6 @@ #include "psycopg/adapter_pdecimal.h" #include "psycopg/microprotocols_proto.h" -#include #include #include diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index f75424ef..f6ad6361 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -29,7 +29,6 @@ #include "psycopg/adapter_pfloat.h" #include "psycopg/microprotocols_proto.h" -#include #include #include diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index d4338be8..9da6a212 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -30,8 +30,6 @@ #include "psycopg/adapter_qstring.h" #include "psycopg/microprotocols_proto.h" -#include -#include #include diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 0f10276a..21e5e8e2 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -33,9 +33,6 @@ #include "psycopg/green.h" #include "psycopg/xid.h" -#include -#include - #include #include diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 0a398963..cd62d508 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -34,7 +34,6 @@ #include "psycopg/microprotocols.h" #include "psycopg/microprotocols_proto.h" -#include #include #include diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 29dcddc6..d07becbe 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -32,7 +32,6 @@ #include "psycopg/microprotocols_proto.h" #include "psycopg/pqpath.h" -#include #include diff --git a/psycopg/microprotocols.c b/psycopg/microprotocols.c index 8fd3346c..f41d85f1 100644 --- a/psycopg/microprotocols.c +++ b/psycopg/microprotocols.c @@ -31,8 +31,6 @@ #include "psycopg/cursor.h" #include "psycopg/connection.h" -#include - /** the adapters registry **/ diff --git a/psycopg/microprotocols_proto.c b/psycopg/microprotocols_proto.c index d929c569..775889d9 100644 --- a/psycopg/microprotocols_proto.c +++ b/psycopg/microprotocols_proto.c @@ -28,9 +28,6 @@ #include "psycopg/microprotocols_proto.h" -#include -#include - #include diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index 02873120..3d2308ad 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -28,8 +28,6 @@ #include "psycopg/notify.h" -#include - static const char notify_doc[] = "A notification received from the backend.\n\n" diff --git a/psycopg/python.h b/psycopg/python.h index 6faa5b96..b514c2c6 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -27,6 +27,9 @@ #define PSYCOPG_PYTHON_H 1 #include +#if PY_MAJOR_VERSION < 3 +#include +#endif #if PY_VERSION_HEX < 0x02040000 # error "psycopg requires Python >= 2.4" diff --git a/psycopg/typecast.c b/psycopg/typecast.c index 46b9c9f4..47c3c58b 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -26,8 +26,6 @@ #define PSYCOPG_MODULE #include "psycopg/psycopg.h" -#include - #include "psycopg/typecast.h" #include "psycopg/cursor.h" diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index 9aed22ac..b7251359 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -29,8 +29,6 @@ #include "psycopg/xid.h" -#include - static const char xid_doc[] = "A transaction identifier used for two-phase commit.\n\n" From b96dcef8a264ab166608c9f821e73d8510c2c871 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 14 Dec 2010 02:21:49 +0000 Subject: [PATCH 14/80] Fixed PG -> Py encodings mapping with non-alnum chars. We mangle the encoding names a little bit before asking it to the backend: be sure to be able to find the equivalent Python code back or decoding (unicode cast or Py3) will barf. --- ChangeLog | 4 ++++ lib/extensions.py | 8 ++++++++ tests/test_connection.py | 8 ++++++++ 3 files changed, 20 insertions(+) diff --git a/ChangeLog b/ChangeLog index 41a90c6f..11081e5e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2010-12-14 Daniele Varrazzo + + * lib/extensions.py: Improved mapping from PG to Py encodings. + 2010-12-04 Daniele Varrazzo * setup.py: bumped to version 2.3.1.dev0 diff --git a/lib/extensions.py b/lib/extensions.py index fca79738..7618751c 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -128,4 +128,12 @@ class SQL_IN(object): __str__ = getquoted +# Add the "cleaned" version of the encodings to the key. +# When the encoding is set its name is cleaned up from - and _ and turned +# uppercase, so an encoding not respecting these rules wouldn't be found in the +# encodings keys and would raise an exception with the unicode typecaster +for k, v in encodings.items(): + k = k.replace('_', '').replace('-', '').upper() + encodings[k] = v + __all__ = filter(lambda k: not k.startswith('_'), locals().keys()) diff --git a/tests/test_connection.py b/tests/test_connection.py index 8d87e730..de775e08 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -111,6 +111,14 @@ class ConnectionTests(unittest.TestCase): self.assert_(time.time() - t0 < 3, "something broken in concurrency") + def test_encoding_name(self): + self.conn.set_client_encoding("EUC_JP") + # conn.encoding is 'EUCJP' now. + cur = self.conn.cursor() + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, cur) + cur.execute("select 'foo'::text;") + self.assertEqual(cur.fetchone()[0], u'foo') + class IsolationLevelsTestCase(unittest.TestCase): From 657bcb48283932274132b8676d5362c46f905cbd Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 14 Dec 2010 03:10:23 +0000 Subject: [PATCH 15/80] Encodings mapping reordered in a more maintainable order. --- psycopg/psycopgmodule.c | 76 ++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 8f9f2d1a..f4f57417 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -325,7 +325,28 @@ psyco_adapters_init(PyObject *mod) Fill the module's postgresql<->python encoding table */ static encodingPair encodings[] = { - {"SQL_ASCII", "ascii"}, + {"ABC", "cp1258"}, + {"ALT", "cp866"}, + {"BIG5", "big5"}, + {"EUC_JP", "euc_jp"}, + {"EUC_KR", "euc_kr"}, + {"GB18030", "gb18030"}, + {"GBK", "gbk"}, + {"ISO88591", "iso8859_1"}, + {"ISO885913", "iso8859_13"}, + {"ISO885914", "iso8859_14"}, + {"ISO885915", "iso8859_15"}, + {"ISO88592", "iso8859_2"}, + {"ISO88593", "iso8859_3"}, + {"ISO88595", "iso8859_5"}, + {"ISO88596", "iso8859_6"}, + {"ISO88597", "iso8859_7"}, + {"ISO88598", "iso8859_8"}, + {"ISO88599", "iso8859_9"}, + {"JOHAB", "johab"}, + {"KOI8", "koi8_r"}, + {"KOI8R", "koi8_r"}, + {"KOI8U", "koi8_u"}, {"LATIN1", "iso8859_1"}, {"LATIN2", "iso8859_2"}, {"LATIN3", "iso8859_3"}, @@ -335,46 +356,25 @@ static encodingPair encodings[] = { {"LATIN7", "iso8859_13"}, {"LATIN8", "iso8859_14"}, {"LATIN9", "iso8859_15"}, - {"ISO88591", "iso8859_1"}, - {"ISO88592", "iso8859_2"}, - {"ISO88593", "iso8859_3"}, - {"ISO88595", "iso8859_5"}, - {"ISO88596", "iso8859_6"}, - {"ISO88597", "iso8859_7"}, - {"ISO885913", "iso8859_13"}, - {"ISO88598", "iso8859_8"}, - {"ISO88599", "iso8859_9"}, - {"ISO885914", "iso8859_14"}, - {"ISO885915", "iso8859_15"}, - {"UNICODE", "utf_8"}, /* Not valid in 8.2, backward compatibility */ - {"UTF8", "utf_8"}, - {"WIN950", "cp950"}, - {"Windows950", "cp950"}, - {"BIG5", "big5"}, - {"EUC_JP", "euc_jp"}, - {"EUC_KR", "euc_kr"}, - {"GB18030", "gb18030"}, - {"GBK", "gbk"}, - {"WIN936", "gbk"}, - {"Windows936", "gbk"}, - {"JOHAB", "johab"}, - {"KOI8", "koi8_r"}, /* in PG: KOI8 == KOI8R == KOI8-R == KOI8-U - but in Python there is koi8_r AND koi8_u */ - {"KOI8R", "koi8_r"}, - {"SJIS", "cp932"}, {"Mskanji", "cp932"}, {"ShiftJIS", "cp932"}, - {"WIN932", "cp932"}, - {"Windows932", "cp932"}, + {"SJIS", "cp932"}, + {"SQL_ASCII", "ascii"}, + {"TCVN", "cp1258"}, + {"TCVN5712", "cp1258"}, {"UHC", "cp949"}, - {"WIN949", "cp949"}, - {"Windows949", "cp949"}, + {"UNICODE", "utf_8"}, /* Not valid in 8.2, backward compatibility */ + {"UTF8", "utf_8"}, + {"VSCII", "cp1258"}, + {"WIN", "cp1251"}, {"WIN866", "cp866"}, - {"ALT", "cp866"}, {"WIN874", "cp874"}, + {"WIN932", "cp932"}, + {"WIN936", "gbk"}, + {"WIN949", "cp949"}, + {"WIN950", "cp950"}, {"WIN1250", "cp1250"}, {"WIN1251", "cp1251"}, - {"WIN", "cp1251"}, {"WIN1252", "cp1252"}, {"WIN1253", "cp1253"}, {"WIN1254", "cp1254"}, @@ -382,10 +382,10 @@ static encodingPair encodings[] = { {"WIN1256", "cp1256"}, {"WIN1257", "cp1257"}, {"WIN1258", "cp1258"}, - {"ABC", "cp1258"}, - {"TCVN", "cp1258"}, - {"TCVN5712", "cp1258"}, - {"VSCII", "cp1258"}, + {"Windows932", "cp932"}, + {"Windows936", "gbk"}, + {"Windows949", "cp949"}, + {"Windows950", "cp950"}, /* those are missing from Python: */ /* {"EUC_CN", "?"}, */ From 7b5d80d36d47bcbd04038844902b5786e656f6f5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 14 Dec 2010 03:32:30 +0000 Subject: [PATCH 16/80] Added a few missing encodings. EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, LATIN10, SHIFT_JIS_2004. --- ChangeLog | 3 +++ NEWS-2.3 | 8 ++++++++ psycopg/psycopgmodule.c | 36 +++++++++++++++++++++--------------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/ChangeLog b/ChangeLog index 11081e5e..06d72523 100644 --- a/ChangeLog +++ b/ChangeLog @@ -2,6 +2,9 @@ * lib/extensions.py: Improved mapping from PG to Py encodings. + * psycopg/psycopgmodule.c: Added a few missing encodings: EUC_CN, + EUC_JIS_2004, ISO885910, ISO885916, LATIN10, SHIFT_JIS_2004. + 2010-12-04 Daniele Varrazzo * setup.py: bumped to version 2.3.1.dev0 diff --git a/NEWS-2.3 b/NEWS-2.3 index d8208d9b..312d80e9 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -1,3 +1,11 @@ +What's new in psycopg 2.3.2 +--------------------------- + + - Improved PostgreSQL-Python encodings mapping. Added a few + missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, + LATIN10, SHIFT_JIS_2004. + + What's new in psycopg 2.3.1 --------------------------- diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index f4f57417..9b3f54cd 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -328,21 +328,25 @@ static encodingPair encodings[] = { {"ABC", "cp1258"}, {"ALT", "cp866"}, {"BIG5", "big5"}, + {"EUC_CN", "euccn"}, + {"EUC_JIS_2004", "euc_jis_2004"}, {"EUC_JP", "euc_jp"}, {"EUC_KR", "euc_kr"}, {"GB18030", "gb18030"}, {"GBK", "gbk"}, - {"ISO88591", "iso8859_1"}, - {"ISO885913", "iso8859_13"}, - {"ISO885914", "iso8859_14"}, - {"ISO885915", "iso8859_15"}, - {"ISO88592", "iso8859_2"}, - {"ISO88593", "iso8859_3"}, - {"ISO88595", "iso8859_5"}, - {"ISO88596", "iso8859_6"}, - {"ISO88597", "iso8859_7"}, - {"ISO88598", "iso8859_8"}, - {"ISO88599", "iso8859_9"}, + {"ISO_8859_1", "iso8859_1"}, + {"ISO_8859_2", "iso8859_2"}, + {"ISO_8859_3", "iso8859_3"}, + {"ISO_8859_5", "iso8859_5"}, + {"ISO_8859_6", "iso8859_6"}, + {"ISO_8859_7", "iso8859_7"}, + {"ISO_8859_8", "iso8859_8"}, + {"ISO_8859_9", "iso8859_9"}, + {"ISO_8859_10", "iso8859_10"}, + {"ISO_8859_13", "iso8859_13"}, + {"ISO_8859_14", "iso8859_14"}, + {"ISO_8859_15", "iso8859_15"}, + {"ISO_8859_16", "iso8859_16"}, {"JOHAB", "johab"}, {"KOI8", "koi8_r"}, {"KOI8R", "koi8_r"}, @@ -356,10 +360,15 @@ static encodingPair encodings[] = { {"LATIN7", "iso8859_13"}, {"LATIN8", "iso8859_14"}, {"LATIN9", "iso8859_15"}, + {"LATIN10", "iso8859_16"}, {"Mskanji", "cp932"}, {"ShiftJIS", "cp932"}, + {"SHIFT_JIS_2004", "shift_jis_2004"}, {"SJIS", "cp932"}, - {"SQL_ASCII", "ascii"}, + {"SQL_ASCII", "ascii"}, /* XXX this is wrong: SQL_ASCII means "no + * encoding" we should fix the unicode + * typecaster to return a str or bytes in Py3 + */ {"TCVN", "cp1258"}, {"TCVN5712", "cp1258"}, {"UHC", "cp949"}, @@ -388,10 +397,7 @@ static encodingPair encodings[] = { {"Windows950", "cp950"}, /* those are missing from Python: */ -/* {"EUC_CN", "?"}, */ /* {"EUC_TW", "?"}, */ -/* {"LATIN10", "?"}, */ -/* {"ISO885916", "?"}, */ /* {"MULE_INTERNAL", "?"}, */ {NULL, NULL} }; From a50a91fc7bbce34cca6db5d89cdcb0db7abd3ab6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 14 Dec 2010 19:49:27 +0000 Subject: [PATCH 17/80] No need to put connection fields to zero: tp_alloc already did. --- ChangeLog | 2 ++ psycopg/connection_type.c | 9 +-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/ChangeLog b/ChangeLog index 06d72523..676cded3 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,7 @@ 2010-12-14 Daniele Varrazzo + * psycopg/connection_type.c: No need to put connection fields to zero. + * lib/extensions.py: Improved mapping from PG to Py encodings. * psycopg/psycopgmodule.c: Added a few missing encodings: EUC_CN, diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 21e5e8e2..7bfdd347 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -849,19 +849,12 @@ connection_setup(connectionObject *self, const char *dsn, long int async) self->dsn = strdup(dsn); self->notice_list = PyList_New(0); self->notifies = PyList_New(0); - self->closed = 0; self->async = async; self->status = CONN_STATUS_SETUP; - self->critical = NULL; - self->async_cursor = NULL; self->async_status = ASYNC_DONE; - self->pgconn = NULL; - self->cancel = NULL; - self->mark = 0; self->string_types = PyDict_New(); self->binary_types = PyDict_New(); - self->notice_pending = NULL; - self->encoding = NULL; + /* other fields have been zeroed by tp_alloc */ pthread_mutex_init(&(self->lock), NULL); From ae06fb03e75c47f662606490203f365be26afd46 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 15 Dec 2010 03:07:13 +0000 Subject: [PATCH 18/80] Added psycopg_strdup utility function. --- ChangeLog | 4 ++++ psycopg/psycopg.h | 2 ++ psycopg/utils.c | 22 ++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/ChangeLog b/ChangeLog index 676cded3..ac6cc66c 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +2010-12-15 Daniele Varrazzo + + * psycopg/utils.c: Added psycopg_strdup function. + 2010-12-14 Daniele Varrazzo * psycopg/connection_type.c: No need to put connection fields to zero. diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index d023e47e..00cb98ad 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -120,6 +120,8 @@ HIDDEN void psyco_set_error(PyObject *exc, PyObject *curs, const char *msg, HIDDEN char *psycopg_escape_string(PyObject *conn, const char *from, Py_ssize_t len, char *to, Py_ssize_t *tolen); +HIDDEN char *psycopg_strdup(const char *from, Py_ssize_t len); + /* Exceptions docstrings */ #define Error_doc \ "Base class for error exceptions." diff --git a/psycopg/utils.c b/psycopg/utils.c index 22464e60..f2a0ab09 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -70,3 +70,25 @@ psycopg_escape_string(PyObject *obj, const char *from, Py_ssize_t len, return to; } + +/* Duplicate a string. + * + * Allocate a new buffer on the Python heap containing the new string. + * 'len' is optional: if 0 the length is calculated. + * + * Return NULL and set an exception in case of error. + */ +char * +psycopg_strdup(const char *from, Py_ssize_t len) +{ + char *rv; + + if (!len) { len = strlen(from); } + if (!(rv = PyMem_Malloc(len + 1))) { + PyErr_NoMemory(); + return NULL; + } + strcpy(rv, from); + return rv; +} + From e182201e6ee8075a88b63c9d8338f833bd2471b0 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 18 Dec 2010 05:02:11 +0000 Subject: [PATCH 19/80] Added Python codec name to the connection. This allows dropping repeated dictionary lookups with unicode query/parameters. --- ChangeLog | 5 ++ NEWS-2.3 | 1 + psycopg/adapter_qstring.c | 23 ++----- psycopg/adapter_qstring.h | 4 ++ psycopg/connection.h | 1 + psycopg/connection_int.c | 126 ++++++++++++++++++++++++++++---------- psycopg/connection_type.c | 40 ++++++------ psycopg/cursor_type.c | 20 ++---- psycopg/typecast_basic.c | 15 +---- 9 files changed, 140 insertions(+), 95 deletions(-) diff --git a/ChangeLog b/ChangeLog index ac6cc66c..844062e6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +2010-12-18 Daniele Varrazzo + + * connection.h: added codec attribute to avoid repeated codec name + lookups during unicode query/params manipulations. + 2010-12-15 Daniele Varrazzo * psycopg/utils.c: Added psycopg_strdup function. diff --git a/NEWS-2.3 b/NEWS-2.3 index 312d80e9..0e2229a1 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -4,6 +4,7 @@ What's new in psycopg 2.3.2 - Improved PostgreSQL-Python encodings mapping. Added a few missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, LATIN10, SHIFT_JIS_2004. + - Dropped repeated dictionary lookups with unicode query/parameters. What's new in psycopg 2.3.1 diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 9da6a212..72240c84 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -49,22 +49,9 @@ qstring_quote(qstringObject *self) Dprintf("qstring_quote: encoding to %s", self->encoding); if (PyUnicode_Check(self->wrapped) && self->encoding) { - PyObject *enc = PyDict_GetItemString(psycoEncodings, self->encoding); - /* note that enc is a borrowed reference */ - - if (enc) { - const char *s = PyString_AsString(enc); - Dprintf("qstring_quote: encoding unicode object to %s", s); - str = PyUnicode_AsEncodedString(self->wrapped, s, NULL); - Dprintf("qstring_quote: got encoded object at %p", str); - if (str == NULL) return NULL; - } - else { - /* can't find the right encoder, raise exception */ - PyErr_Format(InterfaceError, - "can't encode unicode string to %s", self->encoding); - return NULL; - } + str = PyUnicode_AsEncodedString(self->wrapped, self->encoding, NULL); + Dprintf("qstring_quote: got encoded object at %p", str); + if (str == NULL) return NULL; } /* if the wrapped object is a simple string, we don't know how to @@ -144,8 +131,8 @@ qstring_prepare(qstringObject *self, PyObject *args) we don't need the encoding if that's not the case */ if (PyUnicode_Check(self->wrapped)) { if (self->encoding) free(self->encoding); - self->encoding = strdup(conn->encoding); - Dprintf("qstring_prepare: set encoding to %s", conn->encoding); + self->encoding = strdup(conn->codec); + Dprintf("qstring_prepare: set encoding to %s", conn->codec); } Py_CLEAR(self->conn); diff --git a/psycopg/adapter_qstring.h b/psycopg/adapter_qstring.h index db986be3..d825fe0e 100644 --- a/psycopg/adapter_qstring.h +++ b/psycopg/adapter_qstring.h @@ -37,6 +37,10 @@ typedef struct { PyObject *wrapped; PyObject *buffer; + /* NOTE: this used to be a PostgreSQL encoding: changed in 2.3.2 to be a + * Python codec name. I don't expect there has been any user for this + * object other than adapting str/unicode, so I don't expect client code + * broken for this reason. */ char *encoding; PyObject *conn; diff --git a/psycopg/connection.h b/psycopg/connection.h index 477c6901..41e0cdc1 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -83,6 +83,7 @@ typedef struct { char *dsn; /* data source name */ char *critical; /* critical error on this connection */ char *encoding; /* current backend encoding */ + char *codec; /* python codec name for encoding */ long int closed; /* 1 means connection has been closed; 2 that something horrible happened */ diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index e04d5104..aa4eca70 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -212,38 +212,89 @@ conn_get_standard_conforming_strings(PGconn *pgconn) return equote; } -/* Return a string containing the client_encoding setting. +/* Convert a PostgreSQL encoding to a Python codec. * - * Return a new string allocated by malloc(): use free() to free it. - * Return NULL in case of failure. + * Return a new copy of the codec name allocated on the Python heap, + * NULL with exception in case of error. */ static char * -conn_get_encoding(PGconn *pgconn) +conn_encoding_to_codec(const char *enc) { - const char *tmp, *i; - char *encoding, *j; + char *tmp; + Py_ssize_t size; + PyObject *pyenc; + char *rv = NULL; + + if (!(pyenc = PyDict_GetItemString(psycoEncodings, enc))) { + PyErr_Format(OperationalError, + "no Python codec for client encoding '%s'", enc); + goto exit; + } + if (-1 == PyString_AsStringAndSize(pyenc, &tmp, &size)) { + goto exit; + } + + /* have our own copy of the python codec name */ + rv = psycopg_strdup(tmp, size); + +exit: + /* pyenc is borrowed: no decref. */ + return rv; +} + +/* Read the client encoding from the connection. + * + * Store the encoding in the pgconn->encoding field and the name of the + * matching python codec in codec. The buffers are allocated on the Python + * heap. + * + * Return 0 on success, else nonzero. + */ +static int +conn_read_encoding(connectionObject *self, PGconn *pgconn) +{ + char *enc = NULL, *codec = NULL, *j; + const char *tmp; + int rv = -1; tmp = PQparameterStatus(pgconn, "client_encoding"); Dprintf("conn_connect: client encoding: %s", tmp ? tmp : "(none)"); if (!tmp) { PyErr_SetString(OperationalError, "server didn't return client encoding"); - return NULL; + goto exit; } - encoding = malloc(strlen(tmp)+1); - if (encoding == NULL) { + if (!(enc = PyMem_Malloc(strlen(tmp)+1))) { PyErr_NoMemory(); - return NULL; + goto exit; } - /* return in uppercase */ - i = tmp; - j = encoding; - while (*i) { *j++ = toupper(*i++); } + /* turn encoding in uppercase */ + j = enc; + while (*tmp) { *j++ = toupper(*tmp++); } *j = '\0'; - return encoding; + /* Look for this encoding in Python codecs. */ + if (!(codec = conn_encoding_to_codec(enc))) { + goto exit; + } + + /* Good, success: store the encoding/codec in the connection. */ + PyMem_Free(self->encoding); + self->encoding = enc; + enc = NULL; + + PyMem_Free(self->codec); + self->codec = codec; + codec = NULL; + + rv = 0; + +exit: + PyMem_Free(enc); + PyMem_Free(codec); + return rv; } int @@ -319,9 +370,8 @@ conn_setup(connectionObject *self, PGconn *pgconn) PyErr_SetString(InterfaceError, "only protocol 3 supported"); return -1; } - /* conn_get_encoding returns a malloc'd string */ - self->encoding = conn_get_encoding(pgconn); - if (self->encoding == NULL) { + + if (conn_read_encoding(self, pgconn)) { return -1; } @@ -651,9 +701,7 @@ _conn_poll_setup_async(connectionObject *self) PyErr_SetString(InterfaceError, "only protocol 3 supported"); break; } - /* conn_get_encoding returns a malloc'd string */ - self->encoding = conn_get_encoding(self->pgconn); - if (self->encoding == NULL) { + if (conn_read_encoding(self, self->pgconn)) { break; } self->cancel = conn_get_cancel(self->pgconn); @@ -873,11 +921,15 @@ conn_set_client_encoding(connectionObject *self, const char *enc) char *error = NULL; char query[48]; int res = 0; + char *codec; /* If the current encoding is equal to the requested one we don't issue any query to the backend */ if (strcmp(self->encoding, enc) == 0) return 0; + /* We must know what python codec this encoding is. */ + if (!(codec = conn_encoding_to_codec(enc))) { return -1; } + Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); @@ -886,19 +938,29 @@ conn_set_client_encoding(connectionObject *self, const char *enc) /* abort the current transaction, to set the encoding ouside of transactions */ - res = pq_abort_locked(self, &pgres, &error, &_save); - - if (res == 0) { - res = pq_execute_command_locked(self, query, &pgres, &error, &_save); - if (res == 0) { - /* no error, we can proceeed and store the new encoding */ - if (self->encoding) free(self->encoding); - self->encoding = strdup(enc); - Dprintf("conn_set_client_encoding: set encoding to %s", - self->encoding); - } + if ((res = pq_abort_locked(self, &pgres, &error, &_save))) { + goto endlock; } + if ((res = pq_execute_command_locked(self, query, &pgres, &error, &_save))) { + goto endlock; + } + + /* no error, we can proceeed and store the new encoding */ + PyMem_Free(self->encoding); + if (!(self->encoding = psycopg_strdup(enc, 0))) { + res = 1; /* don't call pq_complete_error below */ + goto endlock; + } + + /* Store the python codec too. */ + PyMem_Free(self->codec); + self->codec = codec; + + Dprintf("conn_set_client_encoding: set encoding to %s (codec: %s)", + self->encoding, self->codec); + +endlock: pthread_mutex_unlock(&self->lock); Py_END_ALLOW_THREADS; diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 7bfdd347..9e799bee 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -425,35 +425,38 @@ psyco_conn_set_isolation_level(connectionObject *self, PyObject *args) static PyObject * psyco_conn_set_client_encoding(connectionObject *self, PyObject *args) { - const char *enc = NULL; - char *buffer; - size_t i, j; + const char *enc; + char *buffer, *dest; + PyObject *rv = NULL; + Py_ssize_t len; EXC_IF_CONN_CLOSED(self); EXC_IF_CONN_ASYNC(self, set_client_encoding); EXC_IF_TPC_PREPARED(self, set_client_encoding); - if (!PyArg_ParseTuple(args, "s", &enc)) return NULL; + if (!PyArg_ParseTuple(args, "s#", &enc, &len)) return NULL; /* convert to upper case and remove '-' and '_' from string */ - buffer = PyMem_Malloc(strlen(enc)+1); - for (i=j=0 ; i < strlen(enc) ; i++) { - if (enc[i] == '_' || enc[i] == '-') - continue; - else - buffer[j++] = toupper(enc[i]); + if (!(dest = buffer = PyMem_Malloc(len+1))) { + return PyErr_NoMemory(); } - buffer[j] = '\0'; + + while (*enc) { + if (*enc == '_' || *enc == '-') { + ++enc; + } + else { + *dest++ = toupper(*enc++); + } + } + *dest = '\0'; if (conn_set_client_encoding(self, buffer) == 0) { - PyMem_Free(buffer); Py_INCREF(Py_None); - return Py_None; - } - else { - PyMem_Free(buffer); - return NULL; + rv = Py_None; } + PyMem_Free(buffer); + return rv; } /* get_transaction_status method - Get backend transaction status */ @@ -892,7 +895,8 @@ connection_dealloc(PyObject* obj) conn_notice_clean(self); if (self->dsn) free(self->dsn); - if (self->encoding) free(self->encoding); + PyMem_Free(self->encoding); + PyMem_Free(self->codec); if (self->critical) free(self->critical); Py_CLEAR(self->async_cursor); diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index cd62d508..d9e925f1 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -272,21 +272,11 @@ static PyObject *_psyco_curs_validate_sql_basic( Py_INCREF(sql); } else if (PyUnicode_Check(sql)) { - PyObject *enc = PyDict_GetItemString(psycoEncodings, - self->conn->encoding); - /* enc is a borrowed reference; we won't decref it */ - - if (enc) { - sql = PyUnicode_AsEncodedString(sql, PyString_AsString(enc), NULL); - /* if there was an error during the encoding from unicode to the - target encoding, we just let the exception propagate */ - if (sql == NULL) { goto fail; } - } else { - PyErr_Format(InterfaceError, - "can't encode unicode SQL statement to %s", - self->conn->encoding); - goto fail; - } + char *enc = self->conn->codec; + sql = PyUnicode_AsEncodedString(sql, enc, NULL); + /* if there was an error during the encoding from unicode to the + target encoding, we just let the exception propagate */ + if (sql == NULL) { goto fail; } } else { /* the is not unicode or string, raise an error */ diff --git a/psycopg/typecast_basic.c b/psycopg/typecast_basic.c index e9ad527a..634fc456 100644 --- a/psycopg/typecast_basic.c +++ b/psycopg/typecast_basic.c @@ -82,21 +82,12 @@ typecast_STRING_cast(const char *s, Py_ssize_t len, PyObject *curs) static PyObject * typecast_UNICODE_cast(const char *s, Py_ssize_t len, PyObject *curs) { - PyObject *enc; + char *enc; if (s == NULL) {Py_INCREF(Py_None); return Py_None;} - enc = PyDict_GetItemString(psycoEncodings, - ((cursorObject*)curs)->conn->encoding); - if (enc) { - return PyUnicode_Decode(s, len, PyString_AsString(enc), NULL); - } - else { - PyErr_Format(InterfaceError, - "can't decode into unicode string from %s", - ((cursorObject*)curs)->conn->encoding); - return NULL; - } + enc = ((cursorObject*)curs)->conn->codec; + return PyUnicode_Decode(s, len, enc, NULL); } /** BOOLEAN - cast boolean value into right python object **/ From cb6b52945b4c114d39613e6a7b35c73762e5045f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 12 Dec 2010 21:48:54 +0000 Subject: [PATCH 20/80] The library can be compiled with Python 3. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just compiled! No test run yet and many points to review, marked in the code. The patch is largely Martin von Löwis work, simplified after refactoring in the previous commits and adapted to the new code (as the patch was originally for Psycopg 2.0.9) --- psycopg/adapter_asis.c | 2 +- psycopg/adapter_binary.c | 10 ++- psycopg/adapter_datetime.c | 16 +++-- psycopg/adapter_list.c | 10 ++- psycopg/adapter_pboolean.c | 8 +-- psycopg/adapter_pdecimal.c | 6 +- psycopg/adapter_pfloat.c | 4 +- psycopg/adapter_qstring.c | 7 +- psycopg/connection_int.c | 35 +++++++-- psycopg/connection_type.c | 2 +- psycopg/cursor_type.c | 36 ++++++---- psycopg/lobject_type.c | 2 +- psycopg/microprotocols.c | 8 ++- psycopg/notify_type.c | 7 +- psycopg/pqpath.c | 11 +-- psycopg/psycopgmodule.c | 143 +++++++++++++++++++++++++++---------- psycopg/python.h | 62 ++++++++++++++++ psycopg/typecast.c | 24 +++---- psycopg/typecast_basic.c | 17 ++++- psycopg/typecast_binary.c | 24 +++++++ psycopg/xid_type.c | 16 ++--- 21 files changed, 335 insertions(+), 115 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index 0d0a7f67..1aa1d0da 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -38,7 +38,7 @@ static PyObject * asis_str(asisObject *self) { if (self->wrapped == Py_None) { - return PyString_FromString("NULL"); + return Text_FromUTF8("NULL"); } else { return PyObject_Str(self->wrapped); diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index cf31ccbf..22105070 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -58,7 +58,13 @@ binary_quote(binaryObject *self) size_t len = 0; /* if we got a plain string or a buffer we escape it and save the buffer */ - if (PyString_Check(self->wrapped) || PyBuffer_Check(self->wrapped)) { + if (Bytes_Check(self->wrapped) +#if PY_MAJOR_VERSION < 3 + || PyBuffer_Check(self->wrapped) +#else + || PyMemoryView_Check(self->wrapped) +#endif + ) { /* escape and build quoted buffer */ if (PyObject_AsReadBuffer(self->wrapped, (const void **)&buffer, &buffer_len) < 0) @@ -76,7 +82,7 @@ binary_quote(binaryObject *self) (self->conn && ((connectionObject*)self->conn)->equote) ? "E'%s'::bytea" : "'%s'::bytea" , to); else - self->buffer = PyString_FromString("''::bytea"); + self->buffer = Text_FromUTF8("''::bytea"); PQfreemem(to); } diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index f640684a..19fd97e2 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -62,26 +62,30 @@ pydatetime_str(pydatetimeObject *self) if (self->type <= PSYCO_DATETIME_TIMESTAMP) { PyObject *tz; - /* Select the right PG type to cast into. */ + /* Select the right PG type to cast into. + * fmt contains %s in Py2 and %U in Py3, + * So it can be used with the Text_FromFormatS macro */ char *fmt = NULL; switch (self->type) { case PSYCO_DATETIME_TIME: - fmt = "'%s'::time"; + fmt = "'" Text_S "'::time"; break; case PSYCO_DATETIME_DATE: - fmt = "'%s'::date"; + fmt = "'" Text_S "'::date"; break; case PSYCO_DATETIME_TIMESTAMP: tz = PyObject_GetAttrString(self->wrapped, "tzinfo"); if (!tz) { return NULL; } - fmt = (tz == Py_None) ? "'%s'::timestamp" : "'%s'::timestamptz"; + fmt = (tz == Py_None) + ? "'" Text_S "'::timestamp" + : "'" Text_S "'::timestamptz"; Py_DECREF(tz); break; } iso = PyObject_CallMethod(self->wrapped, "isoformat", NULL); if (iso) { - res = PyString_FromFormat(fmt, PyString_AsString(iso)); + res = Text_FromFormatS(fmt, iso); Py_DECREF(iso); } return res; @@ -89,7 +93,7 @@ pydatetime_str(pydatetimeObject *self) else { PyDateTime_Delta *obj = (PyDateTime_Delta*)self->wrapped; - char buffer[8]; + char buffer[8]; int i; int a = obj->microseconds; diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 8bd66881..8973e9dc 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -45,7 +45,7 @@ list_quote(listObject *self) /* empty arrays are converted to NULLs (still searching for a way to insert an empty array in postgresql */ - if (len == 0) return PyString_FromString("'{}'"); + if (len == 0) return Text_FromUTF8("'{}'"); tmp = PyTuple_New(len); @@ -53,7 +53,7 @@ list_quote(listObject *self) PyObject *quoted; PyObject *wrapped = PyList_GET_ITEM(self->wrapped, i); if (wrapped == Py_None) - quoted = PyString_FromString("NULL"); + quoted = Text_FromUTF8("NULL"); else quoted = microprotocol_getquoted(wrapped, (connectionObject*)self->connection); @@ -67,11 +67,15 @@ list_quote(listObject *self) /* now that we have a tuple of adapted objects we just need to join them and put "ARRAY[] around the result */ - str = PyString_FromString(", "); + str = Text_FromUTF8(", "); joined = PyObject_CallMethod(str, "join", "(O)", tmp); if (joined == NULL) goto error; +#if PY_MAJOR_VERSION < 3 res = PyString_FromFormat("ARRAY[%s]", PyString_AsString(joined)); +#else + res = PyUnicode_FromFormat("ARRAY[%U]", joined); +#endif error: Py_XDECREF(tmp); diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index a74d9f37..cdd3ef4e 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -39,17 +39,17 @@ pboolean_str(pbooleanObject *self) { #ifdef PSYCOPG_NEW_BOOLEAN if (PyObject_IsTrue(self->wrapped)) { - return PyString_FromString("true"); + return Text_FromUTF8("true"); } else { - return PyString_FromString("false"); + return Text_FromUTF8("false"); } #else if (PyObject_IsTrue(self->wrapped)) { - return PyString_FromString("'t'"); + return Text_FromUTF8("'t'"); } else { - return PyString_FromString("'f'"); + return Text_FromUTF8("'f'"); } #endif } diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index ed79ea1a..fa4acfab 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -45,7 +45,7 @@ pdecimal_str(pdecimalObject *self) goto end; } else if (check) { - res = PyString_FromString("'NaN'::numeric"); + res = Text_FromUTF8("'NaN'::numeric"); goto end; } @@ -57,7 +57,7 @@ pdecimal_str(pdecimalObject *self) goto end; } if (PyObject_IsTrue(check)) { - res = PyString_FromString("'NaN'::numeric"); + res = Text_FromUTF8("'NaN'::numeric"); goto end; } @@ -66,7 +66,7 @@ pdecimal_str(pdecimalObject *self) goto end; } if (PyObject_IsTrue(check)) { - res = PyString_FromString("'NaN'::numeric"); + res = Text_FromUTF8("'NaN'::numeric"); goto end; } diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index f6ad6361..3260d2cd 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -40,9 +40,9 @@ pfloat_str(pfloatObject *self) { double n = PyFloat_AsDouble(self->wrapped); if (isnan(n)) - return PyString_FromString("'NaN'::float"); + return Text_FromUTF8("'NaN'::float"); else if (isinf(n)) - return PyString_FromString("'Infinity'::float"); + return Text_FromUTF8("'Infinity'::float"); else return PyObject_Repr(self->wrapped); } diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 72240c84..c1082d8e 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -54,6 +54,7 @@ qstring_quote(qstringObject *self) if (str == NULL) return NULL; } +#if PY_MAJOR_VERSION < 3 /* if the wrapped object is a simple string, we don't know how to (re)encode it, so we pass it as-is */ else if (PyString_Check(self->wrapped)) { @@ -61,6 +62,7 @@ qstring_quote(qstringObject *self) /* INCREF to make it ref-wise identical to unicode one */ Py_INCREF(str); } +#endif /* if the wrapped object is not a string, this is an error */ else { @@ -70,7 +72,7 @@ qstring_quote(qstringObject *self) } /* encode the string into buffer */ - PyString_AsStringAndSize(str, &s, &len); + Bytes_AsStringAndSize(str, &s, &len); /* Call qstring_escape with the GIL released, then reacquire the GIL before verifying that the results can fit into a Python string; raise @@ -94,7 +96,8 @@ qstring_quote(qstringObject *self) return NULL; } - self->buffer = PyString_FromStringAndSize(buffer, qlen); + /* XXX need to decode in connection's encoding in 3.0 */ + self->buffer = Text_FromUTF8AndSize(buffer, qlen); PyMem_Free(buffer); Py_DECREF(str); diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index aa4eca70..21889938 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -76,7 +76,8 @@ conn_notice_process(connectionObject *self) while (notice != NULL) { PyObject *msg; - msg = PyString_FromString(notice->message); + /* XXX possible other encode I think */ + msg = Text_FromUTF8(notice->message); Dprintf("conn_notice_process: %s", notice->message); @@ -145,8 +146,9 @@ conn_notifies_process(connectionObject *self) (int) pgn->be_pid, pgn->relname); if (!(pid = PyInt_FromLong((long)pgn->be_pid))) { goto error; } - if (!(channel = PyString_FromString(pgn->relname))) { goto error; } - if (!(payload = PyString_FromString(pgn->extra))) { goto error; } + /* XXX in the connection encoding? */ + if (!(channel = Text_FromUTF8(pgn->relname))) { goto error; } + if (!(payload = Text_FromUTF8(pgn->extra))) { goto error; } if (!(notify = PyObject_CallFunctionObjArgs((PyObject *)&NotifyType, pid, channel, payload, NULL))) { @@ -222,15 +224,35 @@ conn_encoding_to_codec(const char *enc) { char *tmp; Py_ssize_t size; - PyObject *pyenc; + PyObject *pyenc = NULL; + PyObject *pybenc = NULL; char *rv = NULL; + /* Find the Py codec name from the PG encoding */ if (!(pyenc = PyDict_GetItemString(psycoEncodings, enc))) { PyErr_Format(OperationalError, "no Python codec for client encoding '%s'", enc); goto exit; } - if (-1 == PyString_AsStringAndSize(pyenc, &tmp, &size)) { + + /* Convert the codec in a bytes string to extract the c string. + * At the end of the block we have pybenc with a new ref. */ + if (PyUnicode_Check(pyenc)) { + if (!(pybenc = PyUnicode_AsEncodedString(pyenc, "ascii", NULL))) { + goto exit; + } + } + else if (Bytes_Check(pyenc)) { + Py_INCREF(pyenc); + pybenc = pyenc; + } + else { + PyErr_Format(PyExc_TypeError, "bad type for encoding: %s", + Py_TYPE(pyenc)->tp_name); + goto exit; + } + + if (-1 == Bytes_AsStringAndSize(pybenc, &tmp, &size)) { goto exit; } @@ -239,6 +261,7 @@ conn_encoding_to_codec(const char *enc) exit: /* pyenc is borrowed: no decref. */ + Py_XDECREF(pybenc); return rv; } @@ -1027,7 +1050,7 @@ conn_tpc_command(connectionObject *self, const char *cmd, XidObject *xid) /* convert the xid into PostgreSQL transaction id while keeping the GIL */ if (!(tid = xid_get_tid(xid))) { goto exit; } - if (!(ctid = PyString_AsString(tid))) { goto exit; } + if (!(ctid = Bytes_AsString(tid))) { goto exit; } Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&self->lock); diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 9e799bee..49c6b942 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -498,7 +498,7 @@ psyco_conn_get_parameter_status(connectionObject *self, PyObject *args) Py_INCREF(Py_None); return Py_None; } - return PyString_FromString(val); + return Text_FromUTF8(val); } diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index d9e925f1..4bf53139 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -87,7 +87,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) just before returning. we also init *new to NULL to exit with an error if we can't complete the mogrification */ n = *new = NULL; - c = PyString_AsString(fmt); + c = Bytes_AsString(fmt); while(*c) { /* handle plain percent symbol in format string */ @@ -116,7 +116,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) for (d = c + 2; *d && *d != ')'; d++); if (*d == ')') { - key = PyString_FromStringAndSize(c+2, (Py_ssize_t) (d-c-2)); + key = Bytes_FromStringAndSize(c+2, (Py_ssize_t) (d-c-2)); value = PyObject_GetItem(var, key); /* key has refcnt 1, value the original value + 1 */ @@ -144,7 +144,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) optimization over the adapting code and can go away in the future if somebody finds a None adapter usefull. */ if (value == Py_None) { - t = PyString_FromString("NULL"); + t = Text_FromUTF8("NULL"); PyDict_SetItem(n, key, t); /* t is a new object, refcnt = 1, key is at 2 */ @@ -220,7 +220,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) d = c+1; if (value == Py_None) { - PyTuple_SET_ITEM(n, index, PyString_FromString("NULL")); + PyTuple_SET_ITEM(n, index, Text_FromUTF8("NULL")); while (*d && !isalpha(*d)) d++; if (*d) *d = 's'; Py_DECREF(value); @@ -267,7 +267,7 @@ static PyObject *_psyco_curs_validate_sql_basic( goto fail; } - if (PyString_Check(sql)) { + if (Bytes_Check(sql)) { /* Necessary for ref-count symmetry with the unicode case: */ Py_INCREF(sql); } @@ -314,7 +314,7 @@ _psyco_curs_merge_query_args(cursorObject *self, the curren exception (we will later restore it if the type or the strings do not match.) */ - if (!(fquery = PyString_Format(query, args))) { + if (!(fquery = Text_Format(query, args))) { PyObject *err, *arg, *trace; int pe = 0; @@ -327,7 +327,7 @@ _psyco_curs_merge_query_args(cursorObject *self, if (PyObject_HasAttrString(arg, "args")) { PyObject *args = PyObject_GetAttrString(arg, "args"); PyObject *str = PySequence_GetItem(args, 0); - const char *s = PyString_AS_STRING(str); + const char *s = Bytes_AS_STRING(str); Dprintf("psyco_curs_execute: -> %s", s); @@ -397,9 +397,15 @@ _psyco_curs_execute(cursorObject *self, } if (self->name != NULL) { + #if PY_MAJOR_VERSION < 3 self->query = PyString_FromFormat( "DECLARE %s CURSOR WITHOUT HOLD FOR %s", self->name, PyString_AS_STRING(fquery)); + #else + self->query = PyUnicode_FromFormat( + "DECLARE %s CURSOR WITHOUT HOLD FOR %U", + self->name, fquery); + #endif Py_DECREF(fquery); } else { @@ -408,9 +414,15 @@ _psyco_curs_execute(cursorObject *self, } else { if (self->name != NULL) { + #if PY_MAJOR_VERSION < 3 self->query = PyString_FromFormat( "DECLARE %s CURSOR WITHOUT HOLD FOR %s", self->name, PyString_AS_STRING(operation)); + #else + self->query = PyUnicode_FromFormat( + "DECLARE %s CURSOR WITHOUT HOLD FOR %U", + self->name, operation); + #endif } else { /* Transfer reference ownership of the str in operation to @@ -423,7 +435,7 @@ _psyco_curs_execute(cursorObject *self, /* At this point, the SQL statement must be str, not unicode */ - res = pq_execute(self, PyString_AS_STRING(self->query), async); + res = pq_execute(self, Bytes_AS_STRING(self->query), async); Dprintf("psyco_curs_execute: res = %d, pgres = %p", res, self->pgres); if (res == -1) { goto fail; } @@ -950,7 +962,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args, PyObject *kwargs) sql[sl-2] = ')'; sql[sl-1] = '\0'; - operation = PyString_FromString(sql); + operation = Text_FromUTF8(sql); PyMem_Free((void*)sql); if (_psyco_curs_execute(self, operation, parameters, self->conn->async)) { @@ -1105,14 +1117,14 @@ static int _psyco_curs_copy_columns(PyObject *columns, char *columnlist) columnlist[0] = '('; while ((col = PyIter_Next(coliter)) != NULL) { - if (!PyString_Check(col)) { + if (!Bytes_Check(col)) { Py_DECREF(col); Py_DECREF(coliter); PyErr_SetString(PyExc_ValueError, "elements in column list must be strings"); return -1; } - PyString_AsStringAndSize(col, &colname, &collen); + Bytes_AsStringAndSize(col, &colname, &collen); if (offset + collen > DEFAULT_COPYBUFF - 2) { Py_DECREF(col); Py_DECREF(coliter); @@ -1417,7 +1429,7 @@ psyco_curs_copy_expert(cursorObject *self, PyObject *args, PyObject *kwargs) self->copyfile = file; /* At this point, the SQL statement must be str, not unicode */ - if (pq_execute(self, PyString_AS_STRING(sql), 0) != 1) { goto fail; } + if (pq_execute(self, Bytes_AS_STRING(sql), 0) != 1) { goto fail; } res = Py_None; Py_INCREF(res); diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index d07becbe..0feffbcb 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -119,7 +119,7 @@ psyco_lobj_read(lobjectObject *self, PyObject *args) return NULL; } - res = PyString_FromStringAndSize(buffer, size); + res = Bytes_FromStringAndSize(buffer, size); PyMem_Free(buffer); return res; diff --git a/psycopg/microprotocols.c b/psycopg/microprotocols.c index f41d85f1..067729f2 100644 --- a/psycopg/microprotocols.c +++ b/psycopg/microprotocols.c @@ -83,7 +83,11 @@ _get_superclass_adapter(PyObject *obj, PyObject *proto) Py_ssize_t i, ii; type = Py_TYPE(obj); - if (!((Py_TPFLAGS_HAVE_CLASS & type->tp_flags) && type->tp_mro)) { + if (!( +#if PY_MAJOR_VERSION < 3 + (Py_TPFLAGS_HAVE_CLASS & type->tp_flags) && +#endif + type->tp_mro)) { /* has no mro */ return NULL; } @@ -134,7 +138,7 @@ microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) /* None is always adapted to NULL */ if (obj == Py_None) - return PyString_FromString("NULL"); + return Text_FromUTF8("NULL"); Dprintf("microprotocols_adapt: trying to adapt %s", Py_TYPE(obj)->tp_name); diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index 3d2308ad..e68daa38 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -78,7 +78,8 @@ notify_init(NotifyObject *self, PyObject *args, PyObject *kwargs) } if (!payload) { - payload = PyString_FromStringAndSize("", 0); + /* XXX review encoding */ + payload = Text_FromUTF8AndSize("", 0); } Py_CLEAR(self->pid); @@ -213,7 +214,7 @@ notify_repr(NotifyObject *self) PyObject *format = NULL; PyObject *args = NULL; - if (!(format = PyString_FromString("Notify(%r, %r, %r)"))) { + if (!(format = Text_FromUTF8("Notify(%r, %r, %r)"))) { goto exit; } @@ -225,7 +226,7 @@ notify_repr(NotifyObject *self) Py_INCREF(self->payload); PyTuple_SET_ITEM(args, 2, self->payload); - rv = PyString_Format(format, args); + rv = Text_Format(format, args); exit: Py_XDECREF(args); diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index e15fc7ae..745919b9 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -980,14 +980,15 @@ _pq_fetch_tuples(cursorObject *curs) } Dprintf("_pq_fetch_tuples: using cast at %p (%s) for type %d", - cast, PyString_AS_STRING(((typecastObject*)cast)->name), + cast, Bytes_AS_STRING(((typecastObject*)cast)->name), PQftype(curs->pgres,i)); Py_INCREF(cast); PyTuple_SET_ITEM(curs->casts, i, cast); /* 1/ fill the other fields */ PyTuple_SET_ITEM(dtitem, 0, - PyString_FromString(PQfname(curs->pgres, i))); + /* XXX guaranteed to be ASCII/UTF8? */ + Text_FromUTF8(PQfname(curs->pgres, i))); PyTuple_SET_ITEM(dtitem, 1, type); /* 2/ display size is the maximum size of this field result tuples. */ @@ -1066,13 +1067,13 @@ _pq_copy_in_v3(cursorObject *curs) while (1) { o = PyObject_CallFunctionObjArgs(func, size, NULL); - if (!(o && PyString_Check(o) && (length = PyString_GET_SIZE(o)) != -1)) { + if (!(o && Bytes_Check(o) && (length = Bytes_GET_SIZE(o)) != -1)) { error = 1; } if (length == 0 || length > INT_MAX || error == 1) break; Py_BEGIN_ALLOW_THREADS; - res = PQputCopyData(curs->conn->pgconn, PyString_AS_STRING(o), + res = PQputCopyData(curs->conn->pgconn, Bytes_AS_STRING(o), /* Py_ssize_t->int cast was validated above */ (int) length); Dprintf("_pq_copy_in_v3: sent %d bytes of data; res = %d", @@ -1219,7 +1220,7 @@ pq_fetch(cursorObject *curs) /* backend status message */ Py_XDECREF(curs->pgstatus); - curs->pgstatus = PyString_FromString(PQcmdStatus(curs->pgres)); + curs->pgstatus = Text_FromUTF8(PQcmdStatus(curs->pgres)); switch(pgstatus) { diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 9b3f54cd..568cf6ec 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -126,17 +126,38 @@ psyco_connect(PyObject *self, PyObject *args, PyObject *keywds) return NULL; } +#if PY_MAJOR_VERSION < 3 if (pyport && PyString_Check(pyport)) { - PyObject *pyint = PyInt_FromString(PyString_AsString(pyport), NULL, 10); - if (!pyint) goto fail; - /* Must use PyInt_AsLong rather than PyInt_AS_LONG, because - * PyInt_FromString can return a PyLongObject: */ - iport = PyInt_AsLong(pyint); - Py_DECREF(pyint); + PyObject *pyint = PyInt_FromString(PyString_AsString(pyport), NULL, 10); + if (!pyint) goto fail; + /* Must use PyInt_AsLong rather than PyInt_AS_LONG, because + * PyInt_FromString can return a PyLongObject: */ + iport = PyInt_AsLong(pyint); + Py_DECREF(pyint); + if (iport == -1 && PyErr_Occurred()) + goto fail; } else if (pyport && PyInt_Check(pyport)) { - iport = PyInt_AsLong(pyport); + iport = PyInt_AsLong(pyport); + if (iport == -1 && PyErr_Occurred()) + goto fail; } +#else + if (pyport && PyUnicode_Check(pyport)) { + PyObject *pyint = PyObject_CallFunction((PyObject*)&PyLong_Type, + "Oi", pyport, 10); + if (!pyint) goto fail; + iport = PyLong_AsLong(pyint); + Py_DECREF(pyint); + if (iport == -1 && PyErr_Occurred()) + goto fail; + } + else if (pyport && PyLong_Check(pyport)) { + iport = PyLong_AsLong(pyport); + if (iport == -1 && PyErr_Occurred()) + goto fail; + } +#endif else if (pyport != NULL) { PyErr_SetString(PyExc_TypeError, "port must be a string or int"); goto fail; @@ -288,13 +309,23 @@ psyco_adapters_init(PyObject *mod) PyTypeObject *type; microprotocols_add(&PyFloat_Type, NULL, (PyObject*)&pfloatType); +#if PY_MAJOR_VERSION < 3 microprotocols_add(&PyInt_Type, NULL, (PyObject*)&asisType); +#endif microprotocols_add(&PyLong_Type, NULL, (PyObject*)&asisType); microprotocols_add(&PyBool_Type, NULL, (PyObject*)&pbooleanType); +#if PY_MAJOR_VERSION < 3 microprotocols_add(&PyString_Type, NULL, (PyObject*)&qstringType); +#endif microprotocols_add(&PyUnicode_Type, NULL, (PyObject*)&qstringType); +#if PY_MAJOR_VERSION < 3 microprotocols_add(&PyBuffer_Type, NULL, (PyObject*)&binaryType); +#else + microprotocols_add(&PyBytes_Type, NULL, (PyObject*)&binaryType); + microprotocols_add(&PyByteArray_Type, NULL, (PyObject*)&binaryType); + microprotocols_add(&PyMemoryView_Type, NULL, (PyObject*)&binaryType); +#endif microprotocols_add(&PyList_Type, NULL, (PyObject*)&listType); if ((type = (PyTypeObject*)psyco_GetDecimalType()) != NULL) @@ -407,7 +438,7 @@ static void psyco_encodings_fill(PyObject *dict) encodingPair *enc; for (enc = encodings; enc->pgenc != NULL; enc++) { - PyObject *value = PyString_FromString(enc->pyenc); + PyObject *value = Text_FromUTF8(enc->pyenc); PyDict_SetItemString(dict, enc->pgenc, value); Py_DECREF(value); } @@ -472,12 +503,18 @@ psyco_errors_init(void) dict = PyDict_New(); if (exctable[i].docstr) { - str = PyString_FromString(exctable[i].docstr); + str = Text_FromUTF8(exctable[i].docstr); PyDict_SetItemString(dict, "__doc__", str); } - if (exctable[i].base == 0) + if (exctable[i].base == 0) { + #if PY_MAJOR_VERSION < 3 base = PyExc_StandardError; + #else + /* StandardError is gone in 3.0 */ + base = NULL; + #endif + } else base = *exctable[i].base; @@ -546,13 +583,16 @@ psyco_set_error(PyObject *exc, PyObject *curs, const char *msg, if (err) { if (pgerror) { - t = PyString_FromString(pgerror); + /* XXX is this always ASCII? If not, it needs + to be decoded properly for Python 3. */ + t = Text_FromUTF8(pgerror); PyObject_SetAttrString(err, "pgerror", t); Py_DECREF(t); } if (pgcode) { - t = PyString_FromString(pgcode); + /* XXX likewise */ + t = Text_FromUTF8(pgcode); PyObject_SetAttrString(err, "pgcode", t); Py_DECREF(t); } @@ -703,12 +743,26 @@ static PyMethodDef psycopgMethods[] = { {NULL, NULL, 0, NULL} /* Sentinel */ }; +#if PY_MAJOR_VERSION > 2 +static struct PyModuleDef psycopgmodule = { + PyModuleDef_HEAD_INIT, + "_psycopg", + NULL, + -1, + psycopgMethods, + NULL, + NULL, + NULL, + NULL +}; +#endif + PyMODINIT_FUNC -init_psycopg(void) +INIT_MODULE(_psycopg)(void) { static void *PSYCOPG_API[PSYCOPG_API_pointers]; - PyObject *module, *dict; + PyObject *module = NULL, *dict; PyObject *c_api_object; #ifdef PSYCOPG_DEBUG @@ -734,36 +788,36 @@ init_psycopg(void) Py_TYPE(&NotifyType) = &PyType_Type; Py_TYPE(&XidType) = &PyType_Type; - if (PyType_Ready(&connectionType) == -1) return; - if (PyType_Ready(&cursorType) == -1) return; - if (PyType_Ready(&typecastType) == -1) return; - if (PyType_Ready(&qstringType) == -1) return; - if (PyType_Ready(&binaryType) == -1) return; - if (PyType_Ready(&isqlquoteType) == -1) return; - if (PyType_Ready(&pbooleanType) == -1) return; - if (PyType_Ready(&pfloatType) == -1) return; - if (PyType_Ready(&pdecimalType) == -1) return; - if (PyType_Ready(&asisType) == -1) return; - if (PyType_Ready(&listType) == -1) return; - if (PyType_Ready(&chunkType) == -1) return; - if (PyType_Ready(&NotifyType) == -1) return; - if (PyType_Ready(&XidType) == -1) return; + if (PyType_Ready(&connectionType) == -1) goto exit; + if (PyType_Ready(&cursorType) == -1) goto exit; + if (PyType_Ready(&typecastType) == -1) goto exit; + if (PyType_Ready(&qstringType) == -1) goto exit; + if (PyType_Ready(&binaryType) == -1) goto exit; + if (PyType_Ready(&isqlquoteType) == -1) goto exit; + if (PyType_Ready(&pbooleanType) == -1) goto exit; + if (PyType_Ready(&pfloatType) == -1) goto exit; + if (PyType_Ready(&pdecimalType) == -1) goto exit; + if (PyType_Ready(&asisType) == -1) goto exit; + if (PyType_Ready(&listType) == -1) goto exit; + if (PyType_Ready(&chunkType) == -1) goto exit; + if (PyType_Ready(&NotifyType) == -1) goto exit; + if (PyType_Ready(&XidType) == -1) goto exit; #ifdef PSYCOPG_EXTENSIONS Py_TYPE(&lobjectType) = &PyType_Type; - if (PyType_Ready(&lobjectType) == -1) return; + if (PyType_Ready(&lobjectType) == -1) goto exit; #endif /* import mx.DateTime module, if necessary */ #ifdef HAVE_MXDATETIME Py_TYPE(&mxdatetimeType) = &PyType_Type; - if (PyType_Ready(&mxdatetimeType) == -1) return; + if (PyType_Ready(&mxdatetimeType) == -1) goto exit; if (mxDateTime_ImportModuleAndAPI() != 0) { Dprintf("initpsycopg: why marc hide mx.DateTime again?!"); PyErr_SetString(PyExc_ImportError, "can't import mx.DateTime module"); - return; + goto exit; } - if (psyco_adapter_mxdatetime_init()) { return; } + if (psyco_adapter_mxdatetime_init()) { goto exit; } #endif /* import python builtin datetime module, if available */ @@ -771,22 +825,22 @@ init_psycopg(void) if (pyDateTimeModuleP == NULL) { Dprintf("initpsycopg: can't import datetime module"); PyErr_SetString(PyExc_ImportError, "can't import datetime module"); - return; + goto exit; } /* Initialize the PyDateTimeAPI everywhere is used */ PyDateTime_IMPORT; - if (psyco_adapter_datetime_init()) { return; } + if (psyco_adapter_datetime_init()) { goto exit; } Py_TYPE(&pydatetimeType) = &PyType_Type; - if (PyType_Ready(&pydatetimeType) == -1) return; + if (PyType_Ready(&pydatetimeType) == -1) goto exit; /* import psycopg2.tz anyway (TODO: replace with C-level module?) */ pyPsycopgTzModule = PyImport_ImportModule("psycopg2.tz"); if (pyPsycopgTzModule == NULL) { Dprintf("initpsycopg: can't import psycopg2.tz module"); PyErr_SetString(PyExc_ImportError, "can't import psycopg2.tz module"); - return; + goto exit; } pyPsycopgTzLOCAL = PyObject_GetAttrString(pyPsycopgTzModule, "LOCAL"); @@ -794,7 +848,13 @@ init_psycopg(void) PyObject_GetAttrString(pyPsycopgTzModule, "FixedOffsetTimezone"); /* initialize the module and grab module's dictionary */ +#if PY_MAJOR_VERSION < 3 module = Py_InitModule("_psycopg", psycopgMethods); +#else + module = PyModule_Create(&psycopgmodule); +#endif + if (!module) { goto exit; } + dict = PyModule_GetDict(module); /* initialize all the module's exported functions */ @@ -812,9 +872,9 @@ init_psycopg(void) /* set some module's parameters */ PyModule_AddStringConstant(module, "__version__", PSYCOPG_VERSION); PyModule_AddStringConstant(module, "__doc__", "psycopg PostgreSQL driver"); - PyModule_AddObject(module, "apilevel", PyString_FromString(APILEVEL)); + PyModule_AddObject(module, "apilevel", Text_FromUTF8(APILEVEL)); PyModule_AddObject(module, "threadsafety", PyInt_FromLong(THREADSAFETY)); - PyModule_AddObject(module, "paramstyle", PyString_FromString(PARAMSTYLE)); + PyModule_AddObject(module, "paramstyle", Text_FromUTF8(PARAMSTYLE)); /* put new types in module dictionary */ PyModule_AddObject(module, "connection", (PyObject*)&connectionType); @@ -866,4 +926,11 @@ init_psycopg(void) #endif Dprintf("initpsycopg: module initialization complete"); + +exit: +#if PY_MAJOR_VERSION > 2 + return module; +#else + return; +#endif } diff --git a/psycopg/python.h b/psycopg/python.h index b514c2c6..8fee7fd7 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -74,4 +74,66 @@ #define FORMAT_CODE_SIZE_T "%zu" #endif +/* Abstract from text type. Only supported for ASCII and UTF-8 */ +#if PY_MAJOR_VERSION < 3 +#define Text_Type PyString_Type +#define Text_Check(s) PyString_Check(s) +#define Text_Format(f,a) PyString_Format(f,a) +#define Text_FromUTF8(s) PyString_FromString(s) +#define Text_FromUTF8AndSize(s,n) PyString_FromStringAndSize(s,n) +/* f must contain exactly a %s placeholder */ +#define Text_FromFormatS(f,s) PyString_FromFormat(f, PyString_AsString(s)) +#define Text_S "%s" +#else +#define Text_Type PyUnicode_Type +#define Text_Check(s) PyUnicode_Check(s) +#define Text_Format(f,a) PyUnicode_Format(f,a) +#define Text_FromUTF8(s) PyUnicode_FromString(s) +#define Text_FromUTF8AndSize(s,n) PyUnicode_FromStringAndSize(s,n) +/* f must contain exactly a %U placeholder */ +#define Text_FromFormatS(f,s) PyUnicode_FromFormat(f, s) +#define Text_S "%U" +#endif + +#if PY_MAJOR_VERSION > 2 +#define PyInt_Type PyLong_Type +#define PyInt_AsLong PyLong_AsLong +#define PyInt_FromLong PyLong_FromLong +#define PyInt_FromSsize_t PyLong_FromSsize_t +#define PyString_FromFormat PyUnicode_FromFormat +#define Py_TPFLAGS_HAVE_ITER 0L +#define Py_TPFLAGS_HAVE_RICHCOMPARE 0L +#ifndef PyNumber_Int +#define PyNumber_Int PyNumber_Long +#endif +#endif /* PY_MAJOR_VERSION > 2 */ + +#if PY_MAJOR_VERSION < 3 +/* XXX BytesType -> Bytes_Type */ +#define BytesType PyString_Type +#define Bytes_Check PyString_Check +#define Bytes_AS_STRING PyString_AS_STRING +#define Bytes_GET_SIZE PyString_GET_SIZE +#define Bytes_Size PyString_Size +#define Bytes_AsString PyString_AsString +#define Bytes_AsStringAndSize PyString_AsStringAndSize +#define Bytes_FromStringAndSize PyString_FromStringAndSize +#else +#define BytesType PyBytes_Type +#define Bytes_Check PyBytes_Check +#define Bytes_AS_STRING PyBytes_AS_STRING +#define Bytes_GET_SIZE PyBytes_GET_SIZE +#define Bytes_Size PyBytes_Size +#define Bytes_AsString PyBytes_AsString +#define Bytes_AsStringAndSize PyBytes_AsStringAndSize +#define Bytes_FromStringAndSize PyBytes_FromStringAndSize +#endif + +/* Mangle the module name into the name of the module init function */ +#if PY_MAJOR_VERSION > 2 +#define INIT_MODULE(m) PyInit_ ## m +#else +#define INIT_MODULE(m) init ## m +#endif + #endif /* !defined(PSYCOPG_PYTHON_H) */ diff --git a/psycopg/typecast.c b/psycopg/typecast.c index 47c3c58b..a58f3a1d 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -429,24 +429,22 @@ typecast_del(void *self) static PyObject * typecast_call(PyObject *obj, PyObject *args, PyObject *kwargs) { - PyObject *string, *cursor; + char *string; + Py_ssize_t length; + PyObject *cursor; - if (!PyArg_ParseTuple(args, "OO", &string, &cursor)) { + if (!PyArg_ParseTuple(args, "z#O", &string, &length, &cursor)) { return NULL; } // If the string is not a string but a None value we're being called - // from a Python-defined caster. There is no need to convert, just - // return it. - - if (string == Py_None) { - Py_INCREF(string); - return string; + // from a Python-defined caster. + if (!string) { + Py_INCREF(Py_None); + return Py_None; } - return typecast_cast(obj, - PyString_AsString(string), PyString_Size(string), - cursor); + return typecast_cast(obj, string, length, cursor); } PyTypeObject typecastType = { @@ -560,7 +558,7 @@ typecast_from_python(PyObject *self, PyObject *args, PyObject *keywds) if (!PyArg_ParseTupleAndKeywords(args, keywds, "O!|O!OO", kwlist, &PyTuple_Type, &v, - &PyString_Type, &name, + &Text_Type, &name, &cast, &base)) { return NULL; } @@ -585,7 +583,7 @@ typecast_from_c(typecastObject_initlist *type, PyObject *dict) } } - name = PyString_FromString(type->name); + name = Text_FromUTF8(type->name); if (!name) goto end; while (type->values[len] != 0) len++; diff --git a/psycopg/typecast_basic.c b/psycopg/typecast_basic.c index 634fc456..20c7b81f 100644 --- a/psycopg/typecast_basic.c +++ b/psycopg/typecast_basic.c @@ -25,6 +25,7 @@ /** INTEGER - cast normal integers (4 bytes) to python int **/ +#if PY_MAJOR_VERSION < 3 static PyObject * typecast_INTEGER_cast(const char *s, Py_ssize_t len, PyObject *curs) { @@ -37,6 +38,9 @@ typecast_INTEGER_cast(const char *s, Py_ssize_t len, PyObject *curs) } return PyInt_FromString((char *)s, NULL, 0); } +#else +#define typecast_INTEGER_cast typecast_LONGINTEGER_cast +#endif /** LONGINTEGER - cast long integers (8 bytes) to python long **/ @@ -59,23 +63,30 @@ static PyObject * typecast_FLOAT_cast(const char *s, Py_ssize_t len, PyObject *curs) { PyObject *str = NULL, *flo = NULL; - char *pend; if (s == NULL) {Py_INCREF(Py_None); return Py_None;} - str = PyString_FromStringAndSize(s, len); - flo = PyFloat_FromString(str, &pend); + str = Text_FromUTF8AndSize(s, len); +#if PY_MAJOR_VERSION < 3 + flo = PyFloat_FromString(str, NULL); +#else + flo = PyFloat_FromString(str); +#endif Py_DECREF(str); return flo; } /** STRING - cast strings of any type to python string **/ +#if PY_MAJOR_VERSION < 3 static PyObject * typecast_STRING_cast(const char *s, Py_ssize_t len, PyObject *curs) { if (s == NULL) {Py_INCREF(Py_None); return Py_None;} return PyString_FromStringAndSize(s, len); } +#else +#define typecast_STRING_cast typecast_UNICODE_cast +#endif /** UNICODE - cast strings of any type to a python unicode object **/ diff --git a/psycopg/typecast_binary.c b/psycopg/typecast_binary.c index 261fc6f2..e93aad57 100644 --- a/psycopg/typecast_binary.c +++ b/psycopg/typecast_binary.c @@ -53,6 +53,9 @@ chunk_repr(chunkObject *self) ); } +#if PY_MAJOR_VERSION < 3 + +/* XXX support 3.0 buffer protocol */ static Py_ssize_t chunk_getreadbuffer(chunkObject *self, Py_ssize_t segment, void **ptr) { @@ -82,6 +85,22 @@ static PyBufferProcs chunk_as_buffer = (charbufferproc) NULL }; +#else + +/* 3.0 buffer interface */ +int chunk_getbuffer(PyObject *_self, Py_buffer *view, int flags) +{ + chunkObject *self = (chunkObject*)_self; + return PyBuffer_FillInfo(view, _self, self->base, self->len, 1, flags); +} +static PyBufferProcs chunk_as_buffer = +{ + chunk_getbuffer, + NULL, +}; + +#endif + #define chunk_doc "memory chunk" PyTypeObject chunkType = { @@ -156,8 +175,13 @@ typecast_BINARY_cast(const char *s, Py_ssize_t l, PyObject *curs) /* size_t->Py_ssize_t cast was validated above: */ chunk->len = (Py_ssize_t) len; +#if PY_MAJOR_VERSION < 3 if ((res = PyBuffer_FromObject((PyObject *)chunk, 0, chunk->len)) == NULL) goto fail; +#else + if ((res = PyMemoryView_FromObject((PyObject*)chunk)) == NULL) + goto fail; +#endif /* PyBuffer_FromObject() created a new reference. We'll release our * reference held in 'chunk' in the 'cleanup' clause. */ diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index b7251359..51cd4490 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -150,11 +150,11 @@ xid_init(XidObject *self, PyObject *args, PyObject *kwargs) Py_XDECREF(tmp); tmp = self->gtrid; - self->gtrid = PyString_FromString(gtrid); + self->gtrid = Text_FromUTF8(gtrid); Py_XDECREF(tmp); tmp = self->bqual; - self->bqual = PyString_FromString(bqual); + self->bqual = Text_FromUTF8(bqual); Py_XDECREF(tmp); return 0; @@ -233,7 +233,7 @@ xid_repr(XidObject *self) PyObject *args = NULL; if (Py_None == self->format_id) { - if (!(format = PyString_FromString(""))) { + if (!(format = Text_FromUTF8(""))) { goto exit; } if (!(args = PyTuple_New(1))) { goto exit; } @@ -241,7 +241,7 @@ xid_repr(XidObject *self) PyTuple_SET_ITEM(args, 0, self->gtrid); } else { - if (!(format = PyString_FromString(""))) { + if (!(format = Text_FromUTF8(""))) { goto exit; } if (!(args = PyTuple_New(3))) { goto exit; } @@ -253,7 +253,7 @@ xid_repr(XidObject *self) PyTuple_SET_ITEM(args, 2, self->bqual); } - rv = PyString_Format(format, args); + rv = Text_Format(format, args); exit: Py_XDECREF(args); @@ -457,7 +457,7 @@ xid_get_tid(XidObject *self) if (!(ebqual = _xid_encode64(self->bqual))) { goto exit; } /* rv = "%d_%s_%s" % (format_id, egtrid, ebqual) */ - if (!(format = PyString_FromString("%d_%s_%s"))) { goto exit; } + if (!(format = Text_FromUTF8("%d_%s_%s"))) { goto exit; } if (!(args = PyTuple_New(3))) { goto exit; } Py_INCREF(self->format_id); @@ -465,7 +465,7 @@ xid_get_tid(XidObject *self) PyTuple_SET_ITEM(args, 1, egtrid); egtrid = NULL; PyTuple_SET_ITEM(args, 2, ebqual); ebqual = NULL; - if (!(rv = PyString_Format(format, args))) { goto exit; } + if (!(rv = Text_Format(format, args))) { goto exit; } } exit: @@ -621,7 +621,7 @@ XidObject * xid_from_string(PyObject *str) { XidObject *rv; - if (!(PyString_Check(str) || PyUnicode_Check(str))) { + if (!(Bytes_Check(str) || PyUnicode_Check(str))) { PyErr_SetString(PyExc_TypeError, "not a valid transaction id"); return NULL; } From f697410ab4aac2f7ac789e78efc4e5ac5dfb6f71 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 21 Dec 2010 04:24:55 +0000 Subject: [PATCH 21/80] The tests are run from the external of the package. If __init__ is treated as a script, relative import fail. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 12617f06..d33787b3 100644 --- a/Makefile +++ b/Makefile @@ -48,11 +48,11 @@ SDIST := dist/psycopg2-$(VERSION).tar.gz EASY_INSTALL = PYTHONPATH=$(ENV_LIB) $(ENV_BIN)/easy_install-$(PYTHON_VERSION) -d $(ENV_LIB) -s $(ENV_BIN) EZ_SETUP = $(ENV_BIN)/ez_setup.py -.PHONY: env check runtests clean +.PHONY: env check clean default: package -all: package runtests sdist +all: package sdist package: $(PLATLIB) $(PURELIB) @@ -82,7 +82,7 @@ ez_setup: wget -O $(EZ_SETUP) http://peak.telecommunity.com/dist/ez_setup.py check: - PYTHONPATH=$(BUILD_DIR):$(BUILD_DIR)/tests:$(PYTHONPATH) $(PYTHON) $(BUILD_DIR)/psycopg2/tests/__init__.py --verbose + PYTHONPATH=$(BUILD_DIR):$(PYTHONPATH) $(PYTHON) -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" --verbose testdb: @echo "* Creating $(TESTDB)" From ade1b2cc7b9fc7b0fc05f70e2c15c1d1d1e56ca8 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 21 Dec 2010 04:58:38 +0000 Subject: [PATCH 22/80] Test suite converted into a proper package. Dropped cyclic import from modules to tests: they were only working because a second copy of the package was found in the project dir. Use relative import so that 2to3 can do a good conversion. --- tests/__init__.py | 32 ++------------------------------ tests/bug_gc.py | 8 ++------ tests/extras_dictcursor.py | 9 ++++----- tests/test_async.py | 13 ++++--------- tests/test_cancel.py | 10 +++++----- tests/test_connection.py | 28 ++++++++++++++-------------- tests/test_copy.py | 6 +++--- tests/test_cursor.py | 4 ++-- tests/test_dates.py | 6 +++--- tests/test_green.py | 4 ++-- tests/test_lobject.py | 8 ++++---- tests/test_notify.py | 12 ++++-------- tests/test_psycopg2_dbapi20.py | 6 +++--- tests/test_quote.py | 4 ++-- tests/test_transaction.py | 9 ++++----- tests/testconfig.py | 33 +++++++++++++++++++++++++++++++++ tests/types_basic.py | 4 ++-- tests/types_extras.py | 6 +++--- 18 files changed, 96 insertions(+), 106 deletions(-) create mode 100644 tests/testconfig.py diff --git a/tests/__init__.py b/tests/__init__.py index ad8feb28..cc6a92e5 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,37 +1,9 @@ #!/usr/bin/env python -import os import sys +from testconfig import dsn from testutils import unittest -dbname = os.environ.get('PSYCOPG2_TESTDB', 'psycopg2_test') -dbhost = os.environ.get('PSYCOPG2_TESTDB_HOST', None) -dbport = os.environ.get('PSYCOPG2_TESTDB_PORT', None) -dbuser = os.environ.get('PSYCOPG2_TESTDB_USER', None) - -# Check if we want to test psycopg's green path. -green = os.environ.get('PSYCOPG2_TEST_GREEN', None) -if green: - if green == '1': - from psycopg2.extras import wait_select as wait_callback - elif green == 'eventlet': - from eventlet.support.psycopg2_patcher import eventlet_wait_callback \ - as wait_callback - else: - raise ValueError("please set 'PSYCOPG2_TEST_GREEN' to a valid value") - - import psycopg2.extensions - psycopg2.extensions.set_wait_callback(wait_callback) - -# Construct a DSN to connect to the test database: -dsn = 'dbname=%s' % dbname -if dbhost is not None: - dsn += ' host=%s' % dbhost -if dbport is not None: - dsn += ' port=%s' % dbport -if dbuser is not None: - dsn += ' user=%s' % dbuser - # If connection to test db fails, bail out early. import psycopg2 try: @@ -81,4 +53,4 @@ def test_suite(): return suite if __name__ == '__main__': - unittest.main(defaultTest='test_suite') + unittest.main(defaultTest=test_suite) diff --git a/tests/bug_gc.py b/tests/bug_gc.py index 89d90d26..3cdcc6a1 100755 --- a/tests/bug_gc.py +++ b/tests/bug_gc.py @@ -6,18 +6,14 @@ import time import unittest import gc -import sys -if sys.version_info < (3,): - import tests -else: - import py3tests as tests +from testconfig import dsn class StolenReferenceTestCase(unittest.TestCase): def test_stolen_reference_bug(self): def fish(val, cur): gc.collect() return 42 - conn = psycopg2.connect(tests.dsn) + conn = psycopg2.connect(dsn) UUID = psycopg2.extensions.new_type((2950,), "UUID", fish) psycopg2.extensions.register_type(UUID, conn) curs = conn.cursor() diff --git a/tests/extras_dictcursor.py b/tests/extras_dictcursor.py index c2bacb0b..1bb44ad4 100644 --- a/tests/extras_dictcursor.py +++ b/tests/extras_dictcursor.py @@ -17,15 +17,14 @@ import psycopg2 import psycopg2.extras from testutils import unittest - -import tests +from testconfig import dsn class ExtrasDictCursorTests(unittest.TestCase): """Test if DictCursor extension class works.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) curs = self.conn.cursor() curs.execute("CREATE TEMPORARY TABLE ExtrasDictCursorTests (foo text)") curs.execute("INSERT INTO ExtrasDictCursorTests VALUES ('bar')") @@ -135,7 +134,7 @@ class NamedTupleCursorTest(unittest.TestCase): self.conn = None return - self.conn = psycopg2.connect(tests.dsn, + self.conn = psycopg2.connect(dsn, connection_factory=NamedTupleConnection) curs = self.conn.cursor() curs.execute("CREATE TEMPORARY TABLE nttest (i int, s text)") @@ -207,7 +206,7 @@ class NamedTupleCursorTest(unittest.TestCase): try: if self.conn is not None: self.conn.close() - self.conn = psycopg2.connect(tests.dsn, + self.conn = psycopg2.connect(dsn, connection_factory=NamedTupleConnection) curs = self.conn.cursor() curs.execute("select 1") diff --git a/tests/test_async.py b/tests/test_async.py index 96d7a2cc..be4a1d88 100755 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -8,12 +8,7 @@ import time import select import StringIO -import sys -if sys.version_info < (3,): - import tests -else: - import py3tests as tests - +from testconfig import dsn class PollableStub(object): """A 'pollable' wrapper allowing analysis of the `poll()` calls.""" @@ -33,8 +28,8 @@ class PollableStub(object): class AsyncTests(unittest.TestCase): def setUp(self): - self.sync_conn = psycopg2.connect(tests.dsn) - self.conn = psycopg2.connect(tests.dsn, async=True) + self.sync_conn = psycopg2.connect(dsn) + self.conn = psycopg2.connect(dsn, async=True) self.wait(self.conn) @@ -309,7 +304,7 @@ class AsyncTests(unittest.TestCase): def __init__(self, dsn, async=0): psycopg2.extensions.connection.__init__(self, dsn, async=async) - conn = psycopg2.connect(tests.dsn, connection_factory=MyConn, async=True) + conn = psycopg2.connect(dsn, connection_factory=MyConn, async=True) self.assert_(isinstance(conn, MyConn)) self.assert_(conn.async) conn.close() diff --git a/tests/test_cancel.py b/tests/test_cancel.py index 52383f83..8e651ffa 100644 --- a/tests/test_cancel.py +++ b/tests/test_cancel.py @@ -2,18 +2,18 @@ import time import threading -from testutils import unittest, skip_if_no_pg_sleep -import tests import psycopg2 import psycopg2.extensions from psycopg2 import extras +from testconfig import dsn +from testutils import unittest, skip_if_no_pg_sleep class CancelTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) cur = self.conn.cursor() cur.execute(''' CREATE TEMPORARY TABLE table1 ( @@ -65,7 +65,7 @@ class CancelTests(unittest.TestCase): @skip_if_no_pg_sleep('conn') def test_async_cancel(self): - async_conn = psycopg2.connect(tests.dsn, async=True) + async_conn = psycopg2.connect(dsn, async=True) self.assertRaises(psycopg2.OperationalError, async_conn.cancel) extras.wait_select(async_conn) cur = async_conn.cursor() @@ -79,7 +79,7 @@ class CancelTests(unittest.TestCase): self.assertEqual(cur.fetchall(), [(1, )]) def test_async_connection_cancel(self): - async_conn = psycopg2.connect(tests.dsn, async=True) + async_conn = psycopg2.connect(dsn, async=True) async_conn.close() self.assertTrue(async_conn.closed) diff --git a/tests/test_connection.py b/tests/test_connection.py index de775e08..1b34b6c9 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -7,12 +7,12 @@ from operator import attrgetter import psycopg2 import psycopg2.extensions -import tests +from testconfig import dsn, dbname class ConnectionTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): if not self.conn.closed: @@ -95,7 +95,7 @@ class ConnectionTests(unittest.TestCase): @skip_if_no_pg_sleep('conn') def test_concurrent_execution(self): def slave(): - cnn = psycopg2.connect(tests.dsn) + cnn = psycopg2.connect(dsn) cur = cnn.cursor() cur.execute("select pg_sleep(2)") cur.close() @@ -141,7 +141,7 @@ class IsolationLevelsTestCase(unittest.TestCase): conn.close() def connect(self): - conn = psycopg2.connect(tests.dsn) + conn = psycopg2.connect(dsn) self._conns.append(conn) return conn @@ -333,7 +333,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): try: cur.execute( "select gid from pg_prepared_xacts where database = %s", - (tests.dbname,)) + (dbname,)) except psycopg2.ProgrammingError: cnn.rollback() cnn.close() @@ -362,7 +362,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cur.execute(""" select count(*) from pg_prepared_xacts where database = %s;""", - (tests.dbname,)) + (dbname,)) rv = cur.fetchone()[0] cnn.close() return rv @@ -377,7 +377,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): return rv def connect(self): - conn = psycopg2.connect(tests.dsn) + conn = psycopg2.connect(dsn) self._conns.append(conn) return conn @@ -540,13 +540,13 @@ class ConnectionTwoPhaseTests(unittest.TestCase): select gid, prepared, owner, database from pg_prepared_xacts where database = %s;""", - (tests.dbname,)) + (dbname,)) okvals = cur.fetchall() okvals.sort() cnn = self.connect() xids = cnn.tpc_recover() - xids = [ xid for xid in xids if xid.database == tests.dbname ] + xids = [ xid for xid in xids if xid.database == dbname ] xids.sort(key=attrgetter('gtrid')) # check the values returned @@ -566,7 +566,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn = self.connect() cur = cnn.cursor() cur.execute("select gid from pg_prepared_xacts where database = %s;", - (tests.dbname,)) + (dbname,)) self.assertEqual('42_Z3RyaWQ=_YnF1YWw=', cur.fetchone()[0]) def test_xid_roundtrip(self): @@ -583,7 +583,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn = self.connect() xids = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ] + if xid.database == dbname ] self.assertEqual(1, len(xids)) xid = xids[0] self.assertEqual(xid.format_id, fid) @@ -605,7 +605,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn = self.connect() xids = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ] + if xid.database == dbname ] self.assertEqual(1, len(xids)) xid = xids[0] self.assertEqual(xid.format_id, None) @@ -651,7 +651,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn.tpc_prepare() cnn.reset() xid = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ][0] + if xid.database == dbname ][0] self.assertEqual(10, xid.format_id) self.assertEqual('uni', xid.gtrid) self.assertEqual('code', xid.bqual) @@ -667,7 +667,7 @@ class ConnectionTwoPhaseTests(unittest.TestCase): cnn.reset() xid = [ xid for xid in cnn.tpc_recover() - if xid.database == tests.dbname ][0] + if xid.database == dbname ][0] self.assertEqual(None, xid.format_id) self.assertEqual('transaction-id', xid.gtrid) self.assertEqual(None, xid.bqual) diff --git a/tests/test_copy.py b/tests/test_copy.py index b0d9965c..2f09d5a6 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -7,11 +7,11 @@ from itertools import cycle, izip import psycopg2 import psycopg2.extensions -import tests +from testconfig import dsn, green def skip_if_green(f): def skip_if_green_(self): - if tests.green: + if green: return self.skipTest("copy in async mode currently not supported") else: return f(self) @@ -42,7 +42,7 @@ class MinimalWrite(object): class CopyTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) curs = self.conn.cursor() curs.execute(''' CREATE TEMPORARY TABLE tcopy ( diff --git a/tests/test_cursor.py b/tests/test_cursor.py index aedb5f5f..d30a50ef 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -3,12 +3,12 @@ import unittest import psycopg2 import psycopg2.extensions -import tests +from testconfig import dsn class CursorTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() diff --git a/tests/test_dates.py b/tests/test_dates.py index 9958913d..44b1bc57 100644 --- a/tests/test_dates.py +++ b/tests/test_dates.py @@ -2,9 +2,9 @@ import math import unittest -import tests import psycopg2 from psycopg2.tz import FixedOffsetTimezone +from testconfig import dsn class CommonDatetimeTestsMixin: @@ -75,7 +75,7 @@ class DatetimeTests(unittest.TestCase, CommonDatetimeTestsMixin): """Tests for the datetime based date handling in psycopg2.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.curs = self.conn.cursor() self.DATE = psycopg2._psycopg.PYDATE self.TIME = psycopg2._psycopg.PYTIME @@ -293,7 +293,7 @@ class mxDateTimeTests(unittest.TestCase, CommonDatetimeTestsMixin): """Tests for the mx.DateTime based date handling in psycopg2.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.curs = self.conn.cursor() self.DATE = psycopg2._psycopg.MXDATE self.TIME = psycopg2._psycopg.MXTIME diff --git a/tests/test_green.py b/tests/test_green.py index 07ecdb7b..bb9acf2c 100644 --- a/tests/test_green.py +++ b/tests/test_green.py @@ -4,7 +4,7 @@ import unittest import psycopg2 import psycopg2.extensions import psycopg2.extras -import tests +from testconfig import dsn class ConnectionStub(object): """A `connection` wrapper allowing analysis of the `poll()` calls.""" @@ -24,7 +24,7 @@ class GreenTests(unittest.TestCase): def setUp(self): self._cb = psycopg2.extensions.get_wait_callback() psycopg2.extensions.set_wait_callback(psycopg2.extras.wait_select) - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 88e18d6f..7c96a6ea 100644 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -2,11 +2,11 @@ import os import shutil import tempfile -from testutils import unittest, decorate_all_tests import psycopg2 import psycopg2.extensions -import tests +from testconfig import dsn, green +from testutils import unittest, decorate_all_tests def skip_if_no_lo(f): def skip_if_no_lo_(self): @@ -19,7 +19,7 @@ def skip_if_no_lo(f): def skip_if_green(f): def skip_if_green_(self): - if tests.green: + if green: return self.skipTest("libpq doesn't support LO in async mode") else: return f(self) @@ -30,7 +30,7 @@ def skip_if_green(f): class LargeObjectMixin(object): # doesn't derive from TestCase to avoid repeating tests twice. def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.lo_oid = None self.tmpdir = None diff --git a/tests/test_notify.py b/tests/test_notify.py index bc14f374..a15aad68 100755 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -3,23 +3,19 @@ from testutils import unittest import psycopg2 from psycopg2 import extensions +from testconfig import dsn +import sys import time import select import signal from subprocess import Popen, PIPE -import sys -if sys.version_info < (3,): - import tests -else: - import py3tests as tests - class NotifiesTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -54,7 +50,7 @@ curs.execute("NOTIFY " %(name)r %(payload)r) curs.close() conn.close() """ - % { 'dsn': tests.dsn, 'sec': sec, 'name': name, 'payload': payload}) + % { 'dsn': dsn, 'sec': sec, 'name': name, 'payload': payload}) return Popen([sys.executable, '-c', script], stdout=PIPE) diff --git a/tests/test_psycopg2_dbapi20.py b/tests/test_psycopg2_dbapi20.py index 93fdfa60..dfe8f53d 100755 --- a/tests/test_psycopg2_dbapi20.py +++ b/tests/test_psycopg2_dbapi20.py @@ -5,12 +5,12 @@ from test_connection import skip_if_tpc_disabled from testutils import unittest, decorate_all_tests import psycopg2 -import tests +from testconfig import dsn class Psycopg2Tests(dbapi20.DatabaseAPI20Test): driver = psycopg2 connect_args = () - connect_kw_args = {'dsn': tests.dsn} + connect_kw_args = {'dsn': dsn} lower_func = 'lower' # For stored procedure test @@ -27,7 +27,7 @@ class Psycopg2TPCTests(dbapi20_tpc.TwoPhaseCommitTests, unittest.TestCase): driver = psycopg2 def connect(self): - return psycopg2.connect(dsn=tests.dsn) + return psycopg2.connect(dsn=dsn) decorate_all_tests(Psycopg2TPCTests, skip_if_tpc_disabled) diff --git a/tests/test_quote.py b/tests/test_quote.py index 95c5d7a5..83662c72 100755 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -3,7 +3,7 @@ from testutils import unittest import psycopg2 import psycopg2.extensions -import tests +from testconfig import dsn class QuotingTestCase(unittest.TestCase): r"""Checks the correct quoting of strings and binary objects. @@ -24,7 +24,7 @@ class QuotingTestCase(unittest.TestCase): http://www.postgresql.org/docs/8.1/static/runtime-config-compatible.html """ def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() diff --git a/tests/test_transaction.py b/tests/test_transaction.py index 5bbed38f..53ddf853 100755 --- a/tests/test_transaction.py +++ b/tests/test_transaction.py @@ -5,13 +5,12 @@ from testutils import unittest, skip_if_no_pg_sleep import psycopg2 from psycopg2.extensions import ( ISOLATION_LEVEL_SERIALIZABLE, STATUS_BEGIN, STATUS_READY) -import tests - +from testconfig import dsn class TransactionTests(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.conn.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE) curs = self.conn.cursor() curs.execute(''' @@ -75,7 +74,7 @@ class DeadlockSerializationTests(unittest.TestCase): """Test deadlock and serialization failure errors.""" def connect(self): - conn = psycopg2.connect(tests.dsn) + conn = psycopg2.connect(dsn) conn.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE) return conn @@ -208,7 +207,7 @@ class QueryCancellationTests(unittest.TestCase): """Tests for query cancellation.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) self.conn.set_isolation_level(ISOLATION_LEVEL_SERIALIZABLE) def tearDown(self): diff --git a/tests/testconfig.py b/tests/testconfig.py new file mode 100644 index 00000000..b2730e02 --- /dev/null +++ b/tests/testconfig.py @@ -0,0 +1,33 @@ +# Configure the test suite from the env variables. + +import os + +dbname = os.environ.get('PSYCOPG2_TESTDB', 'psycopg2_test') +dbhost = os.environ.get('PSYCOPG2_TESTDB_HOST', None) +dbport = os.environ.get('PSYCOPG2_TESTDB_PORT', None) +dbuser = os.environ.get('PSYCOPG2_TESTDB_USER', None) + +# Check if we want to test psycopg's green path. +green = os.environ.get('PSYCOPG2_TEST_GREEN', None) +if green: + if green == '1': + from psycopg2.extras import wait_select as wait_callback + elif green == 'eventlet': + from eventlet.support.psycopg2_patcher import eventlet_wait_callback \ + as wait_callback + else: + raise ValueError("please set 'PSYCOPG2_TEST_GREEN' to a valid value") + + import psycopg2.extensions + psycopg2.extensions.set_wait_callback(wait_callback) + +# Construct a DSN to connect to the test database: +dsn = 'dbname=%s' % dbname +if dbhost is not None: + dsn += ' host=%s' % dbhost +if dbport is not None: + dsn += ' port=%s' % dbport +if dbuser is not None: + dsn += ' user=%s' % dbuser + + diff --git a/tests/types_basic.py b/tests/types_basic.py index 5bcff062..f14acc56 100755 --- a/tests/types_basic.py +++ b/tests/types_basic.py @@ -30,14 +30,14 @@ import sys from testutils import unittest import psycopg2 -import tests +from testconfig import dsn class TypesBasicTests(unittest.TestCase): """Test that all type conversions are working.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() diff --git a/tests/types_extras.py b/tests/types_extras.py index 6f77d976..9a2a2c27 100644 --- a/tests/types_extras.py +++ b/tests/types_extras.py @@ -24,7 +24,7 @@ from testutils import unittest import psycopg2 import psycopg2.extras -import tests +from testconfig import dsn def skip_if_no_uuid(f): @@ -58,7 +58,7 @@ class TypesExtrasTests(unittest.TestCase): """Test that all type conversions are working.""" def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -145,7 +145,7 @@ def skip_if_no_hstore(f): class HstoreTestCase(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() From e18f1c63eab211f7f6edf62aca09a27a3097748c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 03:28:19 +0100 Subject: [PATCH 23/80] Deal with slices passed to __*item__ in Python 3. --- lib/extras.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/extras.py b/lib/extras.py index b4528289..215b59a6 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -141,12 +141,12 @@ class DictRow(list): self[:] = [None] * len(cursor.description) def __getitem__(self, x): - if not isinstance(x, int): + if not isinstance(x, (int, slice)): x = self._index[x] return list.__getitem__(self, x) def __setitem__(self, x, v): - if not isinstance(x, int): + if not isinstance(x, (int, slice)): x = self._index[x] list.__setitem__(self, x, v) From c3196ebd9d7fdc604fd97c97cc42d8f2c2a50224 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 04:05:45 +0100 Subject: [PATCH 24/80] Added PyBytes_Format function. --- psycopg/bytes_format.c | 513 +++++++++++++++++++++++++++++++++++++++++ psycopg/python.h | 11 + setup.py | 3 + 3 files changed, 527 insertions(+) create mode 100644 psycopg/bytes_format.c diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c new file mode 100644 index 00000000..1c49a011 --- /dev/null +++ b/psycopg/bytes_format.c @@ -0,0 +1,513 @@ +/* bytes_format.c - bytes-oriented version of PyString_Format + * + * Copyright (C) 2010 Daniele Varrazzo + * + * This file is part of psycopg. + * + * psycopg2 is free software: you can redistribute it and/or modify it + * under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * In addition, as a special exception, the copyright holders give + * permission to link this program with the OpenSSL library (or with + * modified versions of OpenSSL that use the same license as OpenSSL), + * and distribute linked combinations including the two. + * + * You must obey the GNU Lesser General Public License in all respects for + * all of the code used other than OpenSSL. + * + * psycopg2 is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + * License for more details. + */ + +/* This implementation is based on the PyString_Format function available in + * Python 2.7.1. Original license follows. + * + * PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 + * -------------------------------------------- + * + * 1. This LICENSE AGREEMENT is between the Python Software Foundation + * ("PSF"), and the Individual or Organization ("Licensee") accessing and + * otherwise using this software ("Python") in source or binary form and + * its associated documentation. + * + * 2. Subject to the terms and conditions of this License Agreement, PSF hereby + * grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, + * analyze, test, perform and/or display publicly, prepare derivative works, + * distribute, and otherwise use Python alone or in any derivative version, + * provided, however, that PSF's License Agreement and PSF's notice of copyright, + * i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 + * Python Software Foundation; All Rights Reserved" are retained in Python alone or + * in any derivative version prepared by Licensee. + * + * 3. In the event Licensee prepares a derivative work that is based on + * or incorporates Python or any part thereof, and wants to make + * the derivative work available to others as provided herein, then + * Licensee hereby agrees to include in any such work a brief summary of + * the changes made to Python. + * + * 4. PSF is making Python available to Licensee on an "AS IS" + * basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR + * IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND + * DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS + * FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT + * INFRINGE ANY THIRD PARTY RIGHTS. + * + * 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON + * FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS + * A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, + * OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + * + * 6. This License Agreement will automatically terminate upon a material + * breach of its terms and conditions. + * + * 7. Nothing in this License Agreement shall be deemed to create any + * relationship of agency, partnership, or joint venture between PSF and + * Licensee. This License Agreement does not grant permission to use PSF + * trademarks or trade name in a trademark sense to endorse or promote + * products or services of Licensee, or any third party. + * + * 8. By copying, installing or otherwise using Python, Licensee + * agrees to be bound by the terms and conditions of this License + * Agreement. + */ + +#define PSYCOPG_MODULE +#include "psycopg/psycopg.h" + +#ifndef Py_USING_UNICODE +#define Py_USING_UNICODE 1 +#endif + +/* Helpers for formatstring */ + +Py_LOCAL_INLINE(PyObject *) +getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) +{ + Py_ssize_t argidx = *p_argidx; + if (argidx < arglen) { + (*p_argidx)++; + if (arglen < 0) + return args; + else + return PyTuple_GetItem(args, argidx); + } + PyErr_SetString(PyExc_TypeError, + "not enough arguments for format string"); + return NULL; +} + +/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) + + FORMATBUFLEN is the length of the buffer in which the ints & + chars are formatted. XXX This is a magic number. Each formatting + routine does bounds checking to ensure no overflow, but a better + solution may be to malloc a buffer of appropriate size for each + format. For now, the current solution is sufficient. +*/ +#define FORMATBUFLEN (size_t)120 + +PyObject * +PyBytes_Format(PyObject *format, PyObject *args) +{ + char *fmt, *res; + Py_ssize_t arglen, argidx; + Py_ssize_t reslen, rescnt, fmtcnt; + int args_owned = 0; + PyObject *result, *orig_args; +#ifdef Py_USING_UNICODE + PyObject *v, *w; +#endif + PyObject *dict = NULL; + if (format == NULL || !PyBytes_Check(format) || args == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + orig_args = args; + fmt = PyBytes_AS_STRING(format); + fmtcnt = PyBytes_GET_SIZE(format); + reslen = rescnt = fmtcnt + 100; + result = PyBytes_FromStringAndSize((char *)NULL, reslen); + if (result == NULL) + return NULL; + res = PyBytes_AsString(result); + if (PyTuple_Check(args)) { + arglen = PyTuple_GET_SIZE(args); + argidx = 0; + } + else { + arglen = -1; + argidx = -2; + } + if (Py_TYPE(args)->tp_as_mapping && !PyTuple_Check(args) && + !PyObject_TypeCheck(args, &PyBytes_Type)) + dict = args; + while (--fmtcnt >= 0) { + if (*fmt != '%') { + if (--rescnt < 0) { + rescnt = fmtcnt + 100; + reslen += rescnt; + if (_PyBytes_Resize(&result, reslen)) + return NULL; + res = PyBytes_AS_STRING(result) + + reslen - rescnt; + --rescnt; + } + *res++ = *fmt++; + } + else { + /* Got a format specifier */ + int flags = 0; + Py_ssize_t width = -1; + int prec = -1; + int c = '\0'; + int fill; + int isnumok; + PyObject *v = NULL; + PyObject *temp = NULL; + char *pbuf; + int sign; + Py_ssize_t len; + char formatbuf[FORMATBUFLEN]; + /* For format{int,char}() */ +#ifdef Py_USING_UNICODE + char *fmt_start = fmt; + Py_ssize_t argidx_start = argidx; +#endif + + fmt++; + if (*fmt == '(') { + char *keystart; + Py_ssize_t keylen; + PyObject *key; + int pcount = 1; + + if (dict == NULL) { + PyErr_SetString(PyExc_TypeError, + "format requires a mapping"); + goto error; + } + ++fmt; + --fmtcnt; + keystart = fmt; + /* Skip over balanced parentheses */ + while (pcount > 0 && --fmtcnt >= 0) { + if (*fmt == ')') + --pcount; + else if (*fmt == '(') + ++pcount; + fmt++; + } + keylen = fmt - keystart - 1; + if (fmtcnt < 0 || pcount > 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format key"); + goto error; + } + key = PyBytes_FromStringAndSize(keystart, + keylen); + if (key == NULL) + goto error; + if (args_owned) { + Py_DECREF(args); + args_owned = 0; + } + args = PyObject_GetItem(dict, key); + Py_DECREF(key); + if (args == NULL) { + goto error; + } + args_owned = 1; + arglen = -1; + argidx = -2; + } + while (--fmtcnt >= 0) { + switch (c = *fmt++) { + case '-': flags |= F_LJUST; continue; + case '+': flags |= F_SIGN; continue; + case ' ': flags |= F_BLANK; continue; + case '#': flags |= F_ALT; continue; + case '0': flags |= F_ZERO; continue; + } + break; + } + if (c == '*') { + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto error; + if (!PyLong_Check(v)) { + PyErr_SetString(PyExc_TypeError, + "* wants int"); + goto error; + } + width = PyLong_AsLong(v); + if (width < 0) { + flags |= F_LJUST; + width = -width; + } + if (--fmtcnt >= 0) + c = *fmt++; + } + else if (c >= 0 && isdigit(c)) { + width = c - '0'; + while (--fmtcnt >= 0) { + c = Py_CHARMASK(*fmt++); + if (!isdigit(c)) + break; + if ((width*10) / 10 != width) { + PyErr_SetString( + PyExc_ValueError, + "width too big"); + goto error; + } + width = width*10 + (c - '0'); + } + } + if (c == '.') { + prec = 0; + if (--fmtcnt >= 0) + c = *fmt++; + if (c == '*') { + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto error; + if (!PyLong_Check(v)) { + PyErr_SetString( + PyExc_TypeError, + "* wants int"); + goto error; + } + prec = PyLong_AsLong(v); + if (prec < 0) + prec = 0; + if (--fmtcnt >= 0) + c = *fmt++; + } + else if (c >= 0 && isdigit(c)) { + prec = c - '0'; + while (--fmtcnt >= 0) { + c = Py_CHARMASK(*fmt++); + if (!isdigit(c)) + break; + if ((prec*10) / 10 != prec) { + PyErr_SetString( + PyExc_ValueError, + "prec too big"); + goto error; + } + prec = prec*10 + (c - '0'); + } + } + } /* prec */ + if (fmtcnt >= 0) { + if (c == 'h' || c == 'l' || c == 'L') { + if (--fmtcnt >= 0) + c = *fmt++; + } + } + if (fmtcnt < 0) { + PyErr_SetString(PyExc_ValueError, + "incomplete format"); + goto error; + } + if (c != '%') { + v = getnextarg(args, arglen, &argidx); + if (v == NULL) + goto error; + } + sign = 0; + fill = ' '; + switch (c) { + case '%': + pbuf = "%"; + len = 1; + break; + case 's': + /* only bytes! */ + if (!PyBytes_CheckExact(v)) { + PyErr_Format(PyExc_ValueError, + "only bytes values expected, got %s", + Py_TYPE(v)->tp_name); + goto error; + } + temp = v; + Py_INCREF(v); + /* Fall through */ + case 'r': + if (c == 'r') + temp = PyObject_Repr(v); + if (temp == NULL) + goto error; + if (!PyBytes_Check(temp)) { + PyErr_SetString(PyExc_TypeError, + "%s argument has non-string str()"); + Py_DECREF(temp); + goto error; + } + pbuf = PyBytes_AS_STRING(temp); + len = PyBytes_GET_SIZE(temp); + if (prec >= 0 && len > prec) + len = prec; + break; + default: + PyErr_Format(PyExc_ValueError, + "unsupported format character '%c' (0x%x) " + "at index %zd", + c, c, + (Py_ssize_t)(fmt - 1 - + PyBytes_AsString(format))); + goto error; + } + if (sign) { + if (*pbuf == '-' || *pbuf == '+') { + sign = *pbuf++; + len--; + } + else if (flags & F_SIGN) + sign = '+'; + else if (flags & F_BLANK) + sign = ' '; + else + sign = 0; + } + if (width < len) + width = len; + if (rescnt - (sign != 0) < width) { + reslen -= rescnt; + rescnt = width + fmtcnt + 100; + reslen += rescnt; + if (reslen < 0) { + Py_DECREF(result); + Py_XDECREF(temp); + return PyErr_NoMemory(); + } + if (_PyBytes_Resize(&result, reslen)) { + Py_XDECREF(temp); + return NULL; + } + res = PyBytes_AS_STRING(result) + + reslen - rescnt; + } + if (sign) { + if (fill != ' ') + *res++ = sign; + rescnt--; + if (width > len) + width--; + } + if ((flags & F_ALT) && (c == 'x' || c == 'X')) { + assert(pbuf[0] == '0'); + assert(pbuf[1] == c); + if (fill != ' ') { + *res++ = *pbuf++; + *res++ = *pbuf++; + } + rescnt -= 2; + width -= 2; + if (width < 0) + width = 0; + len -= 2; + } + if (width > len && !(flags & F_LJUST)) { + do { + --rescnt; + *res++ = fill; + } while (--width > len); + } + if (fill == ' ') { + if (sign) + *res++ = sign; + if ((flags & F_ALT) && + (c == 'x' || c == 'X')) { + assert(pbuf[0] == '0'); + assert(pbuf[1] == c); + *res++ = *pbuf++; + *res++ = *pbuf++; + } + } + Py_MEMCPY(res, pbuf, len); + res += len; + rescnt -= len; + while (--width >= len) { + --rescnt; + *res++ = ' '; + } + if (dict && (argidx < arglen) && c != '%') { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + Py_XDECREF(temp); + goto error; + } + Py_XDECREF(temp); + } /* '%' */ + } /* until end */ + if (argidx < arglen && !dict) { + PyErr_SetString(PyExc_TypeError, + "not all arguments converted during string formatting"); + goto error; + } + if (args_owned) { + Py_DECREF(args); + } + if (_PyBytes_Resize(&result, reslen - rescnt)) + return NULL; + return result; + +#ifdef Py_USING_UNICODE + unicode: + if (args_owned) { + Py_DECREF(args); + args_owned = 0; + } + /* Fiddle args right (remove the first argidx arguments) */ + if (PyTuple_Check(orig_args) && argidx > 0) { + PyObject *v; + Py_ssize_t n = PyTuple_GET_SIZE(orig_args) - argidx; + v = PyTuple_New(n); + if (v == NULL) + goto error; + while (--n >= 0) { + PyObject *w = PyTuple_GET_ITEM(orig_args, n + argidx); + Py_INCREF(w); + PyTuple_SET_ITEM(v, n, w); + } + args = v; + } else { + Py_INCREF(orig_args); + args = orig_args; + } + args_owned = 1; + /* Take what we have of the result and let the Unicode formatting + function format the rest of the input. */ + rescnt = res - PyBytes_AS_STRING(result); + if (_PyBytes_Resize(&result, rescnt)) + goto error; + fmtcnt = PyBytes_GET_SIZE(format) - \ + (fmt - PyBytes_AS_STRING(format)); + format = PyUnicode_Decode(fmt, fmtcnt, NULL, NULL); + if (format == NULL) + goto error; + v = PyUnicode_Format(format, args); + Py_DECREF(format); + if (v == NULL) + goto error; + /* Paste what we have (result) to what the Unicode formatting + function returned (v) and return the result (or error) */ + w = PyUnicode_Concat(result, v); + Py_DECREF(result); + Py_DECREF(v); + Py_DECREF(args); + return w; +#endif /* Py_USING_UNICODE */ + + error: + Py_DECREF(result); + if (args_owned) { + Py_DECREF(args); + } + return NULL; +} + diff --git a/psycopg/python.h b/psycopg/python.h index 8fee7fd7..40895bbf 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -117,8 +117,13 @@ #define Bytes_Size PyString_Size #define Bytes_AsString PyString_AsString #define Bytes_AsStringAndSize PyString_AsStringAndSize +#define Bytes_FromString PyString_FromString #define Bytes_FromStringAndSize PyString_FromStringAndSize +#define Bytes_FromFormat PyString_FromFormat +#define Bytes_Format PyString_Format + #else + #define BytesType PyBytes_Type #define Bytes_Check PyBytes_Check #define Bytes_AS_STRING PyBytes_AS_STRING @@ -126,7 +131,13 @@ #define Bytes_Size PyBytes_Size #define Bytes_AsString PyBytes_AsString #define Bytes_AsStringAndSize PyBytes_AsStringAndSize +#define Bytes_FromString PyBytes_FromString #define Bytes_FromStringAndSize PyBytes_FromStringAndSize +#define Bytes_FromFormat PyBytes_FromFormat +#define Bytes_Format PyBytes_Format + +HIDDEN PyObject *PyBytes_Format(PyObject *format, PyObject *args); + #endif /* Mangle the module name into the name of the module init function */ diff --git a/setup.py b/setup.py index 36cdc97d..b05bf9a7 100644 --- a/setup.py +++ b/setup.py @@ -376,6 +376,9 @@ sources = [ 'typecast.c', ] +if sys.version_info[0] >= 3: + sources.append('bytes_format.c') + depends = [ # headers 'config.h', 'pgtypes.h', 'psycopg.h', 'python.h', From 87a7ebac10fb0055a0d685aa0ba50a09abf8fc86 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 04:06:34 +0100 Subject: [PATCH 25/80] Query mogrification in bytes. --- psycopg/cursor_type.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 4bf53139..922288b5 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -314,7 +314,7 @@ _psyco_curs_merge_query_args(cursorObject *self, the curren exception (we will later restore it if the type or the strings do not match.) */ - if (!(fquery = Text_Format(query, args))) { + if (!(fquery = Bytes_Format(query, args))) { PyObject *err, *arg, *trace; int pe = 0; @@ -397,15 +397,9 @@ _psyco_curs_execute(cursorObject *self, } if (self->name != NULL) { - #if PY_MAJOR_VERSION < 3 - self->query = PyString_FromFormat( + self->query = Bytes_FromFormat( "DECLARE %s CURSOR WITHOUT HOLD FOR %s", - self->name, PyString_AS_STRING(fquery)); - #else - self->query = PyUnicode_FromFormat( - "DECLARE %s CURSOR WITHOUT HOLD FOR %U", - self->name, fquery); - #endif + self->name, Bytes_AS_STRING(fquery)); Py_DECREF(fquery); } else { @@ -414,15 +408,9 @@ _psyco_curs_execute(cursorObject *self, } else { if (self->name != NULL) { - #if PY_MAJOR_VERSION < 3 - self->query = PyString_FromFormat( + self->query = Bytes_FromFormat( "DECLARE %s CURSOR WITHOUT HOLD FOR %s", - self->name, PyString_AS_STRING(operation)); - #else - self->query = PyUnicode_FromFormat( - "DECLARE %s CURSOR WITHOUT HOLD FOR %U", - self->name, operation); - #endif + self->name, Bytes_AS_STRING(operation)); } else { /* Transfer reference ownership of the str in operation to From 03dde732f6f14581d7f19ecda260087d4e6e6aaa Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 04:08:32 +0100 Subject: [PATCH 26/80] Datetime adaptation in bytes. --- psycopg/adapter_datetime.c | 29 ++++++++++++++++++----------- psycopg/python.h | 6 ------ 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 19fd97e2..2b9f69fd 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -62,30 +62,37 @@ pydatetime_str(pydatetimeObject *self) if (self->type <= PSYCO_DATETIME_TIMESTAMP) { PyObject *tz; - /* Select the right PG type to cast into. - * fmt contains %s in Py2 and %U in Py3, - * So it can be used with the Text_FromFormatS macro */ + /* Select the right PG type to cast into. */ char *fmt = NULL; switch (self->type) { case PSYCO_DATETIME_TIME: - fmt = "'" Text_S "'::time"; + fmt = "'%s'::time"; break; case PSYCO_DATETIME_DATE: - fmt = "'" Text_S "'::date"; + fmt = "'%s'::date"; break; case PSYCO_DATETIME_TIMESTAMP: tz = PyObject_GetAttrString(self->wrapped, "tzinfo"); if (!tz) { return NULL; } - fmt = (tz == Py_None) - ? "'" Text_S "'::timestamp" - : "'" Text_S "'::timestamptz"; + fmt = (tz == Py_None) ? "'%s'::timestamp" : "'%s'::timestamptz"; Py_DECREF(tz); break; } iso = PyObject_CallMethod(self->wrapped, "isoformat", NULL); if (iso) { - res = Text_FromFormatS(fmt, iso); +#if PY_MAJOR_VERSION > 2 + { + PyObject *biso; + if (!(biso = PyUnicode_AsEncodedString(iso, "ascii", NULL))) { + Py_DECREF(iso); + return NULL; + } + Py_DECREF(iso); + iso = biso; + } +#endif + res = Bytes_FromFormat(fmt, Bytes_AsString(iso)); Py_DECREF(iso); } return res; @@ -103,8 +110,8 @@ pydatetime_str(pydatetimeObject *self) } buffer[6] = '\0'; - return PyString_FromFormat("'%d days %d.%s seconds'::interval", - obj->days, obj->seconds, buffer); + return Bytes_FromFormat("'%d days %d.%s seconds'::interval", + obj->days, obj->seconds, buffer); } } diff --git a/psycopg/python.h b/psycopg/python.h index 40895bbf..452c7341 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -81,18 +81,12 @@ #define Text_Format(f,a) PyString_Format(f,a) #define Text_FromUTF8(s) PyString_FromString(s) #define Text_FromUTF8AndSize(s,n) PyString_FromStringAndSize(s,n) -/* f must contain exactly a %s placeholder */ -#define Text_FromFormatS(f,s) PyString_FromFormat(f, PyString_AsString(s)) -#define Text_S "%s" #else #define Text_Type PyUnicode_Type #define Text_Check(s) PyUnicode_Check(s) #define Text_Format(f,a) PyUnicode_Format(f,a) #define Text_FromUTF8(s) PyUnicode_FromString(s) #define Text_FromUTF8AndSize(s,n) PyUnicode_FromStringAndSize(s,n) -/* f must contain exactly a %U placeholder */ -#define Text_FromFormatS(f,s) PyUnicode_FromFormat(f, s) -#define Text_S "%U" #endif #if PY_MAJOR_VERSION > 2 From b4685bba4afff9eedfa0b03c1de815cfe738151f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 19:10:07 +0100 Subject: [PATCH 27/80] Added utility function to get bytes from a str/unicode. --- psycopg/psycopg.h | 1 + psycopg/python.h | 2 ++ psycopg/utils.c | 26 ++++++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index 00cb98ad..57a73d02 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -121,6 +121,7 @@ HIDDEN char *psycopg_escape_string(PyObject *conn, const char *from, Py_ssize_t len, char *to, Py_ssize_t *tolen); HIDDEN char *psycopg_strdup(const char *from, Py_ssize_t len); +HIDDEN PyObject * psycopg_ensure_bytes(PyObject *obj); /* Exceptions docstrings */ #define Error_doc \ diff --git a/psycopg/python.h b/psycopg/python.h index 452c7341..ee2dee76 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -106,6 +106,7 @@ /* XXX BytesType -> Bytes_Type */ #define BytesType PyString_Type #define Bytes_Check PyString_Check +#define Bytes_CheckExact PyString_CheckExact #define Bytes_AS_STRING PyString_AS_STRING #define Bytes_GET_SIZE PyString_GET_SIZE #define Bytes_Size PyString_Size @@ -120,6 +121,7 @@ #define BytesType PyBytes_Type #define Bytes_Check PyBytes_Check +#define Bytes_CheckExact PyBytes_CheckExact #define Bytes_AS_STRING PyBytes_AS_STRING #define Bytes_GET_SIZE PyBytes_GET_SIZE #define Bytes_Size PyBytes_Size diff --git a/psycopg/utils.c b/psycopg/utils.c index f2a0ab09..e5b221f9 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -92,3 +92,29 @@ psycopg_strdup(const char *from, Py_ssize_t len) return rv; } +/* Ensure a Python object is a bytes string. + * + * Useful when a char * is required out of it. + * + * Return a new reference. NULL on error. + */ +PyObject * +psycopg_ensure_bytes(PyObject *obj) +{ + PyObject *rv = NULL; + + if (PyUnicode_CheckExact(obj)) { + rv = PyUnicode_AsUTF8String(obj); + } + else if (Bytes_CheckExact(obj)) { + Py_INCREF(obj); + rv = obj; + } + else { + PyErr_Format(PyExc_TypeError, "I'm not into ensuring %s as bytes", + obj ? Py_TYPE(obj)->tp_name : "NULL"); + } + + return rv; +} + From b5ef5ef21d7e5602ae1125631db4e95e2db2cd6c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 19:11:25 +0100 Subject: [PATCH 28/80] Added typecasters repr(). --- psycopg/typecast.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/psycopg/typecast.c b/psycopg/typecast.c index a58f3a1d..b59edb05 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -426,6 +426,24 @@ typecast_del(void *self) PyObject_GC_Del(self); } +static PyObject * +typecast_repr(PyObject *self) +{ + PyObject *name = ((typecastObject *)self)->name; + PyObject *bname; + PyObject *rv; + + if (!(bname = psycopg_ensure_bytes(name))) { + return NULL; + } + + rv = PyString_FromFormat("<%s '%s' at %p>", + Py_TYPE(self)->tp_name, PyBytes_AS_STRING(name), self); + + Py_DECREF(bname); + return rv; +} + static PyObject * typecast_call(PyObject *obj, PyObject *args, PyObject *kwargs) { @@ -458,7 +476,7 @@ PyTypeObject typecastType = { 0, /*tp_getattr*/ 0, /*tp_setattr*/ typecast_cmp, /*tp_compare*/ - 0, /*tp_repr*/ + typecast_repr, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ From 89e4d4c7bb9534711c03df054f90fcad4787d209 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 03:38:37 +0100 Subject: [PATCH 29/80] Empty lists correctly roundtrip. --- NEWS-2.3 | 1 + psycopg/adapter_list.c | 2 +- tests/types_basic.py | 7 ++++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/NEWS-2.3 b/NEWS-2.3 index 0e2229a1..704996e4 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -5,6 +5,7 @@ What's new in psycopg 2.3.2 missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, LATIN10, SHIFT_JIS_2004. - Dropped repeated dictionary lookups with unicode query/parameters. + - Empty lists correctly roundtrip Python -> PostgreSQL -> Python. What's new in psycopg 2.3.1 diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 8973e9dc..946dd221 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -45,7 +45,7 @@ list_quote(listObject *self) /* empty arrays are converted to NULLs (still searching for a way to insert an empty array in postgresql */ - if (len == 0) return Text_FromUTF8("'{}'"); + if (len == 0) return Bytes_FromString("'{}'::text[]"); tmp = PyTuple_New(len); diff --git a/tests/types_basic.py b/tests/types_basic.py index f14acc56..6dd08913 100755 --- a/tests/types_basic.py +++ b/tests/types_basic.py @@ -126,11 +126,12 @@ class TypesBasicTests(unittest.TestCase): self.failUnless(str(buf2) == s, "wrong binary quoting") def testArray(self): + s = self.execute("SELECT %s AS foo", ([],)) + self.failUnlessEqual(s, []) s = self.execute("SELECT %s AS foo", ([[1,2],[3,4]],)) - self.failUnless(s == [[1,2],[3,4]], "wrong array quoting " + str(s)) + self.failUnlessEqual(s, [[1,2],[3,4]]) s = self.execute("SELECT %s AS foo", (['one', 'two', 'three'],)) - self.failUnless(s == ['one', 'two', 'three'], - "wrong array quoting " + str(s)) + self.failUnlessEqual(s, ['one', 'two', 'three']) def testTypeRoundtripBinary(self): o1 = buffer("".join(map(chr, range(256)))) From 014b6a6d5b6ba1cbc35c49f7200644ba3dfb86be Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 23 Dec 2010 19:25:59 +0100 Subject: [PATCH 30/80] Use psycopg_ensure_bytes() to unify Py2/3 code paths. --- psycopg/adapter_datetime.c | 118 +++++++++++++++++++++---------------- psycopg/connection_int.c | 16 +---- 2 files changed, 68 insertions(+), 66 deletions(-) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 2b9f69fd..12547b05 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -54,64 +54,78 @@ psyco_adapter_datetime_init(void) /* datetime_str, datetime_getquoted - return result of quoting */ +static PyObject * +_pydatetime_string_date_time(pydatetimeObject *self) +{ + PyObject *rv = NULL; + PyObject *iso = NULL; + PyObject *biso = NULL; + PyObject *tz; + + /* Select the right PG type to cast into. */ + char *fmt = NULL; + switch (self->type) { + case PSYCO_DATETIME_TIME: + fmt = "'%s'::time"; + break; + case PSYCO_DATETIME_DATE: + fmt = "'%s'::date"; + break; + case PSYCO_DATETIME_TIMESTAMP: + tz = PyObject_GetAttrString(self->wrapped, "tzinfo"); + if (!tz) { goto error; } + fmt = (tz == Py_None) ? "'%s'::timestamp" : "'%s'::timestamptz"; + Py_DECREF(tz); + break; + } + + if (!(iso = PyObject_CallMethod(self->wrapped, "isoformat", NULL))) { + goto error; + } + + if (!(biso = psycopg_ensure_bytes(iso))) { + goto error; + } + + rv = Bytes_FromFormat(fmt, Bytes_AsString(biso)); + + Py_DECREF(biso); + Py_DECREF(iso); + return rv; + +error: + Py_XDECREF(biso); + Py_XDECREF(iso); + return rv; +} + +static PyObject * +_pydatetime_string_delta(pydatetimeObject *self) +{ + PyDateTime_Delta *obj = (PyDateTime_Delta*)self->wrapped; + + char buffer[8]; + int i; + int a = obj->microseconds; + + for (i=0; i < 6 ; i++) { + buffer[5-i] = '0' + (a % 10); + a /= 10; + } + buffer[6] = '\0'; + + return Bytes_FromFormat("'%d days %d.%s seconds'::interval", + obj->days, obj->seconds, buffer); +} + static PyObject * pydatetime_str(pydatetimeObject *self) { - PyObject *res = NULL; - PyObject *iso; if (self->type <= PSYCO_DATETIME_TIMESTAMP) { - PyObject *tz; - - /* Select the right PG type to cast into. */ - char *fmt = NULL; - switch (self->type) { - case PSYCO_DATETIME_TIME: - fmt = "'%s'::time"; - break; - case PSYCO_DATETIME_DATE: - fmt = "'%s'::date"; - break; - case PSYCO_DATETIME_TIMESTAMP: - tz = PyObject_GetAttrString(self->wrapped, "tzinfo"); - if (!tz) { return NULL; } - fmt = (tz == Py_None) ? "'%s'::timestamp" : "'%s'::timestamptz"; - Py_DECREF(tz); - break; - } - - iso = PyObject_CallMethod(self->wrapped, "isoformat", NULL); - if (iso) { -#if PY_MAJOR_VERSION > 2 - { - PyObject *biso; - if (!(biso = PyUnicode_AsEncodedString(iso, "ascii", NULL))) { - Py_DECREF(iso); - return NULL; - } - Py_DECREF(iso); - iso = biso; - } -#endif - res = Bytes_FromFormat(fmt, Bytes_AsString(iso)); - Py_DECREF(iso); - } - return res; + return _pydatetime_string_date_time(self); } else { - PyDateTime_Delta *obj = (PyDateTime_Delta*)self->wrapped; - - char buffer[8]; - int i; - int a = obj->microseconds; - - for (i=0; i < 6 ; i++) { - buffer[5-i] = '0' + (a % 10); - a /= 10; - } - buffer[6] = '\0'; - - return Bytes_FromFormat("'%d days %d.%s seconds'::interval", - obj->days, obj->seconds, buffer); + return _pydatetime_string_delta(self); } } diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 21889938..6d876228 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -235,20 +235,8 @@ conn_encoding_to_codec(const char *enc) goto exit; } - /* Convert the codec in a bytes string to extract the c string. - * At the end of the block we have pybenc with a new ref. */ - if (PyUnicode_Check(pyenc)) { - if (!(pybenc = PyUnicode_AsEncodedString(pyenc, "ascii", NULL))) { - goto exit; - } - } - else if (Bytes_Check(pyenc)) { - Py_INCREF(pyenc); - pybenc = pyenc; - } - else { - PyErr_Format(PyExc_TypeError, "bad type for encoding: %s", - Py_TYPE(pyenc)->tp_name); + /* Convert the codec in a bytes string to extract the c string. */ + if (!(pybenc = psycopg_ensure_bytes(pyenc))) { goto exit; } From 56e4c2bd55239dc30d4feee6adafe851f13b8e24 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 24 Dec 2010 15:43:27 +0100 Subject: [PATCH 31/80] Redefining the microprotocol on Py3 as returning bytes. --- psycopg/adapter_asis.c | 2 +- psycopg/adapter_binary.c | 17 ++++++++++++----- psycopg/adapter_list.c | 10 +++------- psycopg/adapter_qstring.c | 5 ++--- psycopg/cursor_type.c | 6 +++--- psycopg/microprotocols.c | 15 ++++++++++++++- 6 files changed, 35 insertions(+), 20 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index 1aa1d0da..7c7cb811 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -38,7 +38,7 @@ static PyObject * asis_str(asisObject *self) { if (self->wrapped == Py_None) { - return Text_FromUTF8("NULL"); + return Bytes_FromString("NULL"); } else { return PyObject_Str(self->wrapped); diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 22105070..6fa7b590 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -62,6 +62,7 @@ binary_quote(binaryObject *self) #if PY_MAJOR_VERSION < 3 || PyBuffer_Check(self->wrapped) #else + || PyByteArray_Check(self->wrapped) || PyMemoryView_Check(self->wrapped) #endif ) { @@ -78,11 +79,11 @@ binary_quote(binaryObject *self) } if (len > 0) - self->buffer = PyString_FromFormat( + self->buffer = Bytes_FromFormat( (self->conn && ((connectionObject*)self->conn)->equote) ? "E'%s'::bytea" : "'%s'::bytea" , to); else - self->buffer = Text_FromUTF8("''::bytea"); + self->buffer = Bytes_FromString("''::bytea"); PQfreemem(to); } @@ -97,15 +98,21 @@ binary_quote(binaryObject *self) } /* binary_str, binary_getquoted - return result of quoting */ - +/* XXX what is the point of this method? */ static PyObject * binary_str(binaryObject *self) { if (self->buffer == NULL) { - binary_quote(self); + if (!(binary_quote(self))) { + return NULL; + } } - Py_XINCREF(self->buffer); +#if PY_MAJOR_VERSION < 3 + Py_INCREF(self->buffer); return self->buffer; +#else + return PyUnicode_FromEncodedObject(self->buffer, "ascii", "replace"); +#endif } static PyObject * diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 946dd221..522c49dd 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -53,7 +53,7 @@ list_quote(listObject *self) PyObject *quoted; PyObject *wrapped = PyList_GET_ITEM(self->wrapped, i); if (wrapped == Py_None) - quoted = Text_FromUTF8("NULL"); + quoted = Bytes_FromString("NULL"); else quoted = microprotocol_getquoted(wrapped, (connectionObject*)self->connection); @@ -67,15 +67,11 @@ list_quote(listObject *self) /* now that we have a tuple of adapted objects we just need to join them and put "ARRAY[] around the result */ - str = Text_FromUTF8(", "); + str = Bytes_FromString(", "); joined = PyObject_CallMethod(str, "join", "(O)", tmp); if (joined == NULL) goto error; -#if PY_MAJOR_VERSION < 3 - res = PyString_FromFormat("ARRAY[%s]", PyString_AsString(joined)); -#else - res = PyUnicode_FromFormat("ARRAY[%U]", joined); -#endif + res = Bytes_FromFormat("ARRAY[%s]", Bytes_AsString(joined)); error: Py_XDECREF(tmp); diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index c1082d8e..34ed4101 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -95,9 +95,8 @@ qstring_quote(qstringObject *self) Py_DECREF(str); return NULL; } - - /* XXX need to decode in connection's encoding in 3.0 */ - self->buffer = Text_FromUTF8AndSize(buffer, qlen); + + self->buffer = Bytes_FromStringAndSize(buffer, qlen); PyMem_Free(buffer); Py_DECREF(str); diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 922288b5..5290c238 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -144,7 +144,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) optimization over the adapting code and can go away in the future if somebody finds a None adapter usefull. */ if (value == Py_None) { - t = Text_FromUTF8("NULL"); + t = Bytes_FromString("NULL"); PyDict_SetItem(n, key, t); /* t is a new object, refcnt = 1, key is at 2 */ @@ -220,7 +220,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) d = c+1; if (value == Py_None) { - PyTuple_SET_ITEM(n, index, Text_FromUTF8("NULL")); + PyTuple_SET_ITEM(n, index, Bytes_FromString("NULL")); while (*d && !isalpha(*d)) d++; if (*d) *d = 's'; Py_DECREF(value); @@ -950,7 +950,7 @@ psyco_curs_callproc(cursorObject *self, PyObject *args, PyObject *kwargs) sql[sl-2] = ')'; sql[sl-1] = '\0'; - operation = Text_FromUTF8(sql); + operation = Bytes_FromString(sql); PyMem_Free((void*)sql); if (_psyco_curs_execute(self, operation, parameters, self->conn->async)) { diff --git a/psycopg/microprotocols.c b/psycopg/microprotocols.c index 067729f2..45b26f85 100644 --- a/psycopg/microprotocols.c +++ b/psycopg/microprotocols.c @@ -203,7 +203,10 @@ microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) return NULL; } -/* microprotocol_getquoted - utility function that adapt and call getquoted */ +/* microprotocol_getquoted - utility function that adapt and call getquoted. + * + * Return a bytes string, NULL on error. + */ PyObject * microprotocol_getquoted(PyObject *obj, connectionObject *conn) @@ -241,6 +244,16 @@ microprotocol_getquoted(PyObject *obj, connectionObject *conn) adapted to the right protocol) */ res = PyObject_CallMethod(adapted, "getquoted", NULL); + /* Convert to bytes. */ + if (res && PyUnicode_CheckExact(res)) { + PyObject *b; + const char *codec; + codec = (conn && conn->codec) ? conn->codec : "utf8"; + b = PyUnicode_AsEncodedString(res, codec, NULL); + Py_DECREF(res); + res = b; + } + exit: Py_XDECREF(adapted); Py_XDECREF(prepare); From beba0649838e0336279dc3dc52949cb96e3fefb9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 24 Dec 2010 15:44:29 +0100 Subject: [PATCH 32/80] Test on basic adapters pass on Python 3. --- tests/testutils.py | 23 ++++++++++ tests/types_basic.py | 100 ++++++++++++++++++++++++++++++++++++------- 2 files changed, 108 insertions(+), 15 deletions(-) diff --git a/tests/testutils.py b/tests/testutils.py index b7c0faba..e3ea6877 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -4,6 +4,8 @@ # Use unittest2 if available. Otherwise mock a skip facility with warnings. +import sys + try: import unittest2 unittest = unittest2 @@ -69,3 +71,24 @@ def skip_if_no_pg_sleep(name): return skip_if_no_pg_sleep__ return skip_if_no_pg_sleep_ + +def skip_on_python2(f): + """Skip a test on Python 3 and following.""" + def skip_on_python2_(self): + if sys.version_info[0] < 3: + return self.skipTest("skipped because Python 2") + else: + return f(self) + + return skip_on_python2_ + +def skip_on_python3(f): + """Skip a test on Python 3 and following.""" + def skip_on_python3_(self): + if sys.version_info[0] >= 3: + return self.skipTest("skipped because Python 3") + else: + return f(self) + + return skip_on_python3_ + diff --git a/tests/types_basic.py b/tests/types_basic.py index 6dd08913..b9d34437 100755 --- a/tests/types_basic.py +++ b/tests/types_basic.py @@ -27,6 +27,7 @@ try: except: pass import sys +import testutils from testutils import unittest import psycopg2 @@ -62,13 +63,13 @@ class TypesBasicTests(unittest.TestCase): self.failUnless(s == 1971, "wrong integer quoting: " + str(s)) s = self.execute("SELECT %s AS foo", (1971L,)) self.failUnless(s == 1971L, "wrong integer quoting: " + str(s)) - if sys.version_info[0] < 2 or sys.version_info[1] < 4: + if sys.version_info[0:2] < (2, 4): s = self.execute("SELECT %s AS foo", (19.10,)) self.failUnless(abs(s - 19.10) < 0.001, "wrong float quoting: " + str(s)) def testDecimal(self): - if sys.version_info[0] >= 2 and sys.version_info[1] >= 4: + if sys.version_info[0:2] >= (2, 4): s = self.execute("SELECT %s AS foo", (decimal.Decimal("19.10"),)) self.failUnless(s - decimal.Decimal("19.10") == 0, "wrong decimal quoting: " + str(s)) @@ -101,29 +102,44 @@ class TypesBasicTests(unittest.TestCase): return self.skipTest("inf::float not available on the server") except ValueError: return self.skipTest("inf not available on this platform") - s = self.execute("SELECT %s AS foo", (float("inf"),)) self.failUnless(str(s) == "inf", "wrong float quoting: " + str(s)) self.failUnless(type(s) == float, "wrong float conversion: " + repr(s)) def testBinary(self): - s = ''.join([chr(x) for x in range(256)]) - b = psycopg2.Binary(s) - buf = self.execute("SELECT %s::bytea AS foo", (b,)) - self.failUnless(str(buf) == s, "wrong binary quoting") + if sys.version_info[0] < 3: + s = ''.join([chr(x) for x in range(256)]) + b = psycopg2.Binary(s) + buf = self.execute("SELECT %s::bytea AS foo", (b,)) + self.assertEqual(s, str(buf)) + else: + s = bytes(range(256)) + b = psycopg2.Binary(s) + buf = self.execute("SELECT %s::bytea AS foo", (b,)) + self.assertEqual(s, buf) def testBinaryEmptyString(self): # test to make sure an empty Binary is converted to an empty string - b = psycopg2.Binary('') - self.assertEqual(str(b), "''::bytea") + if sys.version_info[0] < 3: + b = psycopg2.Binary('') + self.assertEqual(str(b), "''::bytea") + else: + b = psycopg2.Binary(bytes([])) + self.assertEqual(str(b), "''::bytea") def testBinaryRoundTrip(self): # test to make sure buffers returned by psycopg2 are # understood by execute: - s = ''.join([chr(x) for x in range(256)]) - buf = self.execute("SELECT %s::bytea AS foo", (psycopg2.Binary(s),)) - buf2 = self.execute("SELECT %s::bytea AS foo", (buf,)) - self.failUnless(str(buf2) == s, "wrong binary quoting") + if sys.version_info[0] < 3: + s = ''.join([chr(x) for x in range(256)]) + buf = self.execute("SELECT %s::bytea AS foo", (psycopg2.Binary(s),)) + buf2 = self.execute("SELECT %s::bytea AS foo", (buf,)) + self.assertEqual(s, str(buf2)) + else: + s = bytes(range(256)) + buf = self.execute("SELECT %s::bytea AS foo", (psycopg2.Binary(s),)) + buf2 = self.execute("SELECT %s::bytea AS foo", (buf,)) + self.assertEqual(s, buf2) def testArray(self): s = self.execute("SELECT %s AS foo", ([],)) @@ -133,7 +149,8 @@ class TypesBasicTests(unittest.TestCase): s = self.execute("SELECT %s AS foo", (['one', 'two', 'three'],)) self.failUnlessEqual(s, ['one', 'two', 'three']) - def testTypeRoundtripBinary(self): + @testutils.skip_on_python3 + def testTypeRoundtripBuffer(self): o1 = buffer("".join(map(chr, range(256)))) o2 = self.execute("select %s;", (o1,)) self.assertEqual(type(o1), type(o2)) @@ -143,12 +160,53 @@ class TypesBasicTests(unittest.TestCase): o2 = self.execute("select %s;", (o1,)) self.assertEqual(type(o1), type(o2)) - def testTypeRoundtripBinaryArray(self): + @testutils.skip_on_python3 + def testTypeRoundtripBufferArray(self): o1 = buffer("".join(map(chr, range(256)))) o1 = [o1] o2 = self.execute("select %s;", (o1,)) self.assertEqual(type(o1[0]), type(o2[0])) + @testutils.skip_on_python2 + def testTypeRoundtripBytes(self): + o1 = bytes(range(256)) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + # Test with an empty buffer + o1 = bytes([]) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + @testutils.skip_on_python2 + def testTypeRoundtripBytesArray(self): + o1 = bytes(range(256)) + o1 = [o1] + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2[0])) + + @testutils.skip_on_python2 + def testAdaptBytearray(self): + o1 = bytearray(range(256)) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + # Test with an empty buffer + o1 = bytearray([]) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + @testutils.skip_on_python2 + def testAdaptMemoryview(self): + o1 = memoryview(bytes(range(256))) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + + # Test with an empty buffer + o1 = memoryview(bytes([])) + o2 = self.execute("select %s;", (o1,)) + self.assertEqual(memoryview, type(o2)) + class AdaptSubclassTest(unittest.TestCase): def test_adapt_subtype(self): @@ -169,6 +227,7 @@ class AdaptSubclassTest(unittest.TestCase): register_adapter(B, lambda b: AsIs("b")) self.assertEqual('b', adapt(C()).getquoted()) + @testutils.skip_on_python3 def test_no_mro_no_joy(self): from psycopg2.extensions import adapt, register_adapter, AsIs @@ -178,6 +237,17 @@ class AdaptSubclassTest(unittest.TestCase): register_adapter(A, lambda a: AsIs("a")) self.assertRaises(psycopg2.ProgrammingError, adapt, B()) + @testutils.skip_on_python2 + def test_adapt_subtype_3(self): + from psycopg2.extensions import adapt, register_adapter, AsIs + + class A: pass + class B(A): pass + + register_adapter(A, lambda a: AsIs("a")) + self.assertEqual("a", adapt(B()).getquoted()) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 60841c65676941cd26d91cc6b704bd4c67fe2cb9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 24 Dec 2010 15:54:01 +0100 Subject: [PATCH 33/80] Added regression test on bool adaptation. --- tests/types_basic.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/types_basic.py b/tests/types_basic.py index b9d34437..4591290b 100755 --- a/tests/types_basic.py +++ b/tests/types_basic.py @@ -68,6 +68,12 @@ class TypesBasicTests(unittest.TestCase): self.failUnless(abs(s - 19.10) < 0.001, "wrong float quoting: " + str(s)) + def testBoolean(self): + x = self.execute("SELECT %s as foo", (False,)) + self.assert_(x is False) + x = self.execute("SELECT %s as foo", (True,)) + self.assert_(x is True) + def testDecimal(self): if sys.version_info[0:2] >= (2, 4): s = self.execute("SELECT %s AS foo", (decimal.Decimal("19.10"),)) From d3f3f1caf0177f32b70dcb7632ad0a164abcfc65 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 25 Dec 2010 11:43:42 +0100 Subject: [PATCH 34/80] Added utility method to return a string in the connection encoding. In Py2 the result is plain string, in Py3 an unicode decoded in the connection encoding. --- psycopg/connection.h | 1 + psycopg/connection_int.c | 26 ++++++++++++++++++++------ psycopg/connection_type.c | 2 +- psycopg/notify_type.c | 3 +-- psycopg/pqpath.c | 5 ++--- psycopg/psycopgmodule.c | 18 +++++++++--------- 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/psycopg/connection.h b/psycopg/connection.h index 41e0cdc1..2b66029c 100644 --- a/psycopg/connection.h +++ b/psycopg/connection.h @@ -119,6 +119,7 @@ typedef struct { } connectionObject; /* C-callable functions in connection_int.c and connection_ext.c */ +HIDDEN PyObject *conn_text_from_chars(connectionObject *pgconn, const char *str); HIDDEN int conn_get_standard_conforming_strings(PGconn *pgconn); HIDDEN int conn_get_isolation_level(PGresult *pgres); HIDDEN int conn_get_protocol_version(PGconn *pgconn); diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 6d876228..ee4bebcc 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -35,6 +35,23 @@ #include +/* Return a new "string" from a char* from the database. + * + * On Py2 just get a string, on Py3 decode it in the connection codec. + * + * Use a fallback if the connection is NULL. + */ +PyObject * +conn_text_from_chars(connectionObject *self, const char *str) +{ +#if PY_MAJOR_VERSION < 3 + return PyString_FromString(str); +#else + const char *codec = self ? self->codec : "ascii"; + return PyUnicode_Decode(str, strlen(str), codec, "replace"); +#endif +} + /* conn_notice_callback - process notices */ static void @@ -76,9 +93,7 @@ conn_notice_process(connectionObject *self) while (notice != NULL) { PyObject *msg; - /* XXX possible other encode I think */ - msg = Text_FromUTF8(notice->message); - + msg = conn_text_from_chars(self, notice->message); Dprintf("conn_notice_process: %s", notice->message); /* Respect the order in which notices were produced, @@ -146,9 +161,8 @@ conn_notifies_process(connectionObject *self) (int) pgn->be_pid, pgn->relname); if (!(pid = PyInt_FromLong((long)pgn->be_pid))) { goto error; } - /* XXX in the connection encoding? */ - if (!(channel = Text_FromUTF8(pgn->relname))) { goto error; } - if (!(payload = Text_FromUTF8(pgn->extra))) { goto error; } + if (!(channel = conn_text_from_chars(self, pgn->relname))) { goto error; } + if (!(payload = conn_text_from_chars(self, pgn->extra))) { goto error; } if (!(notify = PyObject_CallFunctionObjArgs((PyObject *)&NotifyType, pid, channel, payload, NULL))) { diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index 49c6b942..0162be3c 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -498,7 +498,7 @@ psyco_conn_get_parameter_status(connectionObject *self, PyObject *args) Py_INCREF(Py_None); return Py_None; } - return Text_FromUTF8(val); + return conn_text_from_chars(self, val); } diff --git a/psycopg/notify_type.c b/psycopg/notify_type.c index e68daa38..ec2ec36f 100644 --- a/psycopg/notify_type.c +++ b/psycopg/notify_type.c @@ -78,8 +78,7 @@ notify_init(NotifyObject *self, PyObject *args, PyObject *kwargs) } if (!payload) { - /* XXX review encoding */ - payload = Text_FromUTF8AndSize("", 0); + payload = Text_FromUTF8(""); } Py_CLEAR(self->pid); diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 745919b9..9ba94152 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -987,8 +987,7 @@ _pq_fetch_tuples(cursorObject *curs) /* 1/ fill the other fields */ PyTuple_SET_ITEM(dtitem, 0, - /* XXX guaranteed to be ASCII/UTF8? */ - Text_FromUTF8(PQfname(curs->pgres, i))); + conn_text_from_chars(curs->conn, PQfname(curs->pgres, i))); PyTuple_SET_ITEM(dtitem, 1, type); /* 2/ display size is the maximum size of this field result tuples. */ @@ -1220,7 +1219,7 @@ pq_fetch(cursorObject *curs) /* backend status message */ Py_XDECREF(curs->pgstatus); - curs->pgstatus = Text_FromUTF8(PQcmdStatus(curs->pgres)); + curs->pgstatus = conn_text_from_chars(curs->conn, PQcmdStatus(curs->pgres)); switch(pgstatus) { diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index 568cf6ec..d580fa01 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -582,26 +582,26 @@ psyco_set_error(PyObject *exc, PyObject *curs, const char *msg, PyObject *err = PyObject_CallFunction(exc, "s", msg); if (err) { + connectionObject *conn = NULL; + if (curs) { + PyObject_SetAttrString(err, "cursor", curs); + conn = ((cursorObject *)curs)->conn; + } + if (pgerror) { - /* XXX is this always ASCII? If not, it needs - to be decoded properly for Python 3. */ - t = Text_FromUTF8(pgerror); + t = conn_text_from_chars(conn, pgerror); PyObject_SetAttrString(err, "pgerror", t); Py_DECREF(t); } if (pgcode) { - /* XXX likewise */ - t = Text_FromUTF8(pgcode); + t = conn_text_from_chars(conn, pgcode); PyObject_SetAttrString(err, "pgcode", t); Py_DECREF(t); } - if (curs) - PyObject_SetAttrString(err, "cursor", curs); - PyErr_SetObject(exc, err); - Py_DECREF(err); + Py_DECREF(err); } } From 2e22eef727fc3b57e63c7a6a1b5cbae9533cd632 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 25 Dec 2010 11:57:04 +0100 Subject: [PATCH 35/80] Added utility function to convert bytes to string in Python 3. --- psycopg/psycopg.h | 1 + psycopg/utils.c | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index 57a73d02..e87744c7 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -122,6 +122,7 @@ HIDDEN char *psycopg_escape_string(PyObject *conn, HIDDEN char *psycopg_strdup(const char *from, Py_ssize_t len); HIDDEN PyObject * psycopg_ensure_bytes(PyObject *obj); +HIDDEN PyObject * psycopg_ensure_text(PyObject *obj); /* Exceptions docstrings */ #define Error_doc \ diff --git a/psycopg/utils.c b/psycopg/utils.c index e5b221f9..16b92498 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -118,3 +118,27 @@ psycopg_ensure_bytes(PyObject *obj) return rv; } +/* Take a Python object and return text from it. + * + * On Py3 this means converting bytes to unicode. On Py2 bytes are fine. + * + * The function is ref neutral: steals a ref from obj and adds one to the + * return value. It is safe to call it on NULL. + */ +PyObject * +psycopg_ensure_text(PyObject *obj) +{ +#if PY_MAJOR_VERSION < 3 + return obj; +#else + if (obj) { + /* bytes to unicode in Py3 */ + PyObject *rv = PyUnicode_FromEncodedObject(obj, "utf8", "replace"); + Py_DECREF(obj); + return rv; + } + else { + return NULL; + } +#endif +} From 3214c23f51c8effa7e78f9a7f59735c5b3e10868 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 25 Dec 2010 12:03:15 +0100 Subject: [PATCH 36/80] Fixed adaptation in several adapters. The getquoted methods always return bytes. The str() convert this representation to string on the fly. --- psycopg/adapter_asis.c | 21 ++++++++++++++++----- psycopg/adapter_binary.c | 12 ++++-------- psycopg/adapter_datetime.c | 6 +++--- psycopg/adapter_list.c | 2 +- psycopg/adapter_pboolean.c | 14 +++++++------- psycopg/adapter_pdecimal.c | 20 ++++++++++++++------ psycopg/adapter_pfloat.c | 27 ++++++++++++++++++++------- psycopg/adapter_qstring.c | 6 +++--- psycopg/microprotocols.c | 2 +- 9 files changed, 69 insertions(+), 41 deletions(-) diff --git a/psycopg/adapter_asis.c b/psycopg/adapter_asis.c index 7c7cb811..bc64aa74 100644 --- a/psycopg/adapter_asis.c +++ b/psycopg/adapter_asis.c @@ -35,20 +35,31 @@ /** the AsIs object **/ static PyObject * -asis_str(asisObject *self) +asis_getquoted(asisObject *self, PyObject *args) { + PyObject *rv; if (self->wrapped == Py_None) { - return Bytes_FromString("NULL"); + rv = Bytes_FromString("NULL"); } else { - return PyObject_Str(self->wrapped); + rv = PyObject_Str(self->wrapped); +#if PY_MAJOR_VERSION > 2 + /* unicode to bytes in Py3 */ + if (rv) { + PyObject *tmp = PyUnicode_AsUTF8String(rv); + Py_DECREF(rv); + rv = tmp; + } +#endif } + + return rv; } static PyObject * -asis_getquoted(asisObject *self, PyObject *args) +asis_str(asisObject *self) { - return asis_str(self); + return psycopg_ensure_text(asis_getquoted(self, NULL)); } static PyObject * diff --git a/psycopg/adapter_binary.c b/psycopg/adapter_binary.c index 6fa7b590..ccfaf240 100644 --- a/psycopg/adapter_binary.c +++ b/psycopg/adapter_binary.c @@ -98,27 +98,23 @@ binary_quote(binaryObject *self) } /* binary_str, binary_getquoted - return result of quoting */ -/* XXX what is the point of this method? */ + static PyObject * -binary_str(binaryObject *self) +binary_getquoted(binaryObject *self, PyObject *args) { if (self->buffer == NULL) { if (!(binary_quote(self))) { return NULL; } } -#if PY_MAJOR_VERSION < 3 Py_INCREF(self->buffer); return self->buffer; -#else - return PyUnicode_FromEncodedObject(self->buffer, "ascii", "replace"); -#endif } static PyObject * -binary_getquoted(binaryObject *self, PyObject *args) +binary_str(binaryObject *self) { - return binary_str(self); + return psycopg_ensure_text(binary_getquoted(self, NULL)); } static PyObject * diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 12547b05..56be0c9b 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -119,7 +119,7 @@ _pydatetime_string_delta(pydatetimeObject *self) } static PyObject * -pydatetime_str(pydatetimeObject *self) +pydatetime_getquoted(pydatetimeObject *self, PyObject *args) { if (self->type <= PSYCO_DATETIME_TIMESTAMP) { return _pydatetime_string_date_time(self); @@ -130,9 +130,9 @@ pydatetime_str(pydatetimeObject *self) } static PyObject * -pydatetime_getquoted(pydatetimeObject *self, PyObject *args) +pydatetime_str(pydatetimeObject *self) { - return pydatetime_str(self); + return psycopg_ensure_text(pydatetime_getquoted(self, NULL)); } static PyObject * diff --git a/psycopg/adapter_list.c b/psycopg/adapter_list.c index 522c49dd..b7e1f967 100644 --- a/psycopg/adapter_list.c +++ b/psycopg/adapter_list.c @@ -83,7 +83,7 @@ list_quote(listObject *self) static PyObject * list_str(listObject *self) { - return list_quote(self); + return psycopg_ensure_text(list_quote(self)); } static PyObject * diff --git a/psycopg/adapter_pboolean.c b/psycopg/adapter_pboolean.c index cdd3ef4e..4e2c4464 100644 --- a/psycopg/adapter_pboolean.c +++ b/psycopg/adapter_pboolean.c @@ -35,29 +35,29 @@ /** the Boolean object **/ static PyObject * -pboolean_str(pbooleanObject *self) +pboolean_getquoted(pbooleanObject *self, PyObject *args) { #ifdef PSYCOPG_NEW_BOOLEAN if (PyObject_IsTrue(self->wrapped)) { - return Text_FromUTF8("true"); + return Bytes_FromString("true"); } else { - return Text_FromUTF8("false"); + return Bytes_FromString("false"); } #else if (PyObject_IsTrue(self->wrapped)) { - return Text_FromUTF8("'t'"); + return Bytes_FromString("'t'"); } else { - return Text_FromUTF8("'f'"); + return Bytes_FromString("'f'"); } #endif } static PyObject * -pboolean_getquoted(pbooleanObject *self, PyObject *args) +pboolean_str(pbooleanObject *self) { - return pboolean_str(self); + return psycopg_ensure_text(pboolean_getquoted(self, NULL)); } static PyObject * diff --git a/psycopg/adapter_pdecimal.c b/psycopg/adapter_pdecimal.c index fa4acfab..9b573467 100644 --- a/psycopg/adapter_pdecimal.c +++ b/psycopg/adapter_pdecimal.c @@ -36,7 +36,7 @@ /** the Decimal object **/ static PyObject * -pdecimal_str(pdecimalObject *self) +pdecimal_getquoted(pdecimalObject *self, PyObject *args) { PyObject *check, *res = NULL; check = PyObject_CallMethod(self->wrapped, "is_finite", NULL); @@ -45,7 +45,7 @@ pdecimal_str(pdecimalObject *self) goto end; } else if (check) { - res = Text_FromUTF8("'NaN'::numeric"); + res = Bytes_FromString("'NaN'::numeric"); goto end; } @@ -57,7 +57,7 @@ pdecimal_str(pdecimalObject *self) goto end; } if (PyObject_IsTrue(check)) { - res = Text_FromUTF8("'NaN'::numeric"); + res = Bytes_FromString("'NaN'::numeric"); goto end; } @@ -66,11 +66,19 @@ pdecimal_str(pdecimalObject *self) goto end; } if (PyObject_IsTrue(check)) { - res = Text_FromUTF8("'NaN'::numeric"); + res = Bytes_FromString("'NaN'::numeric"); goto end; } res = PyObject_Str(self->wrapped); +#if PY_MAJOR_VERSION > 2 + /* unicode to bytes in Py3 */ + if (res) { + PyObject *tmp = PyUnicode_AsUTF8String(res); + Py_DECREF(res); + res = tmp; + } +#endif end: Py_XDECREF(check); @@ -78,9 +86,9 @@ end: } static PyObject * -pdecimal_getquoted(pdecimalObject *self, PyObject *args) +pdecimal_str(pdecimalObject *self) { - return pdecimal_str(self); + return psycopg_ensure_text(pdecimal_getquoted(self, NULL)); } static PyObject * diff --git a/psycopg/adapter_pfloat.c b/psycopg/adapter_pfloat.c index 3260d2cd..2753737a 100644 --- a/psycopg/adapter_pfloat.c +++ b/psycopg/adapter_pfloat.c @@ -36,21 +36,34 @@ /** the Float object **/ static PyObject * -pfloat_str(pfloatObject *self) +pfloat_getquoted(pfloatObject *self, PyObject *args) { + PyObject *rv; double n = PyFloat_AsDouble(self->wrapped); if (isnan(n)) - return Text_FromUTF8("'NaN'::float"); + rv = Bytes_FromString("'NaN'::float"); else if (isinf(n)) - return Text_FromUTF8("'Infinity'::float"); - else - return PyObject_Repr(self->wrapped); + rv = Bytes_FromString("'Infinity'::float"); + else { + rv = PyObject_Repr(self->wrapped); + +#if PY_MAJOR_VERSION > 2 + /* unicode to bytes in Py3 */ + if (rv) { + PyObject *tmp = PyUnicode_AsUTF8String(rv); + Py_DECREF(rv); + rv = tmp; + } +#endif + } + + return rv; } static PyObject * -pfloat_getquoted(pfloatObject *self, PyObject *args) +pfloat_str(pfloatObject *self) { - return pfloat_str(self); + return psycopg_ensure_text(pfloat_getquoted(self, NULL)); } static PyObject * diff --git a/psycopg/adapter_qstring.c b/psycopg/adapter_qstring.c index 34ed4101..a9d4ec6e 100644 --- a/psycopg/adapter_qstring.c +++ b/psycopg/adapter_qstring.c @@ -106,7 +106,7 @@ qstring_quote(qstringObject *self) /* qstring_str, qstring_getquoted - return result of quoting */ static PyObject * -qstring_str(qstringObject *self) +qstring_getquoted(qstringObject *self, PyObject *args) { if (self->buffer == NULL) { qstring_quote(self); @@ -116,9 +116,9 @@ qstring_str(qstringObject *self) } static PyObject * -qstring_getquoted(qstringObject *self, PyObject *args) +qstring_str(qstringObject *self) { - return qstring_str(self); + return psycopg_ensure_text(qstring_getquoted(self, NULL)); } static PyObject * diff --git a/psycopg/microprotocols.c b/psycopg/microprotocols.c index 45b26f85..2173815d 100644 --- a/psycopg/microprotocols.c +++ b/psycopg/microprotocols.c @@ -138,7 +138,7 @@ microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) /* None is always adapted to NULL */ if (obj == Py_None) - return Text_FromUTF8("NULL"); + return Bytes_FromString("NULL"); Dprintf("microprotocols_adapt: trying to adapt %s", Py_TYPE(obj)->tp_name); From f6fefbea64d699ca7eb041fb36ebc0ca4d2baa42 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 25 Dec 2010 15:00:05 +0100 Subject: [PATCH 37/80] Function psycopg_ensure_bytes converted in a "filter" stealing a ref. --- psycopg/adapter_datetime.c | 12 +++--------- psycopg/connection_int.c | 9 ++++----- psycopg/typecast.c | 6 +++--- psycopg/utils.c | 15 +++++++++++---- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/psycopg/adapter_datetime.c b/psycopg/adapter_datetime.c index 56be0c9b..ddcd0898 100644 --- a/psycopg/adapter_datetime.c +++ b/psycopg/adapter_datetime.c @@ -59,7 +59,6 @@ _pydatetime_string_date_time(pydatetimeObject *self) { PyObject *rv = NULL; PyObject *iso = NULL; - PyObject *biso = NULL; PyObject *tz; /* Select the right PG type to cast into. */ @@ -79,22 +78,17 @@ _pydatetime_string_date_time(pydatetimeObject *self) break; } - if (!(iso = PyObject_CallMethod(self->wrapped, "isoformat", NULL))) { + if (!(iso = psycopg_ensure_bytes( + PyObject_CallMethod(self->wrapped, "isoformat", NULL)))) { goto error; } - if (!(biso = psycopg_ensure_bytes(iso))) { - goto error; - } + rv = Bytes_FromFormat(fmt, Bytes_AsString(iso)); - rv = Bytes_FromFormat(fmt, Bytes_AsString(biso)); - - Py_DECREF(biso); Py_DECREF(iso); return rv; error: - Py_XDECREF(biso); Py_XDECREF(iso); return rv; } diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index ee4bebcc..6d281b36 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -239,7 +239,6 @@ conn_encoding_to_codec(const char *enc) char *tmp; Py_ssize_t size; PyObject *pyenc = NULL; - PyObject *pybenc = NULL; char *rv = NULL; /* Find the Py codec name from the PG encoding */ @@ -250,11 +249,12 @@ conn_encoding_to_codec(const char *enc) } /* Convert the codec in a bytes string to extract the c string. */ - if (!(pybenc = psycopg_ensure_bytes(pyenc))) { + Py_INCREF(pyenc); + if (!(pyenc = psycopg_ensure_bytes(pyenc))) { goto exit; } - if (-1 == Bytes_AsStringAndSize(pybenc, &tmp, &size)) { + if (-1 == Bytes_AsStringAndSize(pyenc, &tmp, &size)) { goto exit; } @@ -262,8 +262,7 @@ conn_encoding_to_codec(const char *enc) rv = psycopg_strdup(tmp, size); exit: - /* pyenc is borrowed: no decref. */ - Py_XDECREF(pybenc); + Py_XDECREF(pyenc); return rv; } diff --git a/psycopg/typecast.c b/psycopg/typecast.c index b59edb05..cc155f4d 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -430,17 +430,17 @@ static PyObject * typecast_repr(PyObject *self) { PyObject *name = ((typecastObject *)self)->name; - PyObject *bname; PyObject *rv; - if (!(bname = psycopg_ensure_bytes(name))) { + Py_INCREF(name); + if (!(name = psycopg_ensure_bytes(name))) { return NULL; } rv = PyString_FromFormat("<%s '%s' at %p>", Py_TYPE(self)->tp_name, PyBytes_AS_STRING(name), self); - Py_DECREF(bname); + Py_DECREF(name); return rv; } diff --git a/psycopg/utils.c b/psycopg/utils.c index 16b92498..539081ba 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -96,23 +96,30 @@ psycopg_strdup(const char *from, Py_ssize_t len) * * Useful when a char * is required out of it. * - * Return a new reference. NULL on error. + * The function is ref neutral: steals a ref from obj and adds one to the + * return value. This also means that you shouldn't call the function on a + * borrowed ref, if having the object unallocated is not what you want. + * + * It is safe to call the function on NULL. */ PyObject * psycopg_ensure_bytes(PyObject *obj) { PyObject *rv = NULL; + if (!obj) { return NULL; } if (PyUnicode_CheckExact(obj)) { rv = PyUnicode_AsUTF8String(obj); + Py_DECREF(obj); } else if (Bytes_CheckExact(obj)) { - Py_INCREF(obj); rv = obj; } else { - PyErr_Format(PyExc_TypeError, "I'm not into ensuring %s as bytes", - obj ? Py_TYPE(obj)->tp_name : "NULL"); + PyErr_Format(PyExc_TypeError, + "Expected bytes or unicode string, got %s instead", + Py_TYPE(obj)->tp_name); + Py_DECREF(obj); /* steal the ref anyway */ } return rv; From 01565fa6c56fbeb6d4bd4b948ba06bb000f5e8a5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 28 Dec 2010 13:47:29 +0100 Subject: [PATCH 38/80] Added tests to verify a couple of incomplete encodings. --- tests/test_quote.py | 52 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/test_quote.py b/tests/test_quote.py index 83662c72..1e86803c 100755 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -1,9 +1,10 @@ #!/usr/bin/env python +import sys from testutils import unittest +from testconfig import dsn import psycopg2 import psycopg2.extensions -from testconfig import dsn class QuotingTestCase(unittest.TestCase): r"""Checks the correct quoting of strings and binary objects. @@ -78,6 +79,55 @@ class QuotingTestCase(unittest.TestCase): self.assertEqual(res, data) self.assert_(not self.conn.notices) + def test_latin1(self): + self.conn.set_client_encoding('LATIN1') + curs = self.conn.cursor() + if sys.version_info[0] < 3: + data = ''.join(map(chr, range(32, 127) + range(160, 256))) + else: + data = bytes(range(32, 127) + range(160, 256)).decode('latin1') + + # as string + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + # as unicode + if sys.version_info[0] < 3: + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, self.conn) + data = data.decode('latin1') + + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + def test_koi8(self): + self.conn.set_client_encoding('KOI8') + curs = self.conn.cursor() + if sys.version_info[0] < 3: + data = ''.join(map(chr, range(32, 127) + range(128, 256))) + else: + data = bytes(range(32, 127) + range(128, 256)).decode('koi8_r') + + # as string + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + # as unicode + if sys.version_info[0] < 3: + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, self.conn) + data = data.decode('koi8_r') + + curs.execute("SELECT %s::text;", (data,)) + res = curs.fetchone()[0] + self.assertEqual(res, data) + self.assert_(not self.conn.notices) + + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From ab5934dc3533476c1672630b8500cb06551bb75e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 28 Dec 2010 15:12:43 +0100 Subject: [PATCH 39/80] Use ascii_letters instead of letters. The latter is locale-dependent and not available on Py3. --- tests/test_copy.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_copy.py b/tests/test_copy.py index 2f09d5a6..39f99cab 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -105,7 +105,7 @@ class CopyTests(unittest.TestCase): def _copy_from(self, curs, nrecs, srec, copykw): f = StringIO() - for i, c in izip(xrange(nrecs), cycle(string.letters)): + for i, c in izip(xrange(nrecs), cycle(string.ascii_letters)): l = c * srec f.write("%s\t%s\n" % (i,l)) @@ -116,9 +116,9 @@ class CopyTests(unittest.TestCase): self.assertEqual(nrecs, curs.fetchone()[0]) curs.execute("select data from tcopy where id < %s order by id", - (len(string.letters),)) + (len(string.ascii_letters),)) for i, (l,) in enumerate(curs): - self.assertEqual(l, string.letters[i] * srec) + self.assertEqual(l, string.ascii_letters[i] * srec) def _copy_to(self, curs, srec): f = StringIO() @@ -128,11 +128,11 @@ class CopyTests(unittest.TestCase): ntests = 0 for line in f: n, s = line.split() - if int(n) < len(string.letters): - self.assertEqual(s, string.letters[int(n)] * srec) + if int(n) < len(string.ascii_letters): + self.assertEqual(s, string.ascii_letters[int(n)] * srec) ntests += 1 - self.assertEqual(ntests, len(string.letters)) + self.assertEqual(ntests, len(string.ascii_letters)) decorate_all_tests(CopyTests, skip_if_green) From 061079c918a6fdb32ac1ddcd20ba8cc0f0b82043 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 29 Dec 2010 03:28:27 +0100 Subject: [PATCH 40/80] In Py3, decode the tuple values before passing to the typecaster. Not sure this is the best way to go: it is now impossible to write a binary typecaster in Python; furthermore it is the opposite approach of the codecs, which should return bytes. --- psycopg/typecast.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/psycopg/typecast.c b/psycopg/typecast.c index cc155f4d..c321609f 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -640,7 +640,27 @@ typecast_cast(PyObject *obj, const char *str, Py_ssize_t len, PyObject *curs) res = self->ccast(str, len, curs); } else if (self->pcast) { - res = PyObject_CallFunction(self->pcast, "s#O", str, len, curs); + PyObject *s; + /* XXX we have bytes in the adapters and strings in the typecasters. + * are you sure this is ok? + * Notice that this way it is about impossible to create a python + * typecaster on a binary type. */ + if (str) { +#if PY_MAJOR_VERSION < 3 + s = PyString_FromStringAndSize(str, len); +#else + s = PyUnicode_Decode(str, len, + ((cursorObject *)curs)->conn->codec, NULL); +#endif + } + else { + Py_INCREF(Py_None); + s = Py_None; + } + if (s) { + res = PyObject_CallFunctionObjArgs(self->pcast, s, curs, NULL); + Py_DECREF(s); + } } else { PyErr_SetString(Error, "internal error: no casting function found"); From 89c492d3a4a95044344310bb87570c546fc9a9f4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 29 Dec 2010 03:43:19 +0100 Subject: [PATCH 41/80] Added b() function to return bytes in both Py2 and Py3. --- lib/extensions.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/extensions.py b/lib/extensions.py index 7618751c..43092fb3 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -100,6 +100,16 @@ TRANSACTION_STATUS_INTRANS = 2 TRANSACTION_STATUS_INERROR = 3 TRANSACTION_STATUS_UNKNOWN = 4 +import sys as _sys + +# Return bytes from a string +if _sys.version_info[0] < 3: + def b(s): + return s +else: + def b(s): + return s.encode('utf8') + def register_adapter(typ, callable): """Register 'callable' as an ISQLQuote adapter for type 'typ'.""" adapters[(typ, ISQLQuote)] = callable From c176de4bf8327a370fb737bcb3665af8c311decb Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 29 Dec 2010 03:45:24 +0100 Subject: [PATCH 42/80] Hstore adapter compatible with Python 3. --- lib/extras.py | 45 +++++++++++++++++++------------------------ tests/types_extras.py | 35 ++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/lib/extras.py b/lib/extras.py index 215b59a6..ba401a23 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -42,6 +42,7 @@ from psycopg2 import extensions as _ext from psycopg2.extensions import cursor as _cursor from psycopg2.extensions import connection as _connection from psycopg2.extensions import adapt as _A +from psycopg2.extensions import b class DictCursorBase(_cursor): @@ -574,7 +575,7 @@ class HstoreAdapter(object): def _getquoted_8(self): """Use the operators available in PG pre-9.0.""" if not self.wrapped: - return "''::hstore" + return b("''::hstore") adapt = _ext.adapt rv = [] @@ -588,22 +589,23 @@ class HstoreAdapter(object): v.prepare(self.conn) v = v.getquoted() else: - v = 'NULL' + v = b('NULL') - rv.append("(%s => %s)" % (k, v)) + # XXX this b'ing is painfully inefficient! + rv.append(b("(") + k + b(" => ") + v + b(")")) - return "(" + '||'.join(rv) + ")" + return b("(") + b('||').join(rv) + b(")") def _getquoted_9(self): """Use the hstore(text[], text[]) function.""" if not self.wrapped: - return "''::hstore" + return b("''::hstore") k = _ext.adapt(self.wrapped.keys()) k.prepare(self.conn) v = _ext.adapt(self.wrapped.values()) v.prepare(self.conn) - return "hstore(%s, %s)" % (k.getquoted(), v.getquoted()) + return b("hstore(") + k.getquoted() + b(", ") + v.getquoted() + b(")") getquoted = _getquoted_9 @@ -620,13 +622,8 @@ class HstoreAdapter(object): (?:\s*,\s*|$) # pairs separated by comma or end of string. """, regex.VERBOSE) - # backslash decoder - if sys.version_info[0] < 3: - _bsdec = codecs.getdecoder("string_escape") - else: - _bsdec = codecs.getdecoder("unicode_escape") - - def parse(self, s, cur, _decoder=_bsdec): + @classmethod + def parse(self, s, cur, _bsdec=regex.compile(r"\\(.)")): """Parse an hstore representation in a Python string. The hstore is represented as something like:: @@ -644,10 +641,10 @@ class HstoreAdapter(object): if m is None or m.start() != start: raise psycopg2.InterfaceError( "error parsing hstore pair at char %d" % start) - k = _decoder(m.group(1))[0] + k = _bsdec.sub(r'\1', m.group(1)) v = m.group(2) if v is not None: - v = _decoder(v)[0] + v = _bsdec.sub(r'\1', v) rv[k] = v start = m.end() @@ -658,16 +655,14 @@ class HstoreAdapter(object): return rv - parse = classmethod(parse) - + @classmethod def parse_unicode(self, s, cur): """Parse an hstore returning unicode keys and values.""" - codec = codecs.getdecoder(_ext.encodings[cur.connection.encoding]) - bsdec = self._bsdec - decoder = lambda s: codec(bsdec(s)[0]) - return self.parse(s, cur, _decoder=decoder) + if s is None: + return None - parse_unicode = classmethod(parse_unicode) + s = s.decode(_ext.encodings[cur.connection.encoding]) + return self.parse(s, cur) @classmethod def get_oids(self, conn_or_curs): @@ -713,11 +708,11 @@ def register_hstore(conn_or_curs, globally=False, unicode=False): uses a single database you can pass *globally*\=True to have the typecaster registered on all the connections. - By default the returned dicts will have `str` objects as keys and values: + On Python 2, by default the returned dicts will have `str` objects as keys and values: use *unicode*\=True to return `unicode` objects instead. When adapting a dictionary both `str` and `unicode` keys and values are handled (the `unicode` values will be converted according to the current - `~connection.encoding`). + `~connection.encoding`). The option is not available on Python 3. The |hstore| contrib module must be already installed in the database (executing the ``hstore.sql`` script in your ``contrib`` directory). @@ -730,7 +725,7 @@ def register_hstore(conn_or_curs, globally=False, unicode=False): "please install it from your 'contrib/hstore.sql' file") # create and register the typecaster - if unicode: + if sys.version_info[0] < 3 and unicode: cast = HstoreAdapter.parse_unicode else: cast = HstoreAdapter.parse diff --git a/tests/types_extras.py b/tests/types_extras.py index 9a2a2c27..83b4f086 100644 --- a/tests/types_extras.py +++ b/tests/types_extras.py @@ -24,6 +24,8 @@ from testutils import unittest import psycopg2 import psycopg2.extras +from psycopg2.extensions import b + from testconfig import dsn @@ -164,18 +166,17 @@ class HstoreTestCase(unittest.TestCase): a.prepare(self.conn) q = a.getquoted() - self.assert_(q.startswith("(("), q) - self.assert_(q.endswith("))"), q) - ii = q[1:-1].split("||") + self.assert_(q.startswith(b("((")), q) + ii = q[1:-1].split(b("||")) ii.sort() self.assertEqual(len(ii), len(o)) - self.assertEqual(ii[0], filter_scs(self.conn, "(E'a' => E'1')")) - self.assertEqual(ii[1], filter_scs(self.conn, "(E'b' => E'''')")) - self.assertEqual(ii[2], filter_scs(self.conn, "(E'c' => NULL)")) + self.assertEqual(ii[0], filter_scs(self.conn, b("(E'a' => E'1')"))) + self.assertEqual(ii[1], filter_scs(self.conn, b("(E'b' => E'''')"))) + self.assertEqual(ii[2], filter_scs(self.conn, b("(E'c' => NULL)"))) if 'd' in o: encc = u'\xe0'.encode(psycopg2.extensions.encodings[self.conn.encoding]) - self.assertEqual(ii[3], filter_scs(self.conn, "(E'd' => E'%s')" % encc)) + self.assertEqual(ii[3], filter_scs(self.conn, b("(E'd' => E'") + encc + b("')"))) def test_adapt_9(self): if self.conn.server_version < 90000: @@ -191,21 +192,21 @@ class HstoreTestCase(unittest.TestCase): a.prepare(self.conn) q = a.getquoted() - m = re.match(r'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)', q) + m = re.match(b(r'hstore\(ARRAY\[([^\]]+)\], ARRAY\[([^\]]+)\]\)'), q) self.assert_(m, repr(q)) - kk = m.group(1).split(", ") - vv = m.group(2).split(", ") + kk = m.group(1).split(b(", ")) + vv = m.group(2).split(b(", ")) ii = zip(kk, vv) ii.sort() self.assertEqual(len(ii), len(o)) - self.assertEqual(ii[0], ("E'a'", "E'1'")) - self.assertEqual(ii[1], ("E'b'", "E''''")) - self.assertEqual(ii[2], ("E'c'", "NULL")) + self.assertEqual(ii[0], (b("E'a'"), b("E'1'"))) + self.assertEqual(ii[1], (b("E'b'"), b("E''''"))) + self.assertEqual(ii[2], (b("E'c'"), b("NULL"))) if 'd' in o: encc = u'\xe0'.encode(psycopg2.extensions.encodings[self.conn.encoding]) - self.assertEqual(ii[3], ("E'd'", "E'%s'" % encc)) + self.assertEqual(ii[3], (b("E'd'"), b("E'") + encc + b("'"))) def test_parse(self): from psycopg2.extras import HstoreAdapter @@ -321,7 +322,11 @@ class HstoreTestCase(unittest.TestCase): ok({''.join(ab): ''.join(ab)}) self.conn.set_client_encoding('latin1') - ab = map(chr, range(1, 256)) + if sys.version_info[0] < 3: + ab = map(chr, range(32, 127) + range(160, 255)) + else: + ab = bytes(range(32, 127) + range(160, 255)).decode('latin1') + ok({''.join(ab): ''.join(ab)}) ok(dict(zip(ab, ab))) From 2fa911783563f1b92a52f3c275d54eea0aefd067 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 29 Dec 2010 03:46:36 +0100 Subject: [PATCH 43/80] Inet adapter compatible with Python 3. --- lib/extras.py | 2 +- tests/types_extras.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/extras.py b/lib/extras.py index ba401a23..924fd51a 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -508,7 +508,7 @@ class Inet(object): obj = _A(self.addr) if hasattr(obj, 'prepare'): obj.prepare(self._conn) - return obj.getquoted()+"::inet" + return obj.getquoted() + b("::inet") def __conform__(self, foo): if foo is _ext.ISQLQuote: diff --git a/tests/types_extras.py b/tests/types_extras.py index 83b4f086..cb17fdf4 100644 --- a/tests/types_extras.py +++ b/tests/types_extras.py @@ -114,7 +114,7 @@ class TypesExtrasTests(unittest.TestCase): a = psycopg2.extensions.adapt(i) a.prepare(self.conn) self.assertEqual( - filter_scs(self.conn, "E'192.168.1.0/24'::inet"), + filter_scs(self.conn, b("E'192.168.1.0/24'::inet")), a.getquoted()) # adapts ok with unicode too @@ -122,7 +122,7 @@ class TypesExtrasTests(unittest.TestCase): a = psycopg2.extensions.adapt(i) a.prepare(self.conn) self.assertEqual( - filter_scs(self.conn, "E'192.168.1.0/24'::inet"), + filter_scs(self.conn, b("E'192.168.1.0/24'::inet")), a.getquoted()) def test_adapt_fail(self): From b78ff4a2737a407df13bfc5eae28537582ed2ee1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 29 Dec 2010 03:47:29 +0100 Subject: [PATCH 44/80] Several tests ported to Python 3. --- tests/test_cursor.py | 23 ++++++++++--------- tests/test_lobject.py | 53 ++++++++++++++++++++++--------------------- tests/test_quote.py | 15 ++++++++---- tests/types_basic.py | 7 +++--- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index d30a50ef..4faf94d7 100644 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -3,6 +3,7 @@ import unittest import psycopg2 import psycopg2.extensions +from psycopg2.extensions import b from testconfig import dsn class CursorTests(unittest.TestCase): @@ -32,28 +33,28 @@ class CursorTests(unittest.TestCase): # unicode query containing only ascii data cur.execute(u"SELECT 'foo';") self.assertEqual('foo', cur.fetchone()[0]) - self.assertEqual("SELECT 'foo';", cur.mogrify(u"SELECT 'foo';")) + self.assertEqual(b("SELECT 'foo';"), cur.mogrify(u"SELECT 'foo';")) conn.set_client_encoding('UTF8') snowman = u"\u2603" # unicode query with non-ascii data cur.execute(u"SELECT '%s';" % snowman) - self.assertEqual(snowman.encode('utf8'), cur.fetchone()[0]) - self.assertEqual("SELECT '%s';" % snowman.encode('utf8'), - cur.mogrify(u"SELECT '%s';" % snowman).replace("E'", "'")) + self.assertEqual(snowman.encode('utf8'), b(cur.fetchone()[0])) + self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), + cur.mogrify(u"SELECT '%s';" % snowman).replace(b("E'"), b("'"))) # unicode args cur.execute("SELECT %s;", (snowman,)) - self.assertEqual(snowman.encode("utf-8"), cur.fetchone()[0]) - self.assertEqual("SELECT '%s';" % snowman.encode('utf8'), - cur.mogrify("SELECT %s;", (snowman,)).replace("E'", "'")) + self.assertEqual(snowman.encode("utf-8"), b(cur.fetchone()[0])) + self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), + cur.mogrify("SELECT %s;", (snowman,)).replace(b("E'"), b("'"))) # unicode query and args cur.execute(u"SELECT %s;", (snowman,)) - self.assertEqual(snowman.encode("utf-8"), cur.fetchone()[0]) - self.assertEqual("SELECT '%s';" % snowman.encode('utf8'), - cur.mogrify(u"SELECT %s;", (snowman,)).replace("E'", "'")) + self.assertEqual(snowman.encode("utf-8"), b(cur.fetchone()[0])) + self.assertEqual(("SELECT '%s';" % snowman).encode('utf8'), + cur.mogrify(u"SELECT %s;", (snowman,)).replace(b("E'"), b("'"))) def test_mogrify_decimal_explodes(self): # issue #7: explodes on windows with python 2.5 and psycopg 2.2.2 @@ -64,7 +65,7 @@ class CursorTests(unittest.TestCase): conn = self.conn cur = conn.cursor() - self.assertEqual('SELECT 10.3;', + self.assertEqual(b('SELECT 10.3;'), cur.mogrify("SELECT %s;", (Decimal("10.3"),))) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 7c96a6ea..1c71fb4f 100644 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -5,6 +5,7 @@ import tempfile import psycopg2 import psycopg2.extensions +from psycopg2.extensions import b from testconfig import dsn, green from testutils import unittest, decorate_all_tests @@ -72,7 +73,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): lo = self.conn.lobject() lo2 = self.conn.lobject(lo.oid, "w") self.assertEqual(lo2.mode, "w") - lo2.write("some data") + lo2.write(b("some data")) def test_open_mode_n(self): # Openning an object in mode "n" gives us a closed lobject. @@ -103,11 +104,11 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") fp = open(filename, "wb") - fp.write("some data") + fp.write(b("some data")) fp.close() lo = self.conn.lobject(0, "r", 0, filename) - self.assertEqual(lo.read(), "some data") + self.assertEqual(lo.read(), b("some data")) def test_close(self): lo = self.conn.lobject() @@ -117,7 +118,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_write(self): lo = self.conn.lobject() - self.assertEqual(lo.write("some data"), len("some data")) + self.assertEqual(lo.write(b("some data")), len("some data")) def test_write_large(self): lo = self.conn.lobject() @@ -126,43 +127,43 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_read(self): lo = self.conn.lobject() - length = lo.write("some data") + length = lo.write(b("some data")) lo.close() lo = self.conn.lobject(lo.oid) - self.assertEqual(lo.read(4), "some") - self.assertEqual(lo.read(), " data") + self.assertEqual(lo.read(4), b("some")) + self.assertEqual(lo.read(), b(" data")) def test_read_large(self): lo = self.conn.lobject() - data = "data" * 1000000 - length = lo.write("some"+data) + data = b("data") * 1000000 + length = lo.write(b("some") + data) lo.close() lo = self.conn.lobject(lo.oid) - self.assertEqual(lo.read(4), "some") + self.assertEqual(lo.read(4), b("some")) self.assertEqual(lo.read(), data) def test_seek_tell(self): lo = self.conn.lobject() - length = lo.write("some data") + length = lo.write(b("some data")) self.assertEqual(lo.tell(), length) lo.close() lo = self.conn.lobject(lo.oid) self.assertEqual(lo.seek(5, 0), 5) self.assertEqual(lo.tell(), 5) - self.assertEqual(lo.read(), "data") + self.assertEqual(lo.read(), b("data")) # SEEK_CUR: relative current location lo.seek(5) self.assertEqual(lo.seek(2, 1), 7) self.assertEqual(lo.tell(), 7) - self.assertEqual(lo.read(), "ta") + self.assertEqual(lo.read(), b("ta")) # SEEK_END: relative to end of file self.assertEqual(lo.seek(-2, 2), length - 2) - self.assertEqual(lo.read(), "ta") + self.assertEqual(lo.read(), b("ta")) def test_unlink(self): lo = self.conn.lobject() @@ -175,13 +176,13 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_export(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), "some data") + self.assertEqual(open(filename, "rb").read(), b("some data")) def test_close_twice(self): lo = self.conn.lobject() @@ -191,7 +192,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_write_after_close(self): lo = self.conn.lobject() lo.close() - self.assertRaises(psycopg2.InterfaceError, lo.write, "some data") + self.assertRaises(psycopg2.InterfaceError, lo.write, b("some data")) def test_read_after_close(self): lo = self.conn.lobject() @@ -216,14 +217,14 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_export_after_close(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) lo.close() self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), "some data") + self.assertEqual(open(filename, "rb").read(), b("some data")) def test_close_after_commit(self): lo = self.conn.lobject() @@ -238,7 +239,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): self.lo_oid = lo.oid self.conn.commit() - self.assertRaises(psycopg2.ProgrammingError, lo.write, "some data") + self.assertRaises(psycopg2.ProgrammingError, lo.write, b("some data")) def test_read_after_commit(self): lo = self.conn.lobject() @@ -271,14 +272,14 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_export_after_commit(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) self.conn.commit() self.tmpdir = tempfile.mkdtemp() filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), "some data") + self.assertEqual(open(filename, "rb").read(), b("some data")) decorate_all_tests(LargeObjectTests, skip_if_no_lo) decorate_all_tests(LargeObjectTests, skip_if_green) @@ -300,7 +301,7 @@ def skip_if_no_truncate(f): class LargeObjectTruncateTests(LargeObjectMixin, unittest.TestCase): def test_truncate(self): lo = self.conn.lobject() - lo.write("some data") + lo.write(b("some data")) lo.close() lo = self.conn.lobject(lo.oid, "w") @@ -309,17 +310,17 @@ class LargeObjectTruncateTests(LargeObjectMixin, unittest.TestCase): # seek position unchanged self.assertEqual(lo.tell(), 0) # data truncated - self.assertEqual(lo.read(), "some") + self.assertEqual(lo.read(), b("some")) lo.truncate(6) lo.seek(0) # large object extended with zeroes - self.assertEqual(lo.read(), "some\x00\x00") + self.assertEqual(lo.read(), b("some\x00\x00")) lo.truncate() lo.seek(0) # large object empty - self.assertEqual(lo.read(), "") + self.assertEqual(lo.read(), b("")) def test_truncate_after_close(self): lo = self.conn.lobject() diff --git a/tests/test_quote.py b/tests/test_quote.py index 1e86803c..8fb2c12e 100755 --- a/tests/test_quote.py +++ b/tests/test_quote.py @@ -5,6 +5,7 @@ from testconfig import dsn import psycopg2 import psycopg2.extensions +from psycopg2.extensions import b class QuotingTestCase(unittest.TestCase): r"""Checks the correct quoting of strings and binary objects. @@ -44,14 +45,20 @@ class QuotingTestCase(unittest.TestCase): self.assert_(not self.conn.notices) def test_binary(self): - data = """some data with \000\013 binary + data = b("""some data with \000\013 binary stuff into, 'quotes' and \\ a backslash too. - """ - data += "".join(map(chr, range(256))) + """) + if sys.version_info[0] < 3: + data += "".join(map(chr, range(256))) + else: + data += bytes(range(256)) curs = self.conn.cursor() curs.execute("SELECT %s::bytea;", (psycopg2.Binary(data),)) - res = str(curs.fetchone()[0]) + if sys.version_info[0] < 3: + res = str(curs.fetchone()[0]) + else: + res = curs.fetchone()[0].tobytes() self.assertEqual(res, data) self.assert_(not self.conn.notices) diff --git a/tests/types_basic.py b/tests/types_basic.py index 4591290b..3be57c12 100755 --- a/tests/types_basic.py +++ b/tests/types_basic.py @@ -29,9 +29,10 @@ except: import sys import testutils from testutils import unittest +from testconfig import dsn import psycopg2 -from testconfig import dsn +from psycopg2.extensions import b class TypesBasicTests(unittest.TestCase): @@ -231,7 +232,7 @@ class AdaptSubclassTest(unittest.TestCase): register_adapter(A, lambda a: AsIs("a")) register_adapter(B, lambda b: AsIs("b")) - self.assertEqual('b', adapt(C()).getquoted()) + self.assertEqual(b('b'), adapt(C()).getquoted()) @testutils.skip_on_python3 def test_no_mro_no_joy(self): @@ -251,7 +252,7 @@ class AdaptSubclassTest(unittest.TestCase): class B(A): pass register_adapter(A, lambda a: AsIs("a")) - self.assertEqual("a", adapt(B()).getquoted()) + self.assertEqual(b("a"), adapt(B()).getquoted()) def test_suite(): From 0a4eeb4e13470c5b296214530983c9aad6d033ac Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 29 Dec 2010 12:10:21 +0100 Subject: [PATCH 45/80] Fixed notification tests to run on Py3. Call 2to3 on the dynamically generated scripts. --- tests/test_notify.py | 3 ++- tests/testutils.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/test_notify.py b/tests/test_notify.py index a15aad68..683d21d9 100755 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -4,6 +4,7 @@ from testutils import unittest import psycopg2 from psycopg2 import extensions from testconfig import dsn +from testutils import script_to_py3 import sys import time @@ -52,7 +53,7 @@ conn.close() """ % { 'dsn': dsn, 'sec': sec, 'name': name, 'payload': payload}) - return Popen([sys.executable, '-c', script], stdout=PIPE) + return Popen([sys.executable, '-c', script_to_py3(script)], stdout=PIPE) def test_notifies_received_on_poll(self): self.autocommit(self.conn) diff --git a/tests/testutils.py b/tests/testutils.py index e3ea6877..49d26b21 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -4,6 +4,7 @@ # Use unittest2 if available. Otherwise mock a skip facility with warnings. +import os import sys try: @@ -92,3 +93,23 @@ def skip_on_python3(f): return skip_on_python3_ +def script_to_py3(script): + """Convert a script to Python3 syntax if required.""" + if sys.version_info[0] < 3: + return script + + import tempfile + f = tempfile.NamedTemporaryFile(suffix=".py") + f.write(script.encode()) + f.flush() + + # 2to3 is way too chatty + import logging + logging.basicConfig(filename=os.devnull) + + from lib2to3.main import main + if main("lib2to3.fixes", ['--no-diffs', '-w', '-n', f.name]): + raise Exception('py3 conversion failed') + + return open(f.name).read() + From bc28cc8b0025c86e66e4fec3c1b7bfdccfb729b2 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 29 Dec 2010 23:53:03 +0100 Subject: [PATCH 46/80] Use unicode keys as strings in Py3. This fixes pyformat style argument passing. Unicode and bytes don't compare equal (even if they hash the same). --- psycopg/bytes_format.c | 3 +-- psycopg/cursor_type.c | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c index 1c49a011..ab57bc3f 100644 --- a/psycopg/bytes_format.c +++ b/psycopg/bytes_format.c @@ -207,8 +207,7 @@ PyBytes_Format(PyObject *format, PyObject *args) "incomplete format key"); goto error; } - key = PyBytes_FromStringAndSize(keystart, - keylen); + key = PyUnicode_FromStringAndSize(keystart, keylen); if (key == NULL) goto error; if (args_owned) { diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 5290c238..2491081b 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -116,7 +116,7 @@ _mogrify(PyObject *var, PyObject *fmt, connectionObject *conn, PyObject **new) for (d = c + 2; *d && *d != ')'; d++); if (*d == ')') { - key = Bytes_FromStringAndSize(c+2, (Py_ssize_t) (d-c-2)); + key = Text_FromUTF8AndSize(c+2, (Py_ssize_t) (d-c-2)); value = PyObject_GetItem(var, key); /* key has refcnt 1, value the original value + 1 */ From 73917c15e12838f3c0a0b2f278b35c988162a56d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 30 Dec 2010 01:14:42 +0100 Subject: [PATCH 47/80] Fixed COPY FROM to deal with decoded files. --- psycopg/pqpath.c | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 9ba94152..833284ed 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -1065,19 +1065,50 @@ _pq_copy_in_v3(cursorObject *curs) } while (1) { - o = PyObject_CallFunctionObjArgs(func, size, NULL); - if (!(o && Bytes_Check(o) && (length = Bytes_GET_SIZE(o)) != -1)) { + if (!(o = PyObject_CallFunctionObjArgs(func, size, NULL))) { + Dprintf("_pq_copy_in_v3: read() failed"); error = 1; + break; + } + + /* a file may return unicode in Py3: encode in client encoding. */ +#if PY_MAJOR_VERSION > 2 + if (PyUnicode_Check(o)) { + PyObject *tmp; + if (!(tmp = PyUnicode_AsEncodedString(o, curs->conn->codec, NULL))) { + Dprintf("_pq_copy_in_v3: encoding() failed"); + error = 1; + break; + } + Py_DECREF(o); + o = tmp; + } +#endif + + if (!Bytes_Check(o)) { + Dprintf("_pq_copy_in_v3: got %s instead of bytes", + Py_TYPE(o)->tp_name); + error = 1; + break; + } + + if (0 == (length = Bytes_GET_SIZE(o))) { + break; + } + if (length > INT_MAX) { + Dprintf("_pq_copy_in_v3: bad length: " FORMAT_CODE_PY_SSIZE_T, + length); + error = 1; + break; } - if (length == 0 || length > INT_MAX || error == 1) break; Py_BEGIN_ALLOW_THREADS; res = PQputCopyData(curs->conn->pgconn, Bytes_AS_STRING(o), /* Py_ssize_t->int cast was validated above */ (int) length); - Dprintf("_pq_copy_in_v3: sent %d bytes of data; res = %d", - (int) length, res); - + Dprintf("_pq_copy_in_v3: sent " FORMAT_CODE_PY_SSIZE_T " bytes of data; res = %d", + length, res); + if (res == 0) { /* FIXME: in theory this should not happen but adding a check here would be a nice idea */ @@ -1105,6 +1136,7 @@ _pq_copy_in_v3(cursorObject *curs) else if (error == 2) res = PQputCopyEnd(curs->conn->pgconn, "error in PQputCopyData() call"); else + /* XXX would be nice to propagate the exeption */ res = PQputCopyEnd(curs->conn->pgconn, "error in .read() call"); IFCLEARPGRES(curs->pgres); From 89fb60de4b4dac728863b8a5c2c3004c22aa60dc Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 30 Dec 2010 01:15:47 +0100 Subject: [PATCH 48/80] Column names in copy methods can be unicode. --- psycopg/cursor_type.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 2491081b..4c3908b8 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -1105,11 +1105,8 @@ static int _psyco_curs_copy_columns(PyObject *columns, char *columnlist) columnlist[0] = '('; while ((col = PyIter_Next(coliter)) != NULL) { - if (!Bytes_Check(col)) { - Py_DECREF(col); + if (!(col = psycopg_ensure_bytes(col))) { Py_DECREF(coliter); - PyErr_SetString(PyExc_ValueError, - "elements in column list must be strings"); return -1; } Bytes_AsStringAndSize(col, &colname, &collen); From 96a950d3eb188803164d748aaf61d81b9bb985c5 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 30 Dec 2010 01:30:09 +0100 Subject: [PATCH 49/80] Fixed 2-phase commit support in Python 3. --- psycopg/connection_int.c | 2 +- psycopg/xid_type.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/psycopg/connection_int.c b/psycopg/connection_int.c index 6d281b36..83c77ad0 100644 --- a/psycopg/connection_int.c +++ b/psycopg/connection_int.c @@ -1050,7 +1050,7 @@ conn_tpc_command(connectionObject *self, const char *cmd, XidObject *xid) Dprintf("conn_tpc_command: %s", cmd); /* convert the xid into PostgreSQL transaction id while keeping the GIL */ - if (!(tid = xid_get_tid(xid))) { goto exit; } + if (!(tid = psycopg_ensure_bytes(xid_get_tid(xid)))) { goto exit; } if (!(ctid = Bytes_AsString(tid))) { goto exit; } Py_BEGIN_ALLOW_THREADS; diff --git a/psycopg/xid_type.c b/psycopg/xid_type.c index 51cd4490..29577492 100644 --- a/psycopg/xid_type.c +++ b/psycopg/xid_type.c @@ -399,7 +399,11 @@ _xid_base64_enc_dec(const char *funcname, PyObject *s) if (!(base64 = PyImport_ImportModule("base64"))) { goto exit; } if (!(func = PyObject_GetAttrString(base64, funcname))) { goto exit; } - rv = PyObject_CallFunctionObjArgs(func, s, NULL); + + Py_INCREF(s); + if (!(s = psycopg_ensure_bytes(s))) { goto exit; } + rv = psycopg_ensure_text(PyObject_CallFunctionObjArgs(func, s, NULL)); + Py_DECREF(s); exit: Py_XDECREF(func); From 6882ac31d49c35730a4b70643178395c236fc509 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 31 Dec 2010 01:53:42 +0100 Subject: [PATCH 50/80] Dropped warnings in PyBytes_Format function. --- psycopg/bytes_format.c | 62 ------------------------------------------ 1 file changed, 62 deletions(-) diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c index ab57bc3f..2ca20f25 100644 --- a/psycopg/bytes_format.c +++ b/psycopg/bytes_format.c @@ -78,10 +78,6 @@ #define PSYCOPG_MODULE #include "psycopg/psycopg.h" -#ifndef Py_USING_UNICODE -#define Py_USING_UNICODE 1 -#endif - /* Helpers for formatstring */ Py_LOCAL_INLINE(PyObject *) @@ -118,9 +114,6 @@ PyBytes_Format(PyObject *format, PyObject *args) Py_ssize_t reslen, rescnt, fmtcnt; int args_owned = 0; PyObject *result, *orig_args; -#ifdef Py_USING_UNICODE - PyObject *v, *w; -#endif PyObject *dict = NULL; if (format == NULL || !PyBytes_Check(format) || args == NULL) { PyErr_BadInternalCall(); @@ -165,19 +158,11 @@ PyBytes_Format(PyObject *format, PyObject *args) int prec = -1; int c = '\0'; int fill; - int isnumok; PyObject *v = NULL; PyObject *temp = NULL; char *pbuf; int sign; Py_ssize_t len; - char formatbuf[FORMATBUFLEN]; - /* For format{int,char}() */ -#ifdef Py_USING_UNICODE - char *fmt_start = fmt; - Py_ssize_t argidx_start = argidx; -#endif - fmt++; if (*fmt == '(') { char *keystart; @@ -455,53 +440,6 @@ PyBytes_Format(PyObject *format, PyObject *args) return NULL; return result; -#ifdef Py_USING_UNICODE - unicode: - if (args_owned) { - Py_DECREF(args); - args_owned = 0; - } - /* Fiddle args right (remove the first argidx arguments) */ - if (PyTuple_Check(orig_args) && argidx > 0) { - PyObject *v; - Py_ssize_t n = PyTuple_GET_SIZE(orig_args) - argidx; - v = PyTuple_New(n); - if (v == NULL) - goto error; - while (--n >= 0) { - PyObject *w = PyTuple_GET_ITEM(orig_args, n + argidx); - Py_INCREF(w); - PyTuple_SET_ITEM(v, n, w); - } - args = v; - } else { - Py_INCREF(orig_args); - args = orig_args; - } - args_owned = 1; - /* Take what we have of the result and let the Unicode formatting - function format the rest of the input. */ - rescnt = res - PyBytes_AS_STRING(result); - if (_PyBytes_Resize(&result, rescnt)) - goto error; - fmtcnt = PyBytes_GET_SIZE(format) - \ - (fmt - PyBytes_AS_STRING(format)); - format = PyUnicode_Decode(fmt, fmtcnt, NULL, NULL); - if (format == NULL) - goto error; - v = PyUnicode_Format(format, args); - Py_DECREF(format); - if (v == NULL) - goto error; - /* Paste what we have (result) to what the Unicode formatting - function returned (v) and return the result (or error) */ - w = PyUnicode_Concat(result, v); - Py_DECREF(result); - Py_DECREF(v); - Py_DECREF(args); - return w; -#endif /* Py_USING_UNICODE */ - error: Py_DECREF(result); if (args_owned) { From 2930ed3d594ec3652d5bd13cb7d8169541db807c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 31 Dec 2010 02:32:20 +0100 Subject: [PATCH 51/80] Dropped support for all format specifiers except s in PyBytes_Format. --- psycopg/bytes_format.c | 163 +---------------------------------------- 1 file changed, 3 insertions(+), 160 deletions(-) diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c index 2ca20f25..3702a684 100644 --- a/psycopg/bytes_format.c +++ b/psycopg/bytes_format.c @@ -96,15 +96,7 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) return NULL; } -/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) - - FORMATBUFLEN is the length of the buffer in which the ints & - chars are formatted. XXX This is a magic number. Each formatting - routine does bounds checking to ensure no overflow, but a better - solution may be to malloc a buffer of appropriate size for each - format. For now, the current solution is sufficient. -*/ -#define FORMATBUFLEN (size_t)120 +/* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */ PyObject * PyBytes_Format(PyObject *format, PyObject *args) @@ -153,15 +145,11 @@ PyBytes_Format(PyObject *format, PyObject *args) } else { /* Got a format specifier */ - int flags = 0; Py_ssize_t width = -1; - int prec = -1; int c = '\0'; - int fill; PyObject *v = NULL; PyObject *temp = NULL; char *pbuf; - int sign; Py_ssize_t len; fmt++; if (*fmt == '(') { @@ -209,89 +197,9 @@ PyBytes_Format(PyObject *format, PyObject *args) argidx = -2; } while (--fmtcnt >= 0) { - switch (c = *fmt++) { - case '-': flags |= F_LJUST; continue; - case '+': flags |= F_SIGN; continue; - case ' ': flags |= F_BLANK; continue; - case '#': flags |= F_ALT; continue; - case '0': flags |= F_ZERO; continue; - } + c = *fmt++; break; } - if (c == '*') { - v = getnextarg(args, arglen, &argidx); - if (v == NULL) - goto error; - if (!PyLong_Check(v)) { - PyErr_SetString(PyExc_TypeError, - "* wants int"); - goto error; - } - width = PyLong_AsLong(v); - if (width < 0) { - flags |= F_LJUST; - width = -width; - } - if (--fmtcnt >= 0) - c = *fmt++; - } - else if (c >= 0 && isdigit(c)) { - width = c - '0'; - while (--fmtcnt >= 0) { - c = Py_CHARMASK(*fmt++); - if (!isdigit(c)) - break; - if ((width*10) / 10 != width) { - PyErr_SetString( - PyExc_ValueError, - "width too big"); - goto error; - } - width = width*10 + (c - '0'); - } - } - if (c == '.') { - prec = 0; - if (--fmtcnt >= 0) - c = *fmt++; - if (c == '*') { - v = getnextarg(args, arglen, &argidx); - if (v == NULL) - goto error; - if (!PyLong_Check(v)) { - PyErr_SetString( - PyExc_TypeError, - "* wants int"); - goto error; - } - prec = PyLong_AsLong(v); - if (prec < 0) - prec = 0; - if (--fmtcnt >= 0) - c = *fmt++; - } - else if (c >= 0 && isdigit(c)) { - prec = c - '0'; - while (--fmtcnt >= 0) { - c = Py_CHARMASK(*fmt++); - if (!isdigit(c)) - break; - if ((prec*10) / 10 != prec) { - PyErr_SetString( - PyExc_ValueError, - "prec too big"); - goto error; - } - prec = prec*10 + (c - '0'); - } - } - } /* prec */ - if (fmtcnt >= 0) { - if (c == 'h' || c == 'l' || c == 'L') { - if (--fmtcnt >= 0) - c = *fmt++; - } - } if (fmtcnt < 0) { PyErr_SetString(PyExc_ValueError, "incomplete format"); @@ -302,8 +210,6 @@ PyBytes_Format(PyObject *format, PyObject *args) if (v == NULL) goto error; } - sign = 0; - fill = ' '; switch (c) { case '%': pbuf = "%"; @@ -319,22 +225,8 @@ PyBytes_Format(PyObject *format, PyObject *args) } temp = v; Py_INCREF(v); - /* Fall through */ - case 'r': - if (c == 'r') - temp = PyObject_Repr(v); - if (temp == NULL) - goto error; - if (!PyBytes_Check(temp)) { - PyErr_SetString(PyExc_TypeError, - "%s argument has non-string str()"); - Py_DECREF(temp); - goto error; - } pbuf = PyBytes_AS_STRING(temp); len = PyBytes_GET_SIZE(temp); - if (prec >= 0 && len > prec) - len = prec; break; default: PyErr_Format(PyExc_ValueError, @@ -345,21 +237,9 @@ PyBytes_Format(PyObject *format, PyObject *args) PyBytes_AsString(format))); goto error; } - if (sign) { - if (*pbuf == '-' || *pbuf == '+') { - sign = *pbuf++; - len--; - } - else if (flags & F_SIGN) - sign = '+'; - else if (flags & F_BLANK) - sign = ' '; - else - sign = 0; - } if (width < len) width = len; - if (rescnt - (sign != 0) < width) { + if (rescnt < width) { reslen -= rescnt; rescnt = width + fmtcnt + 100; reslen += rescnt; @@ -375,43 +255,6 @@ PyBytes_Format(PyObject *format, PyObject *args) res = PyBytes_AS_STRING(result) + reslen - rescnt; } - if (sign) { - if (fill != ' ') - *res++ = sign; - rescnt--; - if (width > len) - width--; - } - if ((flags & F_ALT) && (c == 'x' || c == 'X')) { - assert(pbuf[0] == '0'); - assert(pbuf[1] == c); - if (fill != ' ') { - *res++ = *pbuf++; - *res++ = *pbuf++; - } - rescnt -= 2; - width -= 2; - if (width < 0) - width = 0; - len -= 2; - } - if (width > len && !(flags & F_LJUST)) { - do { - --rescnt; - *res++ = fill; - } while (--width > len); - } - if (fill == ' ') { - if (sign) - *res++ = sign; - if ((flags & F_ALT) && - (c == 'x' || c == 'X')) { - assert(pbuf[0] == '0'); - assert(pbuf[1] == c); - *res++ = *pbuf++; - *res++ = *pbuf++; - } - } Py_MEMCPY(res, pbuf, len); res += len; rescnt -= len; From ac258169622ca758757e51338443a4cbed1e605d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 31 Dec 2010 02:49:46 +0100 Subject: [PATCH 52/80] Use the same Bytes_Format function for both Python 2 and 3. This makes the behaviour between the two versions similar. It also have the effect of a more specific error message in case an user specifies a placeholder different from 's'. --- psycopg/bytes_format.c | 38 ++++++++++++++++++++------------------ psycopg/python.h | 13 ++++++------- setup.py | 5 +---- 3 files changed, 27 insertions(+), 29 deletions(-) diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c index 3702a684..3c64f4ae 100644 --- a/psycopg/bytes_format.c +++ b/psycopg/bytes_format.c @@ -24,7 +24,9 @@ */ /* This implementation is based on the PyString_Format function available in - * Python 2.7.1. Original license follows. + * Python 2.7.1. The function is altered to be used with both Python 2 strings + * and Python 3 bytes and is stripped of the support of formats different than + * 's'. Original license follows. * * PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 * -------------------------------------------- @@ -99,7 +101,7 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx) /* fmt%(v1,v2,...) is roughly equivalent to sprintf(fmt, v1, v2, ...) */ PyObject * -PyBytes_Format(PyObject *format, PyObject *args) +Bytes_Format(PyObject *format, PyObject *args) { char *fmt, *res; Py_ssize_t arglen, argidx; @@ -107,18 +109,18 @@ PyBytes_Format(PyObject *format, PyObject *args) int args_owned = 0; PyObject *result, *orig_args; PyObject *dict = NULL; - if (format == NULL || !PyBytes_Check(format) || args == NULL) { + if (format == NULL || !Bytes_Check(format) || args == NULL) { PyErr_BadInternalCall(); return NULL; } orig_args = args; - fmt = PyBytes_AS_STRING(format); - fmtcnt = PyBytes_GET_SIZE(format); + fmt = Bytes_AS_STRING(format); + fmtcnt = Bytes_GET_SIZE(format); reslen = rescnt = fmtcnt + 100; - result = PyBytes_FromStringAndSize((char *)NULL, reslen); + result = Bytes_FromStringAndSize((char *)NULL, reslen); if (result == NULL) return NULL; - res = PyBytes_AsString(result); + res = Bytes_AsString(result); if (PyTuple_Check(args)) { arglen = PyTuple_GET_SIZE(args); argidx = 0; @@ -128,16 +130,16 @@ PyBytes_Format(PyObject *format, PyObject *args) argidx = -2; } if (Py_TYPE(args)->tp_as_mapping && !PyTuple_Check(args) && - !PyObject_TypeCheck(args, &PyBytes_Type)) + !PyObject_TypeCheck(args, &Bytes_Type)) dict = args; while (--fmtcnt >= 0) { if (*fmt != '%') { if (--rescnt < 0) { rescnt = fmtcnt + 100; reslen += rescnt; - if (_PyBytes_Resize(&result, reslen)) + if (_Bytes_Resize(&result, reslen)) return NULL; - res = PyBytes_AS_STRING(result) + res = Bytes_AS_STRING(result) + reslen - rescnt; --rescnt; } @@ -180,7 +182,7 @@ PyBytes_Format(PyObject *format, PyObject *args) "incomplete format key"); goto error; } - key = PyUnicode_FromStringAndSize(keystart, keylen); + key = Text_FromUTF8AndSize(keystart, keylen); if (key == NULL) goto error; if (args_owned) { @@ -217,7 +219,7 @@ PyBytes_Format(PyObject *format, PyObject *args) break; case 's': /* only bytes! */ - if (!PyBytes_CheckExact(v)) { + if (!Bytes_CheckExact(v)) { PyErr_Format(PyExc_ValueError, "only bytes values expected, got %s", Py_TYPE(v)->tp_name); @@ -225,8 +227,8 @@ PyBytes_Format(PyObject *format, PyObject *args) } temp = v; Py_INCREF(v); - pbuf = PyBytes_AS_STRING(temp); - len = PyBytes_GET_SIZE(temp); + pbuf = Bytes_AS_STRING(temp); + len = Bytes_GET_SIZE(temp); break; default: PyErr_Format(PyExc_ValueError, @@ -234,7 +236,7 @@ PyBytes_Format(PyObject *format, PyObject *args) "at index %zd", c, c, (Py_ssize_t)(fmt - 1 - - PyBytes_AsString(format))); + Bytes_AsString(format))); goto error; } if (width < len) @@ -248,11 +250,11 @@ PyBytes_Format(PyObject *format, PyObject *args) Py_XDECREF(temp); return PyErr_NoMemory(); } - if (_PyBytes_Resize(&result, reslen)) { + if (_Bytes_Resize(&result, reslen)) { Py_XDECREF(temp); return NULL; } - res = PyBytes_AS_STRING(result) + res = Bytes_AS_STRING(result) + reslen - rescnt; } Py_MEMCPY(res, pbuf, len); @@ -279,7 +281,7 @@ PyBytes_Format(PyObject *format, PyObject *args) if (args_owned) { Py_DECREF(args); } - if (_PyBytes_Resize(&result, reslen - rescnt)) + if (_Bytes_Resize(&result, reslen - rescnt)) return NULL; return result; diff --git a/psycopg/python.h b/psycopg/python.h index ee2dee76..1e46cc51 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -103,8 +103,7 @@ #endif /* PY_MAJOR_VERSION > 2 */ #if PY_MAJOR_VERSION < 3 -/* XXX BytesType -> Bytes_Type */ -#define BytesType PyString_Type +#define Bytes_Type PyString_Type #define Bytes_Check PyString_Check #define Bytes_CheckExact PyString_CheckExact #define Bytes_AS_STRING PyString_AS_STRING @@ -115,11 +114,11 @@ #define Bytes_FromString PyString_FromString #define Bytes_FromStringAndSize PyString_FromStringAndSize #define Bytes_FromFormat PyString_FromFormat -#define Bytes_Format PyString_Format +#define _Bytes_Resize _PyString_Resize #else -#define BytesType PyBytes_Type +#define Bytes_Type PyBytes_Type #define Bytes_Check PyBytes_Check #define Bytes_CheckExact PyBytes_CheckExact #define Bytes_AS_STRING PyBytes_AS_STRING @@ -130,12 +129,12 @@ #define Bytes_FromString PyBytes_FromString #define Bytes_FromStringAndSize PyBytes_FromStringAndSize #define Bytes_FromFormat PyBytes_FromFormat -#define Bytes_Format PyBytes_Format - -HIDDEN PyObject *PyBytes_Format(PyObject *format, PyObject *args); +#define _Bytes_Resize _PyBytes_Resize #endif +HIDDEN PyObject *Bytes_Format(PyObject *format, PyObject *args); + /* Mangle the module name into the name of the module init function */ #if PY_MAJOR_VERSION > 2 #define INIT_MODULE(m) PyInit_ ## m diff --git a/setup.py b/setup.py index b05bf9a7..701ad5bf 100644 --- a/setup.py +++ b/setup.py @@ -362,7 +362,7 @@ ext = [] ; data_files = [] sources = [ 'psycopgmodule.c', - 'green.c', 'pqpath.c', 'utils.c', + 'green.c', 'pqpath.c', 'utils.c', 'bytes_format.c', 'connection_int.c', 'connection_type.c', 'cursor_int.c', 'cursor_type.c', @@ -376,9 +376,6 @@ sources = [ 'typecast.c', ] -if sys.version_info[0] >= 3: - sources.append('bytes_format.c') - depends = [ # headers 'config.h', 'pgtypes.h', 'psycopg.h', 'python.h', From 88bb8eda3e772a1ef6da186c950b39af59e8b0c9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 1 Jan 2011 16:53:11 +0100 Subject: [PATCH 53/80] None/IN adaptation ported to Python 3. --- lib/extensions.py | 11 ++++++----- tests/types_extras.py | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/extensions.py b/lib/extensions.py index e9b6813c..57aa9c68 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -132,10 +132,11 @@ class SQL_IN(object): for obj in pobjs: if hasattr(obj, 'prepare'): obj.prepare(self._conn) - qobjs = [str(o.getquoted()) for o in pobjs] - return '(' + ', '.join(qobjs) + ')' + qobjs = [o.getquoted() for o in pobjs] + return b('(') + b(', ').join(qobjs) + b(')') - __str__ = getquoted + def __str__(self): + return str(self.getquoted()) class NoneAdapter(object): @@ -147,8 +148,8 @@ class NoneAdapter(object): def __init__(self, obj): pass - def getquoted(self): - return "NULL" + def getquoted(self, _null=b("NULL")): + return _null # Add the "cleaned" version of the encodings to the key. diff --git a/tests/types_extras.py b/tests/types_extras.py index 8b2d5fd1..fac5b33f 100644 --- a/tests/types_extras.py +++ b/tests/types_extras.py @@ -356,7 +356,7 @@ class HstoreTestCase(unittest.TestCase): class AdaptTypeTestCase(unittest.TestCase): def setUp(self): - self.conn = psycopg2.connect(tests.dsn) + self.conn = psycopg2.connect(dsn) def tearDown(self): self.conn.close() @@ -364,7 +364,7 @@ class AdaptTypeTestCase(unittest.TestCase): def test_none_in_record(self): curs = self.conn.cursor() s = curs.mogrify("SELECT %s;", [(42, None)]) - self.assertEqual("SELECT (42, NULL);", s) + self.assertEqual(b("SELECT (42, NULL);"), s) curs.execute("SELECT %s;", [(42, None)]) d = curs.fetchone()[0] self.assertEqual("(42,)", d) @@ -385,7 +385,7 @@ class AdaptTypeTestCase(unittest.TestCase): self.assertEqual(ext.adapt(None).getquoted(), "NOPE!") s = curs.mogrify("SELECT %s;", (None,)) - self.assertEqual("SELECT NULL;", s) + self.assertEqual(b("SELECT NULL;"), s) finally: ext.register_adapter(type(None), orig_adapter) From 7e9be4c1338d0479346fee89ec76a28f04ba3b7a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 2 Jan 2011 00:48:19 +0100 Subject: [PATCH 54/80] Added Python 3 trove classifier. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 8296fec7..a17de8c6 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ Intended Audience :: Developers License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) License :: OSI Approved :: Zope Public License Programming Language :: Python +Programming Language :: Python :: 3 Programming Language :: C Programming Language :: SQL Topic :: Database From 9eae66e8cf427484b9770f659a993389291b3b07 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 3 Jan 2011 16:56:26 +0100 Subject: [PATCH 55/80] Added Py3 compatibility macro for Py_TPFLAGS_HAVE_WEAKREFS --- psycopg/python.h | 1 + 1 file changed, 1 insertion(+) diff --git a/psycopg/python.h b/psycopg/python.h index 1e46cc51..25dbd31b 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -97,6 +97,7 @@ #define PyString_FromFormat PyUnicode_FromFormat #define Py_TPFLAGS_HAVE_ITER 0L #define Py_TPFLAGS_HAVE_RICHCOMPARE 0L +#define Py_TPFLAGS_HAVE_WEAKREFS 0L #ifndef PyNumber_Int #define PyNumber_Int PyNumber_Long #endif From b276e3b05dee01396d7703cced6e3b75e05caec3 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 3 Jan 2011 19:14:40 +0100 Subject: [PATCH 56/80] Fixed compiling on Python versions before 2.6 Added a few macros not defined in Py 2.4. Don't know about 2.5. --- psycopg/bytes_format.c | 4 ++++ psycopg/python.h | 5 +++++ psycopg/typecast.c | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c index 3c64f4ae..6d12d6c7 100644 --- a/psycopg/bytes_format.c +++ b/psycopg/bytes_format.c @@ -80,6 +80,10 @@ #define PSYCOPG_MODULE #include "psycopg/psycopg.h" +#ifndef Py_LOCAL_INLINE +#define Py_LOCAL_INLINE(type) static type +#endif + /* Helpers for formatstring */ Py_LOCAL_INLINE(PyObject *) diff --git a/psycopg/python.h b/psycopg/python.h index 25dbd31b..c3f9a0d1 100644 --- a/psycopg/python.h +++ b/psycopg/python.h @@ -62,6 +62,11 @@ #define PyVarObject_HEAD_INIT(x,n) PyObject_HEAD_INIT(x) n, #endif +/* Missing at least in Python 2.4 */ +#ifndef Py_MEMCPY +#define Py_MEMCPY memcpy +#endif + /* FORMAT_CODE_PY_SSIZE_T is for Py_ssize_t: */ #define FORMAT_CODE_PY_SSIZE_T "%" PY_FORMAT_SIZE_T "d" diff --git a/psycopg/typecast.c b/psycopg/typecast.c index c321609f..484ce45e 100644 --- a/psycopg/typecast.c +++ b/psycopg/typecast.c @@ -438,7 +438,7 @@ typecast_repr(PyObject *self) } rv = PyString_FromFormat("<%s '%s' at %p>", - Py_TYPE(self)->tp_name, PyBytes_AS_STRING(name), self); + Py_TYPE(self)->tp_name, Bytes_AS_STRING(name), self); Py_DECREF(name); return rv; From 39dd577c90a1f79c06d0613e73c2addbd53a4468 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 3 Jan 2011 19:17:39 +0100 Subject: [PATCH 57/80] Use the proper printf placeholders to avoid warnings on 64 bit builds --- psycopg/bytes_format.c | 2 +- psycopg/lobject_int.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/psycopg/bytes_format.c b/psycopg/bytes_format.c index 6d12d6c7..206e870d 100644 --- a/psycopg/bytes_format.c +++ b/psycopg/bytes_format.c @@ -237,7 +237,7 @@ Bytes_Format(PyObject *format, PyObject *args) default: PyErr_Format(PyExc_ValueError, "unsupported format character '%c' (0x%x) " - "at index %zd", + "at index " FORMAT_CODE_PY_SSIZE_T, c, c, (Py_ssize_t)(fmt - 1 - Bytes_AsString(format))); diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index d6ebd44e..b78b4a9b 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -203,7 +203,7 @@ lobject_write(lobjectObject *self, const char *buf, size_t len) PGresult *pgres = NULL; char *error = NULL; - Dprintf("lobject_writing: fd = %d, len = " FORMAT_CODE_PY_SSIZE_T, + Dprintf("lobject_writing: fd = %d, len = " FORMAT_CODE_SIZE_T, self->fd, len); Py_BEGIN_ALLOW_THREADS; @@ -338,7 +338,7 @@ lobject_truncate(lobjectObject *self, size_t len) PGresult *pgres = NULL; char *error = NULL; - Dprintf("lobject_truncate: fd = %d, len = " FORMAT_CODE_PY_SSIZE_T, + Dprintf("lobject_truncate: fd = %d, len = " FORMAT_CODE_SIZE_T, self->fd, len); Py_BEGIN_ALLOW_THREADS; From a01700d478765e8dc6044336f21fe84808569a0d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 3 Jan 2011 17:29:58 +0100 Subject: [PATCH 58/80] Fixed test suite execution as a script Don't know why I changed the defaultTest argument into a function when I converted the test suite into a package: that argument should be really a string. --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index b0b7cd0f..2bdfdd81 100755 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -55,4 +55,4 @@ def test_suite(): return suite if __name__ == '__main__': - unittest.main(defaultTest=test_suite) + unittest.main(defaultTest='test_suite') From b8c8cddc2d885921c8ec067615707a187f7f9032 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 4 Jan 2011 02:10:28 +0100 Subject: [PATCH 59/80] Fixed argument parsing in lobject.read Using an int instead of a Py_ssize_t randomly crashed Python 3.1 64 bit. --- psycopg/lobject_type.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 0feffbcb..9a5fb0b8 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -71,7 +71,8 @@ psyco_lobj_close(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_write(lobjectObject *self, PyObject *args) { - int len, res=0; + int res = 0; + Py_ssize_t len; const char *buffer; if (!PyArg_ParseTuple(args, "s#", &buffer, &len)) return NULL; @@ -80,7 +81,7 @@ psyco_lobj_write(lobjectObject *self, PyObject *args) EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); - if ((res = lobject_write(self, buffer, len)) < 0) return NULL; + if ((res = lobject_write(self, buffer, (size_t)len)) < 0) return NULL; return PyInt_FromLong((long)res); } From 9deb16484dee86d94b2ab75868410b360e9e6d5d Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 8 Jan 2011 00:17:56 +0000 Subject: [PATCH 60/80] Don't define a CObject API in Python 3.2 The API is not available: a PyCapsule should be used. Nobody seems needing it for now. --- psycopg/psycopgmodule.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/psycopg/psycopgmodule.c b/psycopg/psycopgmodule.c index d580fa01..f91483df 100644 --- a/psycopg/psycopgmodule.c +++ b/psycopg/psycopgmodule.c @@ -760,10 +760,12 @@ static struct PyModuleDef psycopgmodule = { PyMODINIT_FUNC INIT_MODULE(_psycopg)(void) { +#if PY_VERSION_HEX < 0x03020000 static void *PSYCOPG_API[PSYCOPG_API_pointers]; + PyObject *c_api_object; +#endif PyObject *module = NULL, *dict; - PyObject *c_api_object; #ifdef PSYCOPG_DEBUG if (getenv("PSYCOPG_DEBUG")) @@ -861,9 +863,12 @@ INIT_MODULE(_psycopg)(void) /* PyBoxer_API[PyBoxer_Fake_NUM] = (void *)PyBoxer_Fake; */ /* Create a CObject containing the API pointer array's address */ + /* If anybody asks for a PyCapsule we'll deal with it. */ +#if PY_VERSION_HEX < 0x03020000 c_api_object = PyCObject_FromVoidPtr((void *)PSYCOPG_API, NULL); if (c_api_object != NULL) PyModule_AddObject(module, "_C_API", c_api_object); +#endif /* other mixed initializations of module-level variables */ psycoEncodings = PyDict_New(); From ac5cde8834c85eb0a7cd12812164ce562daeb32a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 8 Jan 2011 00:19:48 +0000 Subject: [PATCH 61/80] Only use absolute imports in the package In Python 3.2b2 the relative imports are not converted into explicit ones (with .). --- lib/__init__.py | 16 ++++++++-------- lib/extensions.py | 44 ++++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/lib/__init__.py b/lib/__init__.py index 065b2a70..4b9e22a2 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -66,17 +66,17 @@ from psycopg2 import tz # Import the DBAPI-2.0 stuff into top-level module. -from _psycopg import BINARY, NUMBER, STRING, DATETIME, ROWID +from psycopg2._psycopg import BINARY, NUMBER, STRING, DATETIME, ROWID -from _psycopg import Binary, Date, Time, Timestamp -from _psycopg import DateFromTicks, TimeFromTicks, TimestampFromTicks +from psycopg2._psycopg import Binary, Date, Time, Timestamp +from psycopg2._psycopg import DateFromTicks, TimeFromTicks, TimestampFromTicks -from _psycopg import Error, Warning, DataError, DatabaseError, ProgrammingError -from _psycopg import IntegrityError, InterfaceError, InternalError -from _psycopg import NotSupportedError, OperationalError +from psycopg2._psycopg import Error, Warning, DataError, DatabaseError, ProgrammingError +from psycopg2._psycopg import IntegrityError, InterfaceError, InternalError +from psycopg2._psycopg import NotSupportedError, OperationalError -from _psycopg import connect, apilevel, threadsafety, paramstyle -from _psycopg import __version__ +from psycopg2._psycopg import connect, apilevel, threadsafety, paramstyle +from psycopg2._psycopg import __version__ # Register default adapters. diff --git a/lib/extensions.py b/lib/extensions.py index 57aa9c68..82e17fa4 100644 --- a/lib/extensions.py +++ b/lib/extensions.py @@ -32,38 +32,38 @@ This module holds all the extensions to the DBAPI-2.0 provided by psycopg. # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. -from _psycopg import UNICODE, INTEGER, LONGINTEGER, BOOLEAN, FLOAT -from _psycopg import TIME, DATE, INTERVAL, DECIMAL -from _psycopg import BINARYARRAY, BOOLEANARRAY, DATEARRAY, DATETIMEARRAY -from _psycopg import DECIMALARRAY, FLOATARRAY, INTEGERARRAY, INTERVALARRAY -from _psycopg import LONGINTEGERARRAY, ROWIDARRAY, STRINGARRAY, TIMEARRAY -from _psycopg import UNICODEARRAY +from psycopg2._psycopg import UNICODE, INTEGER, LONGINTEGER, BOOLEAN, FLOAT +from psycopg2._psycopg import TIME, DATE, INTERVAL, DECIMAL +from psycopg2._psycopg import BINARYARRAY, BOOLEANARRAY, DATEARRAY, DATETIMEARRAY +from psycopg2._psycopg import DECIMALARRAY, FLOATARRAY, INTEGERARRAY, INTERVALARRAY +from psycopg2._psycopg import LONGINTEGERARRAY, ROWIDARRAY, STRINGARRAY, TIMEARRAY +from psycopg2._psycopg import UNICODEARRAY -from _psycopg import Binary, Boolean, Float, QuotedString, AsIs +from psycopg2._psycopg import Binary, Boolean, Float, QuotedString, AsIs try: - from _psycopg import MXDATE, MXDATETIME, MXINTERVAL, MXTIME - from _psycopg import MXDATEARRAY, MXDATETIMEARRAY, MXINTERVALARRAY, MXTIMEARRAY - from _psycopg import DateFromMx, TimeFromMx, TimestampFromMx - from _psycopg import IntervalFromMx -except: + from psycopg2._psycopg import MXDATE, MXDATETIME, MXINTERVAL, MXTIME + from psycopg2._psycopg import MXDATEARRAY, MXDATETIMEARRAY, MXINTERVALARRAY, MXTIMEARRAY + from psycopg2._psycopg import DateFromMx, TimeFromMx, TimestampFromMx + from psycopg2._psycopg import IntervalFromMx +except ImportError: pass try: - from _psycopg import PYDATE, PYDATETIME, PYINTERVAL, PYTIME - from _psycopg import PYDATEARRAY, PYDATETIMEARRAY, PYINTERVALARRAY, PYTIMEARRAY - from _psycopg import DateFromPy, TimeFromPy, TimestampFromPy - from _psycopg import IntervalFromPy -except: + from psycopg2._psycopg import PYDATE, PYDATETIME, PYINTERVAL, PYTIME + from psycopg2._psycopg import PYDATEARRAY, PYDATETIMEARRAY, PYINTERVALARRAY, PYTIMEARRAY + from psycopg2._psycopg import DateFromPy, TimeFromPy, TimestampFromPy + from psycopg2._psycopg import IntervalFromPy +except ImportError: pass -from _psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid -from _psycopg import string_types, binary_types, new_type, register_type -from _psycopg import ISQLQuote, Notify +from psycopg2._psycopg import adapt, adapters, encodings, connection, cursor, lobject, Xid +from psycopg2._psycopg import string_types, binary_types, new_type, register_type +from psycopg2._psycopg import ISQLQuote, Notify -from _psycopg import QueryCanceledError, TransactionRollbackError +from psycopg2._psycopg import QueryCanceledError, TransactionRollbackError try: - from _psycopg import set_wait_callback, get_wait_callback + from psycopg2._psycopg import set_wait_callback, get_wait_callback except ImportError: pass From 43c8fce45c858e96cb1cbb24d2623d721bbbf51c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 8 Jan 2011 00:21:07 +0000 Subject: [PATCH 62/80] Silence warnings due to deprecated TestCase methods With a veiled criticism. --- tests/testutils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/testutils.py b/tests/testutils.py index 49d26b21..67387187 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -41,6 +41,16 @@ else: unittest.TestCase.skipTest = skipTest +# Silence warnings caused by the stubborness of the Python unittest maintainers +# http://bugs.python.org/issue9424 +if not hasattr(unittest.TestCase, 'assert_') \ +or unittest.TestCase.assert_ is not unittest.TestCase.assertTrue: + # mavaff... + unittest.TestCase.assert_ = unittest.TestCase.assertTrue + unittest.TestCase.failUnless = unittest.TestCase.assertTrue + unittest.TestCase.assertEquals = unittest.TestCase.assertEqual + unittest.TestCase.failUnlessEqual = unittest.TestCase.assertEqual + def decorate_all_tests(cls, decorator): """Apply *decorator* to all the tests defined in the TestCase *cls*.""" From f8c4335f3590d8d254fcabcb7b133bd93091e8cf Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 8 Jan 2011 00:40:27 +0000 Subject: [PATCH 63/80] Avoid ResourceWarning in tests in Python 3.2 --- tests/test_lobject.py | 18 +++++++++++++++--- tests/testutils.py | 6 +++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 1c71fb4f..cc6621e4 100644 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -182,7 +182,11 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), b("some data")) + f = open(filename, "rb") + try: + self.assertEqual(f.read(), b("some data")) + finally: + f.close() def test_close_twice(self): lo = self.conn.lobject() @@ -224,7 +228,11 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), b("some data")) + f = open(filename, "rb") + try: + self.assertEqual(f.read(), b("some data")) + finally: + f.close() def test_close_after_commit(self): lo = self.conn.lobject() @@ -279,7 +287,11 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): filename = os.path.join(self.tmpdir, "data.txt") lo.export(filename) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename, "rb").read(), b("some data")) + f = open(filename, "rb") + try: + self.assertEqual(f.read(), b("some data")) + finally: + f.close() decorate_all_tests(LargeObjectTests, skip_if_no_lo) decorate_all_tests(LargeObjectTests, skip_if_green) diff --git a/tests/testutils.py b/tests/testutils.py index 67387187..cd034e50 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -121,5 +121,9 @@ def script_to_py3(script): if main("lib2to3.fixes", ['--no-diffs', '-w', '-n', f.name]): raise Exception('py3 conversion failed') - return open(f.name).read() + f2 = open(f.name) + try: + return f2.read() + finally: + f2.close() From 2cde9033acdcdea86da1c161e0aaec8f9bf441a9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 5 Jan 2011 00:05:29 +0000 Subject: [PATCH 64/80] Added documentation for Unicode support in large object Not implemented yet! --- doc/src/connection.rst | 29 ++++++++++++++++++++++++----- doc/src/extensions.rst | 16 ++++++++++++++-- doc/src/usage.rst | 3 ++- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/doc/src/connection.rst b/doc/src/connection.rst index bf6a6f26..d0f5e1a9 100644 --- a/doc/src/connection.rst +++ b/doc/src/connection.rst @@ -490,13 +490,14 @@ The ``connection`` class .. method:: lobject([oid [, mode [, new_oid [, new_file [, lobject_factory]]]]]) - Return a new database large object. See :ref:`large-objects` for an - overview. + Return a new database large object as a `~psycopg2.extensions.lobject` + instance. + + See :ref:`large-objects` for an overview. :param oid: The OID of the object to read or write. 0 to create a new large object and and have its OID assigned automatically. - :param mode: Access mode to the object: can be ``r``, ``w``, - ``rw`` or ``n`` (meaning don't open it). + :param mode: Access mode to the object, see below. :param new_oid: Create a new object using the specified OID. The function raises `OperationalError` if the OID is already in use. Default is 0, meaning assign a new one automatically. @@ -504,13 +505,31 @@ The ``connection`` class (using the |lo_import|_ function) :param lobject_factory: Subclass of `~psycopg2.extensions.lobject` to be instantiated. - :rtype: `~psycopg2.extensions.lobject` .. |lo_import| replace:: `!lo_import()` .. _lo_import: http://www.postgresql.org/docs/9.0/static/lo-interfaces.html#LO-IMPORT + Available values for *mode* are: + + ======= ========= + *mode* meaning + ======= ========= + ``r`` Open for read only + ``w`` Open for write only + ``rw`` Open for read/write + ``n`` Don't open the file + ``b`` Don't decode read data (return data as `str` in Python 2 or `bytes` in Python 3) + ``t`` Decode read data according to `connection.encoding` (return data as `unicode` in Python 2 or `str` in Python 3) + ======= ========= + + ``b`` and ``t`` can be specified together with a read/write mode. If + neither ``b`` nor ``t`` is specified, the default is ``b`` in Python 2 + and ``t`` in Python 3. + .. versionadded:: 2.0.8 + .. versionchanged:: 2.3.3 added ``b`` and ``t`` mode and unicode + support. .. rubric:: Methods related to asynchronous support. diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index 73b05db1..8fee890e 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -51,17 +51,29 @@ functionalities defined by the |DBAPI|_. .. attribute:: mode - The mode the database was open (``r``, ``w``, ``rw`` or ``n``). + The mode the database was open. See `connection.lobject()` for a + description of the available modes. .. method:: read(bytes=-1) Read a chunk of data from the current file position. If -1 (default) read all the remaining data. + The result is an Unicode string (decoded according to + `connection.encoding`) if the file was open in ``t`` mode, a bytes + string for ``b`` mode. + + .. versionchanged:: 2.3.3 + added Unicode support. + .. method:: write(str) Write a string to the large object. Return the number of bytes - written. + written. Unicode strings are encoded in the `connection.encoding` + before writing. + + .. versionchanged:: 2.3.3 + added Unicode support. .. method:: export(file_name) diff --git a/doc/src/usage.rst b/doc/src/usage.rst index a5efaa45..36bd36be 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -574,7 +574,8 @@ whole. Psycopg allows access to the large object using the `~psycopg2.extensions.lobject` class. Objects are generated using the -`connection.lobject()` factory method. +`connection.lobject()` factory method. Data can be retrieved either as bytes +or as Unicode strings. Psycopg large object support efficient import/export with file system files using the |lo_import|_ and |lo_export|_ libpq functions. From f63167a92c2124620bc37e6c3f217d2781f8124e Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 8 Jan 2011 23:47:24 +0000 Subject: [PATCH 65/80] Added tests to check large objects decoding --- tests/test_lobject.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index b5d5d63e..7f600e31 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -167,9 +167,34 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): lo.close() lo = self.conn.lobject(lo.oid) - self.assertEqual(lo.read(4), b("some")) + x = lo.read(4) + self.assertEqual(type(x), type('')) + self.assertEqual(x, "some") + self.assertEqual(lo.read(), " data") + + def test_read_binary(self): + lo = self.conn.lobject() + length = lo.write(b("some data")) + lo.close() + + lo = self.conn.lobject(lo.oid, "rb") + x = lo.read(4) + self.assertEqual(type(x), type(b(''))) + self.assertEqual(x, "some") self.assertEqual(lo.read(), b(" data")) + def test_read_text(self): + lo = self.conn.lobject() + snowman = u"\u2603" + length = lo.write(u"some data " + snowman) + lo.close() + + lo = self.conn.lobject(lo.oid, "rt") + x = lo.read(4) + self.assertEqual(type(x), type(u'')) + self.assertEqual(x, u"some") + self.assertEqual(lo.read(), u" data " + snowman) + def test_read_large(self): lo = self.conn.lobject() data = b("data") * 1000000 From ba1d77a29751fc3665e446cbc47ca749bd4d153f Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 9 Jan 2011 01:53:58 +0000 Subject: [PATCH 66/80] Large object mode parsing refactored Added parsing of text/binary mode. --- psycopg/connection_type.c | 41 ++++-------- psycopg/lobject.h | 11 +++- psycopg/lobject_int.c | 134 ++++++++++++++++++++++++++++++++------ psycopg/lobject_type.c | 15 +++-- tests/test_lobject.py | 6 +- 5 files changed, 146 insertions(+), 61 deletions(-) diff --git a/psycopg/connection_type.c b/psycopg/connection_type.c index c947850a..ae92d68b 100644 --- a/psycopg/connection_type.c +++ b/psycopg/connection_type.c @@ -514,15 +514,16 @@ static PyObject * psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) { Oid oid=InvalidOid, new_oid=InvalidOid; - char *smode = NULL, *new_file = NULL; - int mode=0; - PyObject *obj, *factory = NULL; + char *new_file = NULL; + const char *smode = ""; + PyObject *factory = (PyObject *)&lobjectType; + PyObject *obj; static char *kwlist[] = {"oid", "mode", "new_oid", "new_file", "cursor_factory", NULL}; - + if (!PyArg_ParseTupleAndKeywords(args, keywds, "|izizO", kwlist, - &oid, &smode, &new_oid, &new_file, + &oid, &smode, &new_oid, &new_file, &factory)) { return NULL; } @@ -537,33 +538,13 @@ psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) oid, smode); Dprintf("psyco_conn_lobject: parameters: new_oid = %d, new_file = %s", new_oid, new_file); - - /* build a mode number out of the mode string: right now we only accept - 'r', 'w' and 'rw' (but note that 'w' implies 'rw' because PostgreSQL - backend does that. */ - if (smode) { - if (strncmp("rw", smode, 2) == 0) - mode = INV_READ+INV_WRITE; - else if (smode[0] == 'r') - mode = INV_READ; - else if (smode[0] == 'w') - mode = INV_WRITE; - else if (smode[0] == 'n') - mode = -1; - else { - PyErr_SetString(PyExc_TypeError, - "mode should be one of 'r', 'w' or 'rw'"); - return NULL; - } - } - if (factory == NULL) factory = (PyObject *)&lobjectType; if (new_file) - obj = PyObject_CallFunction(factory, "Oiiis", - self, oid, mode, new_oid, new_file); + obj = PyObject_CallFunction(factory, "Oisis", + self, oid, smode, new_oid, new_file); else - obj = PyObject_CallFunction(factory, "Oiii", - self, oid, mode, new_oid); + obj = PyObject_CallFunction(factory, "Oisi", + self, oid, smode, new_oid); if (obj == NULL) return NULL; if (PyObject_IsInstance(obj, (PyObject *)&lobjectType) == 0) { @@ -572,7 +553,7 @@ psyco_conn_lobject(connectionObject *self, PyObject *args, PyObject *keywds) Py_DECREF(obj); return NULL; } - + Dprintf("psyco_conn_lobject: new lobject at %p: refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); diff --git a/psycopg/lobject.h b/psycopg/lobject.h index cddfa6e9..293f608f 100644 --- a/psycopg/lobject.h +++ b/psycopg/lobject.h @@ -42,7 +42,8 @@ typedef struct { connectionObject *conn; /* connection owning the lobject */ long int mark; /* copied from conn->mark */ - const char *smode; /* string mode if lobject was opened */ + char *smode; /* string mode if lobject was opened */ + int mode; /* numeric version of smode */ int fd; /* the file descriptor for file-like ops */ Oid oid; /* the oid for this lobject */ @@ -51,7 +52,7 @@ typedef struct { /* functions exported from lobject_int.c */ HIDDEN int lobject_open(lobjectObject *self, connectionObject *conn, - Oid oid, int mode, Oid new_oid, + Oid oid, const char *smode, Oid new_oid, const char *new_file); HIDDEN int lobject_unlink(lobjectObject *self); HIDDEN int lobject_export(lobjectObject *self, const char *filename); @@ -87,6 +88,12 @@ if (self->conn->mark != self->mark) { \ return NULL; \ } +/* Values for the lobject mode */ +#define LOBJECT_READ 1 +#define LOBJECT_WRITE 2 +#define LOBJECT_BINARY 4 +#define LOBJECT_TEXT 8 + #ifdef __cplusplus } #endif diff --git a/psycopg/lobject_int.c b/psycopg/lobject_int.c index 6ee3ecf0..252d1c93 100644 --- a/psycopg/lobject_int.c +++ b/psycopg/lobject_int.c @@ -43,15 +43,118 @@ collect_error(connectionObject *conn, char **error) *error = strdup(msg); } + +/* Check if the mode passed to the large object is valid. + * In case of success return a value >= 0 + * On error return a value < 0 and set an exception. + * + * Valid mode are [r|w|rw|n][t|b] + */ +static int +_lobject_parse_mode(const char *mode) +{ + int rv = 0; + size_t pos = 0; + + if (0 == strncmp("rw", mode, 2)) { + rv |= LOBJECT_READ | LOBJECT_WRITE; + pos += 2; + } + else { + switch (mode[0]) { + case 'r': + rv |= LOBJECT_READ; + pos += 1; + break; + case 'w': + rv |= LOBJECT_WRITE; + pos += 1; + break; + case 'n': + pos += 1; + break; + default: + rv |= LOBJECT_READ; + break; + } + } + + switch (mode[pos]) { + case 't': + rv |= LOBJECT_TEXT; + pos += 1; + break; + case 'b': + rv |= LOBJECT_BINARY; + pos += 1; + break; + default: +#if PY_MAJOR_VERSION < 3 + rv |= LOBJECT_BINARY; +#else + rv |= LOBJECT_TEXT; +#endif + break; + } + + if (pos != strlen(mode)) { + PyErr_Format(PyExc_ValueError, + "bad mode for lobject: '%s'", mode); + rv = -1; + } + + return rv; +} + + +/* Return a string representing the lobject mode. + * + * The return value is a new string allocated on the Python heap. + */ +static char * +_lobject_unparse_mode(int mode) +{ + char *buf; + char *c; + + /* the longest is 'rwt' */ + c = buf = PyMem_Malloc(4); + + if (mode & LOBJECT_READ) { *c++ = 'r'; } + if (mode & LOBJECT_WRITE) { *c++ = 'w'; } + + if (buf == c) { + /* neither read nor write */ + *c++ = 'n'; + } + else { + if (mode & LOBJECT_TEXT) { + *c++ = 't'; + } + else { + *c++ = 'b'; + } + } + *c = '\0'; + + return buf; +} + /* lobject_open - create a new/open an existing lo */ int lobject_open(lobjectObject *self, connectionObject *conn, - Oid oid, int mode, Oid new_oid, const char *new_file) + Oid oid, const char *smode, Oid new_oid, const char *new_file) { int retvalue = -1; PGresult *pgres = NULL; char *error = NULL; + int pgmode = 0; + int mode; + + if (0 > (mode = _lobject_parse_mode(smode))) { + return -1; + } Py_BEGIN_ALLOW_THREADS; pthread_mutex_lock(&(self->conn->lock)); @@ -78,19 +181,19 @@ lobject_open(lobjectObject *self, connectionObject *conn, goto end; } - mode = INV_WRITE; + mode = (mode & ~LOBJECT_READ) | LOBJECT_WRITE; } else { self->oid = oid; - if (mode == 0) mode = INV_READ; } - /* if the oid is a real one we try to open with the given mode, - unless the mode is -1, meaning "don't open!" */ - if (mode != -1) { - self->fd = lo_open(self->conn->pgconn, self->oid, mode); - Dprintf("lobject_open: large object opened with fd = %d", - self->fd); + /* if the oid is a real one we try to open with the given mode */ + if (mode & LOBJECT_READ) { pgmode |= INV_READ; } + if (mode & LOBJECT_WRITE) { pgmode |= INV_WRITE; } + if (pgmode) { + self->fd = lo_open(self->conn->pgconn, self->oid, pgmode); + Dprintf("lobject_open: large object opened with mode = %i fd = %d", + pgmode, self->fd); if (self->fd == -1) { collect_error(self->conn, &error); @@ -98,17 +201,10 @@ lobject_open(lobjectObject *self, connectionObject *conn, goto end; } } + /* set the mode for future reference */ - switch (mode) { - case -1: - self->smode = "n"; break; - case INV_READ: - self->smode = "r"; break; - case INV_WRITE: - self->smode = "w"; break; - case INV_READ+INV_WRITE: - self->smode = "rw"; break; - } + self->mode = mode; + self->smode = _lobject_unparse_mode(mode); retvalue = 0; end: diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 9a5fb0b8..4c9e2aeb 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -277,7 +277,7 @@ static struct PyMemberDef lobjectObject_members[] = { {"oid", T_UINT, offsetof(lobjectObject, oid), READONLY, "The backend OID associated to this lobject."}, {"mode", T_STRING, offsetof(lobjectObject, smode), READONLY, - "Open mode ('r', 'w', 'rw' or 'n')."}, + "Open mode."}, {NULL} }; @@ -293,7 +293,7 @@ static struct PyGetSetDef lobjectObject_getsets[] = { static int lobject_setup(lobjectObject *self, connectionObject *conn, - Oid oid, int mode, Oid new_oid, const char *new_file) + Oid oid, const char *smode, Oid new_oid, const char *new_file) { Dprintf("lobject_setup: init lobject object at %p", self); @@ -311,7 +311,7 @@ lobject_setup(lobjectObject *self, connectionObject *conn, self->fd = -1; self->oid = InvalidOid; - if (lobject_open(self, conn, oid, mode, new_oid, new_file) == -1) + if (lobject_open(self, conn, oid, smode, new_oid, new_file) == -1) return -1; Dprintf("lobject_setup: good lobject object at %p, refcnt = " @@ -328,6 +328,7 @@ lobject_dealloc(PyObject* obj) if (lobject_close(self) < 0) PyErr_Print(); Py_XDECREF((PyObject*)self->conn); + PyMem_Free(self->smode); Dprintf("lobject_dealloc: deleted lobject object at %p, refcnt = " FORMAT_CODE_PY_SSIZE_T, obj, Py_REFCNT(obj)); @@ -339,16 +340,16 @@ static int lobject_init(PyObject *obj, PyObject *args, PyObject *kwds) { Oid oid=InvalidOid, new_oid=InvalidOid; - int mode=0; + const char *smode = ""; const char *new_file = NULL; PyObject *conn; - if (!PyArg_ParseTuple(args, "O|iiis", - &conn, &oid, &mode, &new_oid, &new_file)) + if (!PyArg_ParseTuple(args, "O|iziz", + &conn, &oid, &smode, &new_oid, &new_file)) return -1; return lobject_setup((lobjectObject *)obj, - (connectionObject *)conn, oid, mode, new_oid, new_file); + (connectionObject *)conn, oid, smode, new_oid, new_file); } static PyObject * diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 7f600e31..3e99e868 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -84,7 +84,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_create(self): lo = self.conn.lobject() self.assertNotEqual(lo, None) - self.assertEqual(lo.mode, "w") + self.assertEqual(lo.mode[0], "w") def test_open_non_existent(self): # By creating then removing a large object, we get an Oid that @@ -98,12 +98,12 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): lo2 = self.conn.lobject(lo.oid) self.assertNotEqual(lo2, None) self.assertEqual(lo2.oid, lo.oid) - self.assertEqual(lo2.mode, "r") + self.assertEqual(lo2.mode[0], "r") def test_open_for_write(self): lo = self.conn.lobject() lo2 = self.conn.lobject(lo.oid, "w") - self.assertEqual(lo2.mode, "w") + self.assertEqual(lo2.mode[0], "w") lo2.write(b("some data")) def test_open_mode_n(self): From 789dda1173db364d46cb5db5847e77da39eba9a7 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sun, 9 Jan 2011 02:31:03 +0000 Subject: [PATCH 67/80] lobject read and write can deal with both bytes and unicode On write, unicode is encoded in connection encoding. On read, respect the lobject mode 't' or 'b'. --- NEWS-2.3 | 3 +++ psycopg/lobject_type.c | 47 +++++++++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/NEWS-2.3 b/NEWS-2.3 index 8353861c..405c1e14 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -6,6 +6,9 @@ What's new in psycopg 2.3.3 - Added `register_composite()` function to cast PostgreSQL composite types into Python tuples/namedtuples. - Connections and cursors are weakly referenceable. + - Added 'b' and 't' mode to large objects: write can deal with both bytes + strings and unicode; read can return either bytes strings or decoded + unicode. - The build script refuses to guess values if pg_config is not found. - Improved PostgreSQL-Python encodings mapping. Added a few missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, diff --git a/psycopg/lobject_type.c b/psycopg/lobject_type.c index 4c9e2aeb..49f64bab 100644 --- a/psycopg/lobject_type.c +++ b/psycopg/lobject_type.c @@ -71,19 +71,48 @@ psyco_lobj_close(lobjectObject *self, PyObject *args) static PyObject * psyco_lobj_write(lobjectObject *self, PyObject *args) { - int res = 0; + char *buffer; Py_ssize_t len; - const char *buffer; + Py_ssize_t res; + PyObject *obj; + PyObject *data = NULL; + PyObject *rv = NULL; - if (!PyArg_ParseTuple(args, "s#", &buffer, &len)) return NULL; + if (!PyArg_ParseTuple(args, "O", &obj)) return NULL; EXC_IF_LOBJ_CLOSED(self); EXC_IF_LOBJ_LEVEL0(self); EXC_IF_LOBJ_UNMARKED(self); - if ((res = lobject_write(self, buffer, (size_t)len)) < 0) return NULL; + if (Bytes_Check(obj)) { + Py_INCREF(obj); + data = obj; + } + else if (PyUnicode_Check(obj)) { + if (!(data = PyUnicode_AsEncodedString(obj, self->conn->codec, NULL))) { + goto exit; + } + } + else { + PyErr_Format(PyExc_TypeError, + "lobject.write requires a string; got %s instead", + Py_TYPE(obj)->tp_name); + goto exit; + } - return PyInt_FromLong((long)res); + if (-1 == Bytes_AsStringAndSize(data, &buffer, &len)) { + goto exit; + } + + if (0 > (res = lobject_write(self, buffer, (size_t)len))) { + goto exit; + } + + rv = PyInt_FromLong((long)res); + +exit: + Py_XDECREF(data); + return rv; } /* read method - read data from the lobject */ @@ -120,9 +149,13 @@ psyco_lobj_read(lobjectObject *self, PyObject *args) return NULL; } - res = Bytes_FromStringAndSize(buffer, size); + if (self->mode & LOBJECT_BINARY) { + res = Bytes_FromStringAndSize(buffer, size); + } else { + res = PyUnicode_Decode(buffer, size, self->conn->codec, NULL); + } PyMem_Free(buffer); - + return res; } From 3ba0631dbb75514f51552e9168524e38a3efa787 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 10 Jan 2011 22:05:47 +0000 Subject: [PATCH 68/80] A few large objects tests fixed The behavior changed in Python 3 after the lobject-decode merge. --- tests/test_lobject.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_lobject.py b/tests/test_lobject.py index 3e99e868..d327bb96 100755 --- a/tests/test_lobject.py +++ b/tests/test_lobject.py @@ -144,7 +144,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): fp.close() lo = self.conn.lobject(0, "r", 0, filename) - self.assertEqual(lo.read(), b("some data")) + self.assertEqual(lo.read(), "some data") def test_close(self): lo = self.conn.lobject() @@ -180,7 +180,7 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): lo = self.conn.lobject(lo.oid, "rb") x = lo.read(4) self.assertEqual(type(x), type(b(''))) - self.assertEqual(x, "some") + self.assertEqual(x, b("some")) self.assertEqual(lo.read(), b(" data")) def test_read_text(self): @@ -197,13 +197,16 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): def test_read_large(self): lo = self.conn.lobject() - data = b("data") * 1000000 - length = lo.write(b("some") + data) + data = "data" * 1000000 + length = lo.write("some" + data) lo.close() lo = self.conn.lobject(lo.oid) - self.assertEqual(lo.read(4), b("some")) - self.assertEqual(lo.read(), data) + self.assertEqual(lo.read(4), "some") + data1 = lo.read() + # avoid dumping megacraps in the console in case of error + self.assert_(data == data1, + "%r... != %r..." % (data[:100], data1[:100])) def test_seek_tell(self): lo = self.conn.lobject() @@ -214,17 +217,17 @@ class LargeObjectTests(LargeObjectMixin, unittest.TestCase): self.assertEqual(lo.seek(5, 0), 5) self.assertEqual(lo.tell(), 5) - self.assertEqual(lo.read(), b("data")) + self.assertEqual(lo.read(), "data") # SEEK_CUR: relative current location lo.seek(5) self.assertEqual(lo.seek(2, 1), 7) self.assertEqual(lo.tell(), 7) - self.assertEqual(lo.read(), b("ta")) + self.assertEqual(lo.read(), "ta") # SEEK_END: relative to end of file self.assertEqual(lo.seek(-2, 2), length - 2) - self.assertEqual(lo.read(), b("ta")) + self.assertEqual(lo.read(), "ta") def test_unlink(self): lo = self.conn.lobject() From 332acccc6ec1fedc7a460089a695a772fe341232 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 12 Jan 2011 17:02:40 +0000 Subject: [PATCH 69/80] Replace b('str') with b'str' in Python 3 This avoids an encode() call for each of these constants. Use a custom 2to3 fixer in setup.py to perform the conversion. --- scripts/fix_b.py | 20 ++++++++++++++++++++ setup.py | 6 ++++++ 2 files changed, 26 insertions(+) create mode 100644 scripts/fix_b.py diff --git a/scripts/fix_b.py b/scripts/fix_b.py new file mode 100644 index 00000000..ccc8e1c9 --- /dev/null +++ b/scripts/fix_b.py @@ -0,0 +1,20 @@ +"""Fixer to change b('string') into b'string'.""" +# Author: Daniele Varrazzo + +import token +from lib2to3 import fixer_base +from lib2to3.pytree import Leaf + +class FixB(fixer_base.BaseFix): + + PATTERN = """ + power< wrapper='b' trailer< '(' arg=[any] ')' > rest=any* > + """ + + def transform(self, node, results): + arg = results['arg'] + wrapper = results["wrapper"] + if len(arg) == 1 and arg[0].type == token.STRING: + b = Leaf(token.STRING, 'b' + arg[0].value, prefix=wrapper.prefix) + node.children = [ b ] + results['rest'] + diff --git a/setup.py b/setup.py index 926169c3..1eb2206c 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,12 @@ try: from distutils.command.build_py import build_py_2to3 as build_py except ImportError: from distutils.command.build_py import build_py +else: + # Configure distutils to run our custom 2to3 fixers as well + from lib2to3.refactor import get_fixers_from_package + build_py.fixer_names = get_fixers_from_package('lib2to3.fixes') + build_py.fixer_names.append('fix_b') + sys.path.insert(0, 'scripts') try: import configparser From bde443a90293c5f18b72b100a21940b102ca784b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 18 Jan 2011 02:45:31 +0000 Subject: [PATCH 70/80] Fixed standard_conforming_strings filtering in Python 3 tests --- tests/types_extras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/types_extras.py b/tests/types_extras.py index 1f4f69c4..bb763384 100755 --- a/tests/types_extras.py +++ b/tests/types_extras.py @@ -35,7 +35,7 @@ def filter_scs(conn, s): if conn.get_parameter_status("standard_conforming_strings") == 'off': return s else: - return s.replace("E'", "'") + return s.replace(b("E'"), b("'")) class TypesExtrasTests(unittest.TestCase): """Test that all type conversions are working.""" From 8a1de1ec4e6f9af82538a9674afebac21bc28e0a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 1 Feb 2011 02:59:09 +0000 Subject: [PATCH 71/80] Added test to verify named cursor efficiency. Iter shouldn't fetch one record at time. --- tests/test_cursor.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_cursor.py b/tests/test_cursor.py index dec0cd14..da61c592 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -22,6 +22,7 @@ # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public # License for more details. +import time import unittest import psycopg2 import psycopg2.extensions @@ -128,6 +129,19 @@ class CursorTests(unittest.TestCase): del curs self.assert_(w() is None) + def test_iter_named_cursor_efficient(self): + curs = self.conn.cursor('tmp') + # if these records are fetched in the same roundtrip their + # timestamp will not be influenced by the pause in Python world. + curs.execute("""select clock_timestamp() from generate_series(1,2)""") + i = iter(curs) + t1 = i.next()[0] + time.sleep(0.2) + t2 = i.next()[0] + self.assert_((t2 - t1).microseconds * 1e-6 < 0.1, + "named cursor records fetched in 2 roundtrips (delta: %s)" + % (t2 - t1)) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From b358c54f02323f9d45740e396c5a11162396b1e4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 4 Feb 2011 12:22:07 +0000 Subject: [PATCH 72/80] More efficient cursor.iter: fetch many records at time. --- psycopg/cursor_type.c | 68 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 6 deletions(-) diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index 5a6722c7..e4790b61 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -799,6 +799,56 @@ psyco_curs_fetchone(cursorObject *self, PyObject *args) return res; } +/* Efficient cursor.next() implementation for named cursors. + * + * Fetch several records at time. Return NULL when the cursor is exhausted. + */ +static PyObject * +psyco_curs_next_named(cursorObject *self) +{ + PyObject *res; + + Dprintf("psyco_curs_next_named"); + EXC_IF_CURS_CLOSED(self); + EXC_IF_ASYNC_IN_PROGRESS(self, next); + if (_psyco_curs_prefetch(self) < 0) return NULL; + EXC_IF_NO_TUPLES(self); + + EXC_IF_NO_MARK(self); + EXC_IF_TPC_PREPARED(self->conn, next); + + Dprintf("psyco_curs_next_named: row %ld", self->row); + Dprintf("psyco_curs_next_named: rowcount = %ld", self->rowcount); + if (self->row >= self->rowcount) { + char buffer[128]; + /* FIXME: use a cursor member for the size */ + PyOS_snprintf(buffer, 127, "FETCH FORWARD 10 FROM %s", self->name); + if (pq_execute(self, buffer, 0) == -1) return NULL; + if (_psyco_curs_prefetch(self) < 0) return NULL; + } + + /* We exhausted the data: return NULL to stop iteration. */ + if (self->row >= self->rowcount) { + return NULL; + } + + if (self->tuple_factory == Py_None) + res = _psyco_curs_buildrow(self, self->row); + else + res = _psyco_curs_buildrow_with_factory(self, self->row); + + self->row++; /* move the counter to next line */ + + /* if the query was async aggresively free pgres, to allow + successive requests to reallocate it */ + if (self->row >= self->rowcount + && self->conn->async_cursor + && PyWeakref_GetObject(self->conn->async_cursor) == (PyObject*)self) + IFCLEARPGRES(self->pgres); + + return res; +} + /* fetch many - fetch some results */ @@ -1510,14 +1560,20 @@ cursor_next(PyObject *self) { PyObject *res; - /* we don't parse arguments: psyco_curs_fetchone will do that for us */ - res = psyco_curs_fetchone((cursorObject*)self, NULL); + if (NULL == ((cursorObject*)self)->name) { + /* we don't parse arguments: psyco_curs_fetchone will do that for us */ + res = psyco_curs_fetchone((cursorObject*)self, NULL); - /* convert a None to NULL to signal the end of iteration */ - if (res && res == Py_None) { - Py_DECREF(res); - res = NULL; + /* convert a None to NULL to signal the end of iteration */ + if (res && res == Py_None) { + Py_DECREF(res); + res = NULL; + } } + else { + res = psyco_curs_next_named((cursorObject*)self); + } + return res; } From b544354db2603e753dd2e07b80f880001ec4ccae Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Sat, 5 Feb 2011 15:12:37 +0100 Subject: [PATCH 73/80] COPY sends unicode to a file if it derives from io.TextIoBase Fixes ticket #36. --- NEWS-2.3 | 1 + psycopg/pqpath.c | 18 +++++++++++++- psycopg/psycopg.h | 1 + psycopg/utils.c | 40 +++++++++++++++++++++++++++++++ tests/test_copy.py | 60 +++++++++++++++++++++++++++++++++++++++++++--- tests/testutils.py | 13 ++++++++++ 6 files changed, 129 insertions(+), 4 deletions(-) diff --git a/NEWS-2.3 b/NEWS-2.3 index f406824d..40d86221 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -9,6 +9,7 @@ What's new in psycopg 2.3.3 - Added 'b' and 't' mode to large objects: write can deal with both bytes strings and unicode; read can return either bytes strings or decoded unicode. + - COPY sends Unicode data to files implementing io.TextIOBase. - The build script refuses to guess values if pg_config is not found. - Improved PostgreSQL-Python encodings mapping. Added a few missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, diff --git a/psycopg/pqpath.c b/psycopg/pqpath.c index 76e7c24b..7f9a1013 100644 --- a/psycopg/pqpath.c +++ b/psycopg/pqpath.c @@ -1163,7 +1163,9 @@ static int _pq_copy_out_v3(cursorObject *curs) { PyObject *tmp = NULL, *func; + PyObject *obj = NULL; int ret = -1; + int is_text; char *buffer; Py_ssize_t len; @@ -1173,14 +1175,28 @@ _pq_copy_out_v3(cursorObject *curs) goto exit; } + /* if the file is text we must pass it unicode. */ + if (-1 == (is_text = psycopg_is_text_file(curs->copyfile))) { + goto exit; + } + while (1) { Py_BEGIN_ALLOW_THREADS; len = PQgetCopyData(curs->conn->pgconn, &buffer, 0); Py_END_ALLOW_THREADS; if (len > 0 && buffer) { - tmp = PyObject_CallFunction(func, "s#", buffer, len); + if (is_text) { + obj = PyUnicode_Decode(buffer, len, curs->conn->codec, NULL); + } else { + obj = Bytes_FromStringAndSize(buffer, len); + } + PQfreemem(buffer); + if (!obj) { goto exit; } + tmp = PyObject_CallFunctionObjArgs(func, obj, NULL); + Py_DECREF(obj); + if (tmp == NULL) { goto exit; } else { diff --git a/psycopg/psycopg.h b/psycopg/psycopg.h index e87744c7..8edc42ed 100644 --- a/psycopg/psycopg.h +++ b/psycopg/psycopg.h @@ -123,6 +123,7 @@ HIDDEN char *psycopg_escape_string(PyObject *conn, HIDDEN char *psycopg_strdup(const char *from, Py_ssize_t len); HIDDEN PyObject * psycopg_ensure_bytes(PyObject *obj); HIDDEN PyObject * psycopg_ensure_text(PyObject *obj); +HIDDEN int psycopg_is_text_file(PyObject *f); /* Exceptions docstrings */ #define Error_doc \ diff --git a/psycopg/utils.c b/psycopg/utils.c index 539081ba..4e5c83ed 100644 --- a/psycopg/utils.c +++ b/psycopg/utils.c @@ -149,3 +149,43 @@ psycopg_ensure_text(PyObject *obj) } #endif } + +/* Check if a file derives from TextIOBase. + * + * Return 1 if it does, else 0, -1 on errors. + */ +int +psycopg_is_text_file(PyObject *f) +{ + /* NULL before any call. + * then io.TextIOBase if exists, else None. */ + static PyObject *base; + + /* Try to import os.TextIOBase */ + if (NULL == base) { + PyObject *m; + Dprintf("psycopg_is_text_file: importing io.TextIOBase"); + if (!(m = PyImport_ImportModule("io"))) { + Dprintf("psycopg_is_text_file: io module not found"); + PyErr_Clear(); + Py_INCREF(Py_None); + base = Py_None; + } + else { + if (!(base = PyObject_GetAttrString(m, "TextIOBase"))) { + Dprintf("psycopg_is_text_file: io.TextIOBase not found"); + PyErr_Clear(); + Py_INCREF(Py_None); + base = Py_None; + } + } + Py_XDECREF(m); + } + + if (base != Py_None) { + return PyObject_IsInstance(f, base); + } else { + return 0; + } +} + diff --git a/tests/test_copy.py b/tests/test_copy.py index 877c63c2..b6da4b1b 100755 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -23,8 +23,9 @@ # License for more details. import os +import sys import string -from testutils import unittest, decorate_all_tests +from testutils import unittest, decorate_all_tests, skip_if_no_iobase from cStringIO import StringIO from itertools import cycle, izip @@ -42,7 +43,12 @@ def skip_if_green(f): return skip_if_green_ -class MinimalRead(object): +if sys.version_info[0] < 3: + _base = object +else: + from io import TextIOBase as _base + +class MinimalRead(_base): """A file wrapper exposing the minimal interface to copy from.""" def __init__(self, f): self.f = f @@ -53,7 +59,7 @@ class MinimalRead(object): def readline(self): return self.f.readline() -class MinimalWrite(object): +class MinimalWrite(_base): """A file wrapper exposing the minimal interface to copy to.""" def __init__(self, f): self.f = f @@ -66,6 +72,9 @@ class CopyTests(unittest.TestCase): def setUp(self): self.conn = psycopg2.connect(dsn) + self._create_temp_table() + + def _create_temp_table(self): curs = self.conn.cursor() curs.execute(''' CREATE TEMPORARY TABLE tcopy ( @@ -126,6 +135,51 @@ class CopyTests(unittest.TestCase): finally: curs.close() + @skip_if_no_iobase + def test_copy_text(self): + self.conn.set_client_encoding('latin1') + self._create_temp_table() # the above call closed the xn + + if sys.version_info[0] < 3: + abin = ''.join(map(chr, range(32, 127) + range(160, 256))) + about = abin.decode('latin1').replace('\\', '\\\\') + + else: + abin = bytes(range(32, 127) + range(160, 256)).decode('latin1') + about = abin.replace('\\', '\\\\') + + curs = self.conn.cursor() + curs.execute('insert into tcopy values (%s, %s)', + (42, abin)) + + import io + f = io.StringIO() + curs.copy_to(f, 'tcopy', columns=('data',)) + f.seek(0) + self.assertEqual(f.readline().rstrip(), about) + + @skip_if_no_iobase + def test_copy_bytes(self): + self.conn.set_client_encoding('latin1') + self._create_temp_table() # the above call closed the xn + + if sys.version_info[0] < 3: + abin = ''.join(map(chr, range(32, 127) + range(160, 255))) + about = abin.replace('\\', '\\\\') + else: + abin = bytes(range(32, 127) + range(160, 255)).decode('latin1') + about = abin.replace('\\', '\\\\').encode('latin1') + + curs = self.conn.cursor() + curs.execute('insert into tcopy values (%s, %s)', + (42, abin)) + + import io + f = io.BytesIO() + curs.copy_to(f, 'tcopy', columns=('data',)) + f.seek(0) + self.assertEqual(f.readline().rstrip(), about) + def _copy_from(self, curs, nrecs, srec, copykw): f = StringIO() for i, c in izip(xrange(nrecs), cycle(string.ascii_letters)): diff --git a/tests/testutils.py b/tests/testutils.py index 3e6b6ea8..98297fc6 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -151,6 +151,19 @@ def skip_if_tpc_disabled(f): return skip_if_tpc_disabled_ +def skip_if_no_iobase(f): + """Skip a test if io.TextIOBase is not available.""" + def skip_if_no_iobase_(self): + try: + from io import TextIOBase + except ImportError: + return self.skipTest("io.TextIOBase not found.") + else: + return f(self) + + return skip_if_no_iobase_ + + def skip_on_python2(f): """Skip a test on Python 3 and following.""" def skip_on_python2_(self): From fab31e94418a010f03205c6377c864f47dab58c4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Fri, 4 Feb 2011 17:29:29 +0100 Subject: [PATCH 74/80] Fetch 'arraysize' records per roundtrip in named cursors iteration Closes ticket #33. --- NEWS-2.3 | 1 + doc/src/cursor.rst | 17 +++++++++++++++++ psycopg/cursor_type.c | 9 +++++++-- tests/test_cursor.py | 15 +++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/NEWS-2.3 b/NEWS-2.3 index df9e6afc..5e168f19 100644 --- a/NEWS-2.3 +++ b/NEWS-2.3 @@ -5,6 +5,7 @@ What's new in psycopg 2.3.3 - Added `register_composite()` function to cast PostgreSQL composite types into Python tuples/namedtuples. + - More efficient iteration on named cursors. - The build script refuses to guess values if pg_config is not found. - Connections and cursors are weakly referenceable. diff --git a/doc/src/cursor.rst b/doc/src/cursor.rst index bed2f7a5..57fe73bc 100644 --- a/doc/src/cursor.rst +++ b/doc/src/cursor.rst @@ -208,6 +208,11 @@ The ``cursor`` class (2, None, 'dada') (3, 42, 'bar') + .. versionchanged:: 2.3.3 + iterating over a :ref:`named cursor ` + fetches `~cursor.arraysize` records at time from the backend. + Previously only one record was fetched per roundtrip, resulting + in unefficient iteration. .. method:: fetchone() @@ -300,6 +305,18 @@ The ``cursor`` class This read/write attribute specifies the number of rows to fetch at a time with `~cursor.fetchmany()`. It defaults to 1 meaning to fetch a single row at a time. + + The attribute is also used when iterating a :ref:`named cursor + `: when syntax such as ``for i in cursor:`` is + used, in order to avoid an excessive number of network roundtrips, the + cursor will actually fetch `!arraysize` records at time from the + backend. For this task the default value of 1 is a poor value: if + `!arraysize` is 1, a default value of 2000 will be used instead. If + you really want to retrieve one record at time from the backend use + `fetchone()` in a loop. + + .. versionchanged:: 2.3.3 + `!arraysize` used in named cursor iteration. .. attribute:: rowcount diff --git a/psycopg/cursor_type.c b/psycopg/cursor_type.c index e4790b61..5416269e 100644 --- a/psycopg/cursor_type.c +++ b/psycopg/cursor_type.c @@ -821,8 +821,13 @@ psyco_curs_next_named(cursorObject *self) Dprintf("psyco_curs_next_named: rowcount = %ld", self->rowcount); if (self->row >= self->rowcount) { char buffer[128]; - /* FIXME: use a cursor member for the size */ - PyOS_snprintf(buffer, 127, "FETCH FORWARD 10 FROM %s", self->name); + + /* fetch 'arraysize' records, but shun the default value of 1 */ + long int size = self->arraysize; + if (size == 1) { size = 2000L; } + + PyOS_snprintf(buffer, 127, "FETCH FORWARD %ld FROM %s", + size, self->name); if (pq_execute(self, buffer, 0) == -1) return NULL; if (_psyco_curs_prefetch(self) < 0) return NULL; } diff --git a/tests/test_cursor.py b/tests/test_cursor.py index da61c592..a859f42b 100755 --- a/tests/test_cursor.py +++ b/tests/test_cursor.py @@ -142,6 +142,21 @@ class CursorTests(unittest.TestCase): "named cursor records fetched in 2 roundtrips (delta: %s)" % (t2 - t1)) + def test_iter_named_cursor_default_arraysize(self): + curs = self.conn.cursor('tmp') + curs.execute('select generate_series(1,50)') + rv = [ (r[0], curs.rownumber) for r in curs ] + # everything swallowed in one gulp + self.assertEqual(rv, [(i,i) for i in range(1,51)]) + + def test_iter_named_cursor_arraysize(self): + curs = self.conn.cursor('tmp') + curs.arraysize = 30 + curs.execute('select generate_series(1,50)') + rv = [ (r[0], curs.rownumber) for r in curs ] + # everything swallowed in two gulps + self.assertEqual(rv, [(i,((i - 1) % 30) + 1) for i in range(1,51)]) + def test_suite(): return unittest.TestLoader().loadTestsFromName(__name__) From 352d0d1f07e2ec0c0aeaa853f57bf88189e134c6 Mon Sep 17 00:00:00 2001 From: Federico Di Gregorio Date: Sun, 6 Feb 2011 16:54:35 +0100 Subject: [PATCH 75/80] Preparing 2.4 beta 1 * Unified NEWS (we don 't support 2.0 anymore) * Bumped up version in setup.py and ZPsycopgDA/DA.py --- NEWS-2.0 => NEWS | 140 +++++++++++++++++++++++++++++++++++++++++++++++ NEWS-2.3 | 140 ----------------------------------------------- ZPsycopgDA/DA.py | 2 +- setup.py | 2 +- 4 files changed, 142 insertions(+), 142 deletions(-) rename NEWS-2.0 => NEWS (79%) delete mode 100644 NEWS-2.3 diff --git a/NEWS-2.0 b/NEWS similarity index 79% rename from NEWS-2.0 rename to NEWS index 6f32ee75..74139322 100644 --- a/NEWS-2.0 +++ b/NEWS @@ -1,3 +1,143 @@ +What's new in psycopg 2.4 +------------------------- + +* New features and changes: + + - Added `register_composite()` function to cast PostgreSQL composite types + into Python tuples/namedtuples. + - More efficient iteration on named cursors. + - The build script refuses to guess values if pg_config is not found. + - Connections and cursors are weakly referenceable. + - Added 'b' and 't' mode to large objects: write can deal with both bytes + strings and unicode; read can return either bytes strings or decoded + unicode. + - COPY sends Unicode data to files implementing io.TextIOBase. + - The build script refuses to guess values if pg_config is not found. + - Improved PostgreSQL-Python encodings mapping. Added a few + missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, + LATIN10, SHIFT_JIS_2004. + - Dropped repeated dictionary lookups with unicode query/parameters. + - Empty lists correctly roundtrip Python -> PostgreSQL -> Python. + +* Bug fixes: + + - Fixed adaptation of None in composite types (ticket #26). Bug report by + Karsten Hilbert. + - Fixed several reference leaks in less common code paths. + - Fixed segfault when a large object is closed and its connection no more + available. + - Added missing icon to ZPsycopgDA package, not available in Zope 2.12.9 + (ticket #30). Bug report and patch by Pumukel. + + +What's new in psycopg 2.3.2 +--------------------------- + + - Fixed segfault with middleware not passing DateStyle to the client + (ticket #24). Bug report and patch by Marti Raudsepp. + + +What's new in psycopg 2.3.1 +--------------------------- + + - Fixed build problem on CentOS 5.5 x86_64 (ticket #23). + + +What's new in psycopg 2.3.0 +--------------------------- + +psycopg 2.3 aims to expose some new features introduced in PostgreSQL 9.0. + +* Main new features: + + - `dict` to `hstore` adapter and `hstore` to `dict` typecaster, using both + 9.0 and pre-9.0 syntax. + - Two-phase commit protocol support as per DBAPI specification. + - Support for payload in notifications received from the backed. + - `namedtuple`-returning cursor. + - Query execution cancel. + +* Other features and changes: + + - Dropped support for protocol 2: Psycopg 2.3 can only connect to PostgreSQL + servers with version at least 7.4. + - Don't issue a query at every connection to detect the client encoding + and to set the datestyle to ISO if it is already compatible with what + expected. + - `mogrify()` now supports unicode queries. + - Subclasses of a type that can be adapted are adapted as the superclass. + - `errorcodes` knows a couple of new codes introduced in PostgreSQL 9.0. + - Dropped deprecated Psycopg "own quoting". + - Never issue a ROLLBACK on close/GC. This behaviour was introduced as a bug + in release 2.2, but trying to send a command while being destroyed has been + considered not safe. + +* Bug fixes: + + - Fixed use of `PQfreemem` instead of `free` in binary typecaster. + - Fixed access to freed memory in `conn_get_isolation_level()`. + - Fixed crash during Decimal adaptation with a few 2.5.x Python versions + (ticket #7). + - Fixed notices order (ticket #9). + + +What's new in psycopg 2.2.2 +--------------------------- + +* Bux fixes: + + - the call to logging.basicConfig() in pool.py has been dropped: it was + messing with some projects using logging (and a library should not + initialize the logging system anyway.) + - psycopg now correctly handles time zones with seconds in the UTC offset. + The old register_tstz_w_secs() function is deprecated and will raise a + warning if called. + - Exceptions raised by the column iterator are propagated. + - Exceptions raised by executemany() interators are propagated. + + +What's new in psycopg 2.2.1 +--------------------------- + +* Bux fixes: + + - psycopg now builds again on MS Windows. + + +What's new in psycopg 2.2.0 +--------------------------- + +This is the first release of the new 2.2 series, supporting not just one but +two different ways of executing asynchronous queries, thanks to Jan and Daniele +(with a little help from me and others, but they did 99% of the work so they +deserve their names here in the news.) + +psycopg now supports both classic select() loops and "green" coroutine +libraries. It is all in the documentation, so just point your browser to +doc/html/advanced.html. + +* Other new features: + + - truncate() method for lobjects. + - COPY functions are now a little bit faster. + - All builtin PostgreSQL to Python typecasters are now available from the + psycopg2.extensions module. + - Notifications from the backend are now available right after the execute() + call (before client code needed to call isbusy() to ensure NOTIFY + reception.) + - Better timezone support. + - Lots of documentation updates. + +* Bug fixes: + + - Fixed some gc/refcounting problems. + - Fixed reference leak in NOTIFY reception. + - Fixed problem with PostgreSQL not casting string literals to the correct + types in some situations: psycopg now add an explicit cast to dates, times + and bytea representations. + - Fixed TimestampFromTicks() and TimeFromTicks() for seconds >= 59.5. + - Fixed spurious exception raised when calling C typecasters from Python + ones. What's new in psycopg 2.0.14 ---------------------------- diff --git a/NEWS-2.3 b/NEWS-2.3 deleted file mode 100644 index 40d049a2..00000000 --- a/NEWS-2.3 +++ /dev/null @@ -1,140 +0,0 @@ -What's new in psycopg 2.3.3 ---------------------------- - -* New features and changes: - - - Added `register_composite()` function to cast PostgreSQL composite types - into Python tuples/namedtuples. - - More efficient iteration on named cursors. - - The build script refuses to guess values if pg_config is not found. - - Connections and cursors are weakly referenceable. - - Added 'b' and 't' mode to large objects: write can deal with both bytes - strings and unicode; read can return either bytes strings or decoded - unicode. - - COPY sends Unicode data to files implementing io.TextIOBase. - - The build script refuses to guess values if pg_config is not found. - - Improved PostgreSQL-Python encodings mapping. Added a few - missing encodings: EUC_CN, EUC_JIS_2004, ISO885910, ISO885916, - LATIN10, SHIFT_JIS_2004. - - Dropped repeated dictionary lookups with unicode query/parameters. - - Empty lists correctly roundtrip Python -> PostgreSQL -> Python. - -* Bug fixes: - - - Fixed adaptation of None in composite types (ticket #26). Bug report by - Karsten Hilbert. - - Fixed several reference leaks in less common code paths. - - Fixed segfault when a large object is closed and its connection no more - available. - - Added missing icon to ZPsycopgDA package, not available in Zope 2.12.9 - (ticket #30). Bug report and patch by Pumukel. - - -What's new in psycopg 2.3.2 ---------------------------- - - - Fixed segfault with middleware not passing DateStyle to the client - (ticket #24). Bug report and patch by Marti Raudsepp. - - -What's new in psycopg 2.3.1 ---------------------------- - - - Fixed build problem on CentOS 5.5 x86_64 (ticket #23). - - -What's new in psycopg 2.3.0 ---------------------------- - -psycopg 2.3 aims to expose some new features introduced in PostgreSQL 9.0. - -* Main new features: - - - `dict` to `hstore` adapter and `hstore` to `dict` typecaster, using both - 9.0 and pre-9.0 syntax. - - Two-phase commit protocol support as per DBAPI specification. - - Support for payload in notifications received from the backed. - - `namedtuple`-returning cursor. - - Query execution cancel. - -* Other features and changes: - - - Dropped support for protocol 2: Psycopg 2.3 can only connect to PostgreSQL - servers with version at least 7.4. - - Don't issue a query at every connection to detect the client encoding - and to set the datestyle to ISO if it is already compatible with what - expected. - - `mogrify()` now supports unicode queries. - - Subclasses of a type that can be adapted are adapted as the superclass. - - `errorcodes` knows a couple of new codes introduced in PostgreSQL 9.0. - - Dropped deprecated Psycopg "own quoting". - - Never issue a ROLLBACK on close/GC. This behaviour was introduced as a bug - in release 2.2, but trying to send a command while being destroyed has been - considered not safe. - -* Bug fixes: - - - Fixed use of `PQfreemem` instead of `free` in binary typecaster. - - Fixed access to freed memory in `conn_get_isolation_level()`. - - Fixed crash during Decimal adaptation with a few 2.5.x Python versions - (ticket #7). - - Fixed notices order (ticket #9). - - -What's new in psycopg 2.2.2 ---------------------------- - -* Bux fixes: - - - the call to logging.basicConfig() in pool.py has been dropped: it was - messing with some projects using logging (and a library should not - initialize the logging system anyway.) - - psycopg now correctly handles time zones with seconds in the UTC offset. - The old register_tstz_w_secs() function is deprecated and will raise a - warning if called. - - Exceptions raised by the column iterator are propagated. - - Exceptions raised by executemany() interators are propagated. - - -What's new in psycopg 2.2.1 ---------------------------- - -* Bux fixes: - - - psycopg now builds again on MS Windows. - - -What's new in psycopg 2.2.0 ---------------------------- - -This is the first release of the new 2.2 series, supporting not just one but -two different ways of executing asynchronous queries, thanks to Jan and Daniele -(with a little help from me and others, but they did 99% of the work so they -deserve their names here in the news.) - -psycopg now supports both classic select() loops and "green" coroutine -libraries. It is all in the documentation, so just point your browser to -doc/html/advanced.html. - -* Other new features: - - - truncate() method for lobjects. - - COPY functions are now a little bit faster. - - All builtin PostgreSQL to Python typecasters are now available from the - psycopg2.extensions module. - - Notifications from the backend are now available right after the execute() - call (before client code needed to call isbusy() to ensure NOTIFY - reception.) - - Better timezone support. - - Lots of documentation updates. - -* Bug fixes: - - - Fixed some gc/refcounting problems. - - Fixed reference leak in NOTIFY reception. - - Fixed problem with PostgreSQL not casting string literals to the correct - types in some situations: psycopg now add an explicit cast to dates, times - and bytea representations. - - Fixed TimestampFromTicks() and TimeFromTicks() for seconds >= 59.5. - - Fixed spurious exception raised when calling C typecasters from Python - ones. diff --git a/ZPsycopgDA/DA.py b/ZPsycopgDA/DA.py index 735bc4b3..f1470c42 100644 --- a/ZPsycopgDA/DA.py +++ b/ZPsycopgDA/DA.py @@ -16,7 +16,7 @@ # their work without bothering about the module dependencies. -ALLOWED_PSYCOPG_VERSIONS = ('2.3.0','2.3.1','2.3.2') +ALLOWED_PSYCOPG_VERSIONS = ('2.4-beta1',) import sys import time diff --git a/setup.py b/setup.py index 1eb2206c..6d471999 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ except ImportError: # Take a look at http://www.python.org/dev/peps/pep-0386/ # for a consistent versioning pattern. -PSYCOPG_VERSION = '2.3.3.dev0' +PSYCOPG_VERSION = '2.4-beta1' version_flags = ['dt', 'dec'] From 8ac5f0070a73081d0e619d2390e3546ba5729483 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 9 Feb 2011 01:11:36 +0100 Subject: [PATCH 76/80] Fields order enforced in composite types adapter --- lib/extras.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/extras.py b/lib/extras.py index 69da5264..137d6fbe 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -841,7 +841,8 @@ SELECT t.oid, attname, atttypid FROM pg_type t JOIN pg_namespace ns ON typnamespace = ns.oid JOIN pg_attribute a ON attrelid = typrelid -WHERE typname = %s and nspname = 'public'; +WHERE typname = %s and nspname = 'public' +ORDER BY attnum; """, (name, )) recs = curs.fetchall() From 7a058403caa67d8393bb4837c8ae722d08050afb Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 9 Feb 2011 02:20:25 +0100 Subject: [PATCH 77/80] Fixed mapping for composite types defined in a schema --- lib/extras.py | 13 ++++++++++--- tests/types_extras.py | 27 +++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/extras.py b/lib/extras.py index 137d6fbe..a25f4201 100644 --- a/lib/extras.py +++ b/lib/extras.py @@ -835,15 +835,22 @@ class CompositeCaster(object): # Store the transaction status of the connection to revert it after use conn_status = conn.status + # Use the correct schema + if '.' in name: + schema, tname = name.split('.', 1) + else: + tname = name + schema = 'public' + # get the type oid and attributes curs.execute("""\ SELECT t.oid, attname, atttypid FROM pg_type t JOIN pg_namespace ns ON typnamespace = ns.oid JOIN pg_attribute a ON attrelid = typrelid -WHERE typname = %s and nspname = 'public' +WHERE typname = %s and nspname = %s ORDER BY attnum; -""", (name, )) +""", (tname, schema)) recs = curs.fetchall() @@ -859,7 +866,7 @@ ORDER BY attnum; type_oid = recs[0][0] type_attrs = [ (r[1], r[2]) for r in recs ] - return CompositeCaster(name, type_oid, type_attrs) + return CompositeCaster(tname, type_oid, type_attrs) def register_composite(name, conn_or_curs, globally=False): """Register a typecaster to convert a composite type into a tuple. diff --git a/tests/types_extras.py b/tests/types_extras.py index bb763384..5a55c6ec 100755 --- a/tests/types_extras.py +++ b/tests/types_extras.py @@ -525,6 +525,24 @@ class AdaptTypeTestCase(unittest.TestCase): conn1.close() conn2.close() + @skip_if_no_composite + def test_composite_namespace(self): + curs = self.conn.cursor() + curs.execute(""" + select nspname from pg_namespace + where nspname = 'typens'; + """) + if not curs.fetchone(): + curs.execute("create schema typens;") + self.conn.commit() + + self._create_type("typens.typens_ii", + [("a", "integer"), ("b", "integer")]) + t = psycopg2.extras.register_composite( + "typens.typens_ii", self.conn) + curs.execute("select (4,8)::typens.typens_ii") + self.assertEqual(curs.fetchone()[0], (4,8)) + def _create_type(self, name, fields): curs = self.conn.cursor() try: @@ -534,11 +552,16 @@ class AdaptTypeTestCase(unittest.TestCase): curs.execute("create type %s as (%s);" % (name, ", ".join(["%s %s" % p for p in fields]))) + if '.' in name: + schema, name = name.split('.') + else: + schema = 'public' + curs.execute("""\ SELECT t.oid FROM pg_type t JOIN pg_namespace ns ON typnamespace = ns.oid - WHERE typname = %s and nspname = 'public'; - """, (name,)) + WHERE typname = %s and nspname = %s; + """, (name, schema)) oid = curs.fetchone()[0] self.conn.commit() return oid From 9c81f6c1868ef7f0c1cf05cf93c7c9d3b0643170 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 10 Feb 2011 01:44:37 +0000 Subject: [PATCH 78/80] Improved adaptation documentation Documented __conform__() and prepare(). --- doc/src/advanced.rst | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/doc/src/advanced.rst b/doc/src/advanced.rst index d4baf419..f7f4cd77 100644 --- a/doc/src/advanced.rst +++ b/doc/src/advanced.rst @@ -89,14 +89,33 @@ by the `psycopg2.extensions.adapt()` function. The `~cursor.execute()` method adapts its arguments to the `~psycopg2.extensions.ISQLQuote` protocol. Objects that conform to this protocol expose a `!getquoted()` method returning the SQL representation -of the object as a string. +of the object as a string (the method must return `!bytes` in Python 3). +Optionally the conform object may expose a +`~psycopg2.extensions.ISQLQuote.prepare()` method. -The easiest way to adapt an object to an SQL string is to register an adapter -function via the `~psycopg2.extensions.register_adapter()` function. The -adapter function must take the value to be adapted as argument and return a -conform object. A convenient object is the `~psycopg2.extensions.AsIs` -wrapper, whose `!getquoted()` result is simply the `!str()`\ ing -conversion of the wrapped object. +There are two basic ways to have a Python object adapted to SQL: + +- the object itself is conform, or knows how to make itself conform. Such + object must expose a `__conform__()` method that will be called with the + protocol object as argument. The object can check that the protocol is + `!ISQLQuote`, in which case it can return `!self` (if the object also + implements `!getquoted()`) or a suitable wrapper object. This option is + viable if you are the author of the object and if the object is specifically + designed for the database (i.e. having Psycopg as a dependency and polluting + its interface with the required methods doesn't bother you). For a simple + example you can take a look to the source code for the + `psycopg2.extras.Inet` object. + +- If implementing the `!ISQLQuote` interface directly in the object is not an + option, you can use an adaptation function, taking the object to be adapted + as argument and returning a conforming object. The adapter must be + registered via the `~psycopg2.extensions.register_adapter()` function. A + simple example wrapper is the `!psycopg2.extras.UUID_adapter` used by the + `~psycopg2.extras.register_uuid()` function. + +A convenient object to write adapters is the `~psycopg2.extensions.AsIs` +wrapper, whose `!getquoted()` result is simply the `!str()`\ ing conversion of +the wrapped object. .. index:: single: Example; Types adaptation From 713b86acdf6a51886ed68dd76fb6e698e3e9e55b Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 10 Feb 2011 02:14:51 +0000 Subject: [PATCH 79/80] Added FAQ point about bytea_output in PostgreSQL 9.0 --- doc/src/faq.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/src/faq.rst b/doc/src/faq.rst index c271b625..9172e351 100644 --- a/doc/src/faq.rst +++ b/doc/src/faq.rst @@ -94,6 +94,18 @@ Psycopg converts :sql:`decimal`\/\ :sql:`numeric` database types into Python `!D documentation. If you find `!psycopg2.extensions.DECIMAL` not avalable, use `!psycopg2._psycopg.DECIMAL` instead. +Transferring binary data from PostgreSQL 9.0 doesn't work. + PostgreSQL 9.0 uses by default `the "hex" format`__ to transfer + :sql:`bytea` data: the format can't be parsed by the libpq 8.4 and + earlier. Three options to solve the problem are: + + - set the bytea_output__ parameter to ``escape`` in the server; + - use ``SET bytea_output TO escape`` in the client before reading binary + data; + - upgrade the libpq library on the client to at least 9.0. + + .. __: http://www.postgresql.org/docs/9.0/static/datatype-binary.html + .. __: http://www.postgresql.org/docs/9.0/static/runtime-config-client.html#GUC-BYTEA-OUTPUT Best practices -------------- From 1a0c494417acf759ccac988a0e5d9397820c3663 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 10 Feb 2011 02:15:44 +0000 Subject: [PATCH 80/80] Document difference of string handling in Python 2/3 --- doc/src/extensions.rst | 5 +++-- doc/src/usage.rst | 49 +++++++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/doc/src/extensions.rst b/doc/src/extensions.rst index 8fee890e..87f3dd0c 100644 --- a/doc/src/extensions.rst +++ b/doc/src/extensions.rst @@ -189,8 +189,9 @@ deal with Python objects adaptation: .. method:: getquoted() Subclasses or other conforming objects should return a valid SQL - string representing the wrapped object. The `!ISQLQuote` - implementation does nothing. + string representing the wrapped object. In Python 3 the SQL must be + returned in a `!bytes` object. The `!ISQLQuote` implementation does + nothing. .. method:: prepare(conn) diff --git a/doc/src/usage.rst b/doc/src/usage.rst index 36bd36be..60a2428c 100644 --- a/doc/src/usage.rst +++ b/doc/src/usage.rst @@ -233,15 +233,37 @@ the SQL string that would be sent to the database. .. index:: pair: Strings; Adaptation single: Unicode; Adaptation + +- String types: `!str`, `!unicode` are converted in SQL string syntax. + `!unicode` objects (`!str` in Python 3) are encoded in the connection + `~connection.encoding` to be sent to the backend: trying to send a character + not supported by the encoding will result in an error. Received data can be + converted either as `!str` or `!unicode`: see :ref:`unicode-handling` for + received, either `!str` or `!unicode` + +.. index:: single: Buffer; Adaptation single: bytea; Adaptation single: Binary string -- String types: `!str`, `!unicode` are converted in SQL string - syntax. `!buffer` is converted in PostgreSQL binary string syntax, - suitable for :sql:`bytea` fields. When reading textual fields, either - `!str` or `!unicode` can be received: see - :ref:`unicode-handling`. +- Binary types: Python types such as `!bytes`, `!bytearray`, `!buffer`, + `!memoryview` are converted in PostgreSQL binary string syntax, suitable for + :sql:`bytea` fields. Received data is returned as `!buffer` (in Python 2) or + `!memoryview` (in Python 3). + + .. warning:: + + PostgreSQL 9 uses by default `a new "hex" format`__ to emit :sql:`bytea` + fields. Unfortunately this format can't be parsed by libpq versions + before 9.0. This means that using a library client with version lesser + than 9.0 to talk with a server 9.0 or later you may have problems + receiving :sql:`bytea` data. To work around this problem you can set the + `bytea_output`__ parameter to ``escape``, either in the server + configuration or in the client session using a query such as ``SET + bytea_output TO escape;`` before trying to receive binary data. + + .. __: http://www.postgresql.org/docs/9.0/static/datatype-binary.html + .. __: http://www.postgresql.org/docs/9.0/static/runtime-config-client.html#GUC-BYTEA-OUTPUT .. index:: single: Adaptation; Date/Time objects @@ -338,8 +360,8 @@ defined on the database connection (the `PostgreSQL encoding`__, available in .. __: http://www.postgresql.org/docs/9.0/static/multibyte.html .. __: http://docs.python.org/library/codecs.html#standard-encodings -When reading data from the database, the strings returned are usually 8 bit -`!str` objects encoded in the database client encoding:: +When reading data from the database, in Python 2 the strings returned are +usually 8 bit `!str` objects encoded in the database client encoding:: >>> print conn.encoding UTF8 @@ -356,9 +378,10 @@ When reading data from the database, the strings returned are usually 8 bit >>> print type(x), repr(x) '\xe0\xe8\xec\xf2\xf9\xa4' -In order to obtain `!unicode` objects instead, it is possible to -register a typecaster so that PostgreSQL textual types are automatically -*decoded* using the current client encoding:: +In Python 3 instead the strings are automatically *decoded* in the connection +`~connection.encoding`, as the `!str` object can represent Unicode characters. +In Python 2 you must register a :ref:`typecaster +` in order to receive `!unicode` objects:: >>> psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, cur) @@ -375,9 +398,9 @@ the connection or globally: see the function .. note:: - If you want to receive uniformly all your database input in Unicode, you - can register the related typecasters globally as soon as Psycopg is - imported:: + In Python 2, if you want to receive uniformly all your database input in + Unicode, you can register the related typecasters globally as soon as + Psycopg is imported:: import psycopg2 import psycopg2.extensions