Aggressive threading on fetch (and a new bug!)

INSTALL

@@ -7,7 +7,7 @@ Compiling and installing psycopg
    libpq will leak memory.
 
 While psycopg 1.x used autoconf for its build process psycopg 2 switched to
-the more pythoning setup.py. Before building psycopg look at setup.cfg file
+the more pythonic setup.py. Before building psycopg look at setup.cfg file
 and change any settings to follow your system (or taste); then:
 
     python setup.py build
@@ -53,7 +53,8 @@ You can compile psycopg under Windows platform with mingw32
 Dev-C++ (http://www.bloodshed.net/devcpp.html) and Code::Blocks
 (http://www.codeblocks.org). gcc binaries should be in your PATH.
 
-You need a PostgreSQL with include and libary files installed. At least v8.0 is required.
+You need a PostgreSQL with include and libary files installed. At least v8.0
+is required.
 
 First you need to create a libpython2X.a as described in 
 http://starship.python.net/crew/kernr/mingw32/Notes.html. Then run:

psycopg/adapter_datetime.c

@@ -128,8 +128,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type)
 {
     Dprintf("pydatetime_setup: init datetime object at %p, refcnt = "
         FORMAT_CODE_PY_SSIZE_T,
-        self, ((PyObject *)self)->ob_refcnt
+        self, ((PyObject *)self)->ob_refcnt);
-      );
 
     self->type = type;
     self->wrapped = obj;
@@ -137,8 +136,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type)
 
     Dprintf("pydatetime_setup: good pydatetime object at %p, refcnt = "
         FORMAT_CODE_PY_SSIZE_T,
-        self, ((PyObject *)self)->ob_refcnt
+        self, ((PyObject *)self)->ob_refcnt);
-      );
     return 0;
 }
 
@@ -414,6 +412,7 @@ psyco_TimestampFromTicks(PyObject *self, PyObject *args)
                              (double)tm.tm_sec + ticks,
                              pyPsycopgTzLOCAL);
         if (args) {
+/*            Dprintf("psyco_TimestampFromTicks: args->refcnt = %d", args->ob_refcnt);*/
             res = psyco_Timestamp(self, args);
             Py_DECREF(args);
         }

psycopg/pqpath.c

@@ -359,6 +359,7 @@ pq_abort(connectionObject *conn)
 int
 pq_is_busy(connectionObject *conn)
 {
+    int res;
     PGnotify *pgn;
 
     Dprintf("pq_is_busy: consuming input");
@@ -374,8 +375,6 @@ pq_is_busy(connectionObject *conn)
         return -1;
     }
 
-    pthread_mutex_unlock(&(conn->lock));
-    Py_END_ALLOW_THREADS;
 
     /* now check for notifies */
     while ((pgn = PQnotifies(conn->pgconn)) != NULL) {
@@ -384,14 +383,21 @@ pq_is_busy(connectionObject *conn)
         Dprintf("curs_is_busy: got NOTIFY from pid %d, msg = %s",
                 (int) pgn->be_pid, pgn->relname);
 
+        Py_BLOCK_THREADS;
         notify = PyTuple_New(2);
         PyTuple_SET_ITEM(notify, 0, PyInt_FromLong((long)pgn->be_pid));
         PyTuple_SET_ITEM(notify, 1, PyString_FromString(pgn->relname));
         PyList_Append(conn->notifies, notify);
+        Py_UNBLOCK_THREADS;
         free(pgn);
     }
 
-    return PQisBusy(conn->pgconn);
+    res = PQisBusy(conn->pgconn);
+    
+    pthread_mutex_unlock(&(conn->lock));
+    Py_END_ALLOW_THREADS;
+
+    return res;
 }
 
 /* pq_execute - execute a query, possibly asyncronously
@@ -497,23 +503,32 @@ static void
 _pq_fetch_tuples(cursorObject *curs)
 {
     int i, *dsize = NULL;
+    int pgnfields;
+    int pgbintuples;
 
-    int pgnfields = PQnfields(curs->pgres);
+    Py_BEGIN_ALLOW_THREADS;
-    int pgbintuples = PQbinaryTuples(curs->pgres);
+    pthread_mutex_lock(&(curs->conn->lock));
+    
+    pgnfields = PQnfields(curs->pgres);
+    pgbintuples = PQbinaryTuples(curs->pgres);
 
     curs->notuples = 0;
 
     /* create the tuple for description and typecasting */
+    Py_BLOCK_THREADS;
     Py_XDECREF(curs->description);
     Py_XDECREF(curs->casts);    
     curs->description = PyTuple_New(pgnfields);
     curs->casts = PyTuple_New(pgnfields);
     curs->columns = pgnfields;
+    Py_UNBLOCK_THREADS;
 
     /* calculate the display size for each column (cpu intensive, can be
        switched off at configuration time) */
 #ifdef PSYCOPG_DISPLAY_SIZE
+    Py_BLOCK_THREADS;
     dsize = (int *)PyMem_Malloc(pgnfields * sizeof(int));
+    Py_UNBLOCK_THREADS;
     if (dsize != NULL) {
         int j, len;
         for (i=0; i < pgnfields; i++) {
@@ -534,6 +549,7 @@ _pq_fetch_tuples(cursorObject *curs)
         int fsize = PQfsize(curs->pgres, i);
         int fmod =  PQfmod(curs->pgres, i);
 
+        Py_BLOCK_THREADS;
         PyObject *dtitem = PyTuple_New(7);
         PyObject *type = PyInt_FromLong(ftype);
         PyObject *cast = NULL;
@@ -622,9 +638,18 @@ _pq_fetch_tuples(cursorObject *curs)
         /* 6/ FIXME: null_ok??? */
         Py_INCREF(Py_None);
         PyTuple_SET_ITEM(dtitem, 6, Py_None);
+    
+        Py_UNBLOCK_THREADS;    
     }
 
-    if (dsize) PyMem_Free(dsize);
+    if (dsize) {
+        Py_BLOCK_THREADS;
+        PyMem_Free(dsize);
+        Py_UNBLOCK_THREADS;
+   }
+   
+    pthread_mutex_unlock(&(curs->conn->lock));
+    Py_END_ALLOW_THREADS;
 }
 
 #ifdef HAVE_PQPROTOCOL3
@@ -856,9 +881,15 @@ pq_fetch(cursorObject *curs)
             Py_END_ALLOW_THREADS;
         }
 
+        Py_BEGIN_ALLOW_THREADS;
+        pthread_mutex_lock(&(curs->conn->lock));
+
         Dprintf("pq_fetch: data is probably ready");
         IFCLEARPGRES(curs->pgres);
         curs->pgres = PQgetResult(curs->conn->pgconn);
+
+        pthread_mutex_unlock(&(curs->conn->lock));
+        Py_END_ALLOW_THREADS;
     }
 
     /* check for PGRES_FATAL_ERROR result */

psycopg2.mdp

@@ -0,0 +1,136 @@
+<Project name="psycopg2" fileversion="2.0" language="C" ctype="CProject">
+  <Configurations active="Debug">
+    <Configuration name="Debug" ctype="CProjectConfiguration">
+      <CustomCommands>
+        <Command type="Build" command="/usr/bin/python setup.py build" workingdir="${ProjectDir}" />
+        <Command type="Clean" command="/usr/bin/python setup.py clean -a" workingdir="${ProjectDir}" />
+      </CustomCommands>
+      <Output directory="./" output="psycopg2" />
+      <Build debugmode="True" target="Bin" />
+      <Execution runwithwarnings="True" consolepause="True" />
+      <CodeGeneration WarningLevel="Normal" OptimizationLevel="0" ExtraCompilerArguments="" ExtraLinkerArguments="" DefineSymbols="DEBUG MONODEVELOP" ctype="CCompilationParameters" />
+    </Configuration>
+    <Configuration name="Release" ctype="CProjectConfiguration">
+      <Output directory="./bin/Release" output="psycopg2" />
+      <Build debugmode="False" target="Bin" />
+      <Execution runwithwarnings="True" consolepause="True" />
+      <CodeGeneration WarningLevel="Normal" OptimizationLevel="3" ExtraCompilerArguments="" ExtraLinkerArguments="" DefineSymbols="MONODEVELOP" ctype="CCompilationParameters" />
+    </Configuration>
+  </Configurations>
+  <Contents>
+    <File name="./AUTHORS" subtype="Code" buildaction="Nothing" />
+    <File name="./ChangeLog" subtype="Code" buildaction="Nothing" />
+    <File name="./INSTALL" subtype="Code" buildaction="Nothing" />
+    <File name="./LICENSE" subtype="Code" buildaction="Nothing" />
+    <File name="./MANIFEST.in" subtype="Code" buildaction="Nothing" />
+    <File name="./NEWS" subtype="Code" buildaction="Nothing" />
+    <File name="./README" subtype="Code" buildaction="Nothing" />
+    <File name="./setup.cfg" subtype="Code" buildaction="Nothing" />
+    <File name="./setup.py" subtype="Code" buildaction="Nothing" />
+    <File name="./doc/async.txt" subtype="Code" buildaction="Nothing" />
+    <File name="./doc/extensions.rst" subtype="Code" buildaction="Nothing" />
+    <File name="./doc/HACKING" subtype="Code" buildaction="Nothing" />
+    <File name="./doc/SUCCESS" subtype="Code" buildaction="Nothing" />
+    <File name="./doc/TODO" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/binary.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/copy_from.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/copy_to.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/cursor.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/dialtone.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/dict.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/dt.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/encoding.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/fetch.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/lastrowid.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/mogrify.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/myfirstrecipe.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/notify.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/simple.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/somehackers.jpg" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/threads.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/typecast.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/tz.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/usercast.py" subtype="Code" buildaction="Nothing" />
+    <File name="./examples/whereareyou.jpg" subtype="Code" buildaction="Nothing" />
+    <File name="./lib/errorcodes.py" subtype="Code" buildaction="Nothing" />
+    <File name="./lib/extensions.py" subtype="Code" buildaction="Nothing" />
+    <File name="./lib/extras.py" subtype="Code" buildaction="Nothing" />
+    <File name="./lib/__init__.py" subtype="Code" buildaction="Nothing" />
+    <File name="./lib/pool.py" subtype="Code" buildaction="Nothing" />
+    <File name="./lib/psycopg1.py" subtype="Code" buildaction="Nothing" />
+    <File name="./lib/tz.py" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/adapter_asis.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/adapter_asis.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/adapter_binary.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/adapter_binary.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/adapter_datetime.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/adapter_datetime.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/adapter_list.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/adapter_list.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/adapter_mxdatetime.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/adapter_mxdatetime.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/adapter_pboolean.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/adapter_pboolean.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/adapter_qstring.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/adapter_qstring.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/config.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/connection.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/connection_int.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/connection_type.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/cursor.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/cursor_int.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/cursor_type.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/microprotocols.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/microprotocols.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/microprotocols_proto.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/microprotocols_proto.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/pgtypes.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/pgversion.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/pqpath.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/pqpath.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/psycopg.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/psycopgmodule.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/python.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/typecast.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/typecast.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/typecast_array.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/typecast_basic.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/typecast_binary.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/typecast_binary.h" subtype="Code" buildaction="Nothing" />
+    <File name="./psycopg/typecast_builtins.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/typecast_datetime.c" subtype="Code" buildaction="Compile" />
+    <File name="./psycopg/typecast_mxdatetime.c" subtype="Code" buildaction="Compile" />
+    <File name="./scripts/buildtypes.py" subtype="Code" buildaction="Nothing" />
+    <File name="./scripts/ext2html.py" subtype="Code" buildaction="Nothing" />
+    <File name="./scripts/makedocs.py" subtype="Code" buildaction="Nothing" />
+    <File name="./scripts/maketypes.sh" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/dtml/add.dtml" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/dtml/browse.dtml" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/dtml/edit.dtml" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/dtml/table_info.dtml" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/bin.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/date.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/datetime.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/field.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/float.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/int.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/stable.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/table.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/text.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/time.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/view.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/icons/what.gif" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/DA.py" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/db.py" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/__init__.py" subtype="Code" buildaction="Nothing" />
+    <File name="./ZPsycopgDA/pool.py" subtype="Code" buildaction="Nothing" />
+    <File name="./tests/dbapi20.py" subtype="Code" buildaction="Nothing" />
+    <File name="./tests/extras_dictcursor.py" subtype="Code" buildaction="Nothing" />
+    <File name="./tests/__init__.py" subtype="Code" buildaction="Nothing" />
+    <File name="./tests/test_psycopg2_dbapi20.py" subtype="Code" buildaction="Nothing" />
+    <File name="./tests/test_transaction.py" subtype="Code" buildaction="Nothing" />
+    <File name="./tests/types_basic.py" subtype="Code" buildaction="Nothing" />
+  </Contents>
+  <compiler ctype="GccCompiler" />
+  <MonoDevelop.ChangeLogAddIn.ChangeLogInfo policy="UpdateNearestChangeLog" />
+</Project>

psycopg2.mds

@@ -0,0 +1,16 @@
+<Combine name="psycopg2" fileversion="2.0">
+  <Configurations active="Debug">
+    <Configuration name="Debug" ctype="CombineConfiguration">
+      <Entry build="True" name="psycopg2" configuration="Debug" />
+    </Configuration>
+    <Configuration name="Release" ctype="CombineConfiguration">
+      <Entry build="True" name="psycopg2" configuration="Release" />
+    </Configuration>
+  </Configurations>
+  <StartMode startupentry="psycopg2" single="True">
+    <Execute type="None" entry="psycopg2" />
+  </StartMode>
+  <Entries>
+    <Entry filename="./psycopg2.mdp" />
+  </Entries>
+</Combine>

psycopg2.usertasks

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<ArrayOfUserTask xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />

setup.cfg

@@ -1,5 +1,5 @@
 [build_ext]
-define=PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
+define=PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3,PSYCOPG_DEBUG
 # PSYCOPG_EXTENSIONS enables extensions to PEP-249 (you really want this)
 # PSYCOPG_DISPLAY_SIZE enable display size calculation (a little slower)
 # HAVE_PQFREEMEM should be defined on PostgreSQL >= 7.4