Aggressive threading on fetch (and a new bug!)

This commit is contained in:
Federico Di Gregorio 2007-09-19 13:39:48 +00:00
parent c9e701baa9
commit 07892b29a6
7 changed files with 201 additions and 16 deletions

View File

@ -7,7 +7,7 @@ Compiling and installing psycopg
libpq will leak memory.
While psycopg 1.x used autoconf for its build process psycopg 2 switched to
the more pythoning setup.py. Before building psycopg look at setup.cfg file
the more pythonic setup.py. Before building psycopg look at setup.cfg file
and change any settings to follow your system (or taste); then:
python setup.py build
@ -53,7 +53,8 @@ You can compile psycopg under Windows platform with mingw32
Dev-C++ (http://www.bloodshed.net/devcpp.html) and Code::Blocks
(http://www.codeblocks.org). gcc binaries should be in your PATH.
You need a PostgreSQL with include and libary files installed. At least v8.0 is required.
You need a PostgreSQL with include and libary files installed. At least v8.0
is required.
First you need to create a libpython2X.a as described in
http://starship.python.net/crew/kernr/mingw32/Notes.html. Then run:

View File

@ -64,7 +64,7 @@ pydatetime_str(pydatetimeObject *self)
else {
PyDateTime_Delta *obj = (PyDateTime_Delta*)self->wrapped;
char buffer[8];
char buffer[8];
int i;
int a = obj->microseconds;
@ -128,8 +128,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type)
{
Dprintf("pydatetime_setup: init datetime object at %p, refcnt = "
FORMAT_CODE_PY_SSIZE_T,
self, ((PyObject *)self)->ob_refcnt
);
self, ((PyObject *)self)->ob_refcnt);
self->type = type;
self->wrapped = obj;
@ -137,8 +136,7 @@ pydatetime_setup(pydatetimeObject *self, PyObject *obj, int type)
Dprintf("pydatetime_setup: good pydatetime object at %p, refcnt = "
FORMAT_CODE_PY_SSIZE_T,
self, ((PyObject *)self)->ob_refcnt
);
self, ((PyObject *)self)->ob_refcnt);
return 0;
}
@ -411,9 +409,10 @@ psyco_TimestampFromTicks(PyObject *self, PyObject *args)
args = Py_BuildValue("iiiiidO",
tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday,
tm.tm_hour, tm.tm_min,
(double)tm.tm_sec + ticks,
(double)tm.tm_sec + ticks,
pyPsycopgTzLOCAL);
if (args) {
/* Dprintf("psyco_TimestampFromTicks: args->refcnt = %d", args->ob_refcnt);*/
res = psyco_Timestamp(self, args);
Py_DECREF(args);
}

View File

@ -359,6 +359,7 @@ pq_abort(connectionObject *conn)
int
pq_is_busy(connectionObject *conn)
{
int res;
PGnotify *pgn;
Dprintf("pq_is_busy: consuming input");
@ -374,8 +375,6 @@ pq_is_busy(connectionObject *conn)
return -1;
}
pthread_mutex_unlock(&(conn->lock));
Py_END_ALLOW_THREADS;
/* now check for notifies */
while ((pgn = PQnotifies(conn->pgconn)) != NULL) {
@ -384,14 +383,21 @@ pq_is_busy(connectionObject *conn)
Dprintf("curs_is_busy: got NOTIFY from pid %d, msg = %s",
(int) pgn->be_pid, pgn->relname);
Py_BLOCK_THREADS;
notify = PyTuple_New(2);
PyTuple_SET_ITEM(notify, 0, PyInt_FromLong((long)pgn->be_pid));
PyTuple_SET_ITEM(notify, 1, PyString_FromString(pgn->relname));
PyList_Append(conn->notifies, notify);
Py_UNBLOCK_THREADS;
free(pgn);
}
return PQisBusy(conn->pgconn);
res = PQisBusy(conn->pgconn);
pthread_mutex_unlock(&(conn->lock));
Py_END_ALLOW_THREADS;
return res;
}
/* pq_execute - execute a query, possibly asyncronously
@ -497,23 +503,32 @@ static void
_pq_fetch_tuples(cursorObject *curs)
{
int i, *dsize = NULL;
int pgnfields;
int pgbintuples;
int pgnfields = PQnfields(curs->pgres);
int pgbintuples = PQbinaryTuples(curs->pgres);
Py_BEGIN_ALLOW_THREADS;
pthread_mutex_lock(&(curs->conn->lock));
pgnfields = PQnfields(curs->pgres);
pgbintuples = PQbinaryTuples(curs->pgres);
curs->notuples = 0;
/* create the tuple for description and typecasting */
Py_BLOCK_THREADS;
Py_XDECREF(curs->description);
Py_XDECREF(curs->casts);
Py_XDECREF(curs->casts);
curs->description = PyTuple_New(pgnfields);
curs->casts = PyTuple_New(pgnfields);
curs->columns = pgnfields;
Py_UNBLOCK_THREADS;
/* calculate the display size for each column (cpu intensive, can be
switched off at configuration time) */
#ifdef PSYCOPG_DISPLAY_SIZE
Py_BLOCK_THREADS;
dsize = (int *)PyMem_Malloc(pgnfields * sizeof(int));
Py_UNBLOCK_THREADS;
if (dsize != NULL) {
int j, len;
for (i=0; i < pgnfields; i++) {
@ -534,6 +549,7 @@ _pq_fetch_tuples(cursorObject *curs)
int fsize = PQfsize(curs->pgres, i);
int fmod = PQfmod(curs->pgres, i);
Py_BLOCK_THREADS;
PyObject *dtitem = PyTuple_New(7);
PyObject *type = PyInt_FromLong(ftype);
PyObject *cast = NULL;
@ -622,9 +638,18 @@ _pq_fetch_tuples(cursorObject *curs)
/* 6/ FIXME: null_ok??? */
Py_INCREF(Py_None);
PyTuple_SET_ITEM(dtitem, 6, Py_None);
Py_UNBLOCK_THREADS;
}
if (dsize) PyMem_Free(dsize);
if (dsize) {
Py_BLOCK_THREADS;
PyMem_Free(dsize);
Py_UNBLOCK_THREADS;
}
pthread_mutex_unlock(&(curs->conn->lock));
Py_END_ALLOW_THREADS;
}
#ifdef HAVE_PQPROTOCOL3
@ -856,9 +881,15 @@ pq_fetch(cursorObject *curs)
Py_END_ALLOW_THREADS;
}
Py_BEGIN_ALLOW_THREADS;
pthread_mutex_lock(&(curs->conn->lock));
Dprintf("pq_fetch: data is probably ready");
IFCLEARPGRES(curs->pgres);
curs->pgres = PQgetResult(curs->conn->pgconn);
pthread_mutex_unlock(&(curs->conn->lock));
Py_END_ALLOW_THREADS;
}
/* check for PGRES_FATAL_ERROR result */

136
psycopg2.mdp Normal file
View File

@ -0,0 +1,136 @@
<Project name="psycopg2" fileversion="2.0" language="C" ctype="CProject">
<Configurations active="Debug">
<Configuration name="Debug" ctype="CProjectConfiguration">
<CustomCommands>
<Command type="Build" command="/usr/bin/python setup.py build" workingdir="${ProjectDir}" />
<Command type="Clean" command="/usr/bin/python setup.py clean -a" workingdir="${ProjectDir}" />
</CustomCommands>
<Output directory="./" output="psycopg2" />
<Build debugmode="True" target="Bin" />
<Execution runwithwarnings="True" consolepause="True" />
<CodeGeneration WarningLevel="Normal" OptimizationLevel="0" ExtraCompilerArguments="" ExtraLinkerArguments="" DefineSymbols="DEBUG MONODEVELOP" ctype="CCompilationParameters" />
</Configuration>
<Configuration name="Release" ctype="CProjectConfiguration">
<Output directory="./bin/Release" output="psycopg2" />
<Build debugmode="False" target="Bin" />
<Execution runwithwarnings="True" consolepause="True" />
<CodeGeneration WarningLevel="Normal" OptimizationLevel="3" ExtraCompilerArguments="" ExtraLinkerArguments="" DefineSymbols="MONODEVELOP" ctype="CCompilationParameters" />
</Configuration>
</Configurations>
<Contents>
<File name="./AUTHORS" subtype="Code" buildaction="Nothing" />
<File name="./ChangeLog" subtype="Code" buildaction="Nothing" />
<File name="./INSTALL" subtype="Code" buildaction="Nothing" />
<File name="./LICENSE" subtype="Code" buildaction="Nothing" />
<File name="./MANIFEST.in" subtype="Code" buildaction="Nothing" />
<File name="./NEWS" subtype="Code" buildaction="Nothing" />
<File name="./README" subtype="Code" buildaction="Nothing" />
<File name="./setup.cfg" subtype="Code" buildaction="Nothing" />
<File name="./setup.py" subtype="Code" buildaction="Nothing" />
<File name="./doc/async.txt" subtype="Code" buildaction="Nothing" />
<File name="./doc/extensions.rst" subtype="Code" buildaction="Nothing" />
<File name="./doc/HACKING" subtype="Code" buildaction="Nothing" />
<File name="./doc/SUCCESS" subtype="Code" buildaction="Nothing" />
<File name="./doc/TODO" subtype="Code" buildaction="Nothing" />
<File name="./examples/binary.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/copy_from.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/copy_to.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/cursor.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/dialtone.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/dict.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/dt.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/encoding.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/fetch.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/lastrowid.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/mogrify.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/myfirstrecipe.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/notify.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/simple.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/somehackers.jpg" subtype="Code" buildaction="Nothing" />
<File name="./examples/threads.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/typecast.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/tz.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/usercast.py" subtype="Code" buildaction="Nothing" />
<File name="./examples/whereareyou.jpg" subtype="Code" buildaction="Nothing" />
<File name="./lib/errorcodes.py" subtype="Code" buildaction="Nothing" />
<File name="./lib/extensions.py" subtype="Code" buildaction="Nothing" />
<File name="./lib/extras.py" subtype="Code" buildaction="Nothing" />
<File name="./lib/__init__.py" subtype="Code" buildaction="Nothing" />
<File name="./lib/pool.py" subtype="Code" buildaction="Nothing" />
<File name="./lib/psycopg1.py" subtype="Code" buildaction="Nothing" />
<File name="./lib/tz.py" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/adapter_asis.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/adapter_asis.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/adapter_binary.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/adapter_binary.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/adapter_datetime.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/adapter_datetime.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/adapter_list.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/adapter_list.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/adapter_mxdatetime.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/adapter_mxdatetime.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/adapter_pboolean.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/adapter_pboolean.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/adapter_qstring.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/adapter_qstring.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/config.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/connection.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/connection_int.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/connection_type.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/cursor.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/cursor_int.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/cursor_type.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/microprotocols.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/microprotocols.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/microprotocols_proto.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/microprotocols_proto.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/pgtypes.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/pgversion.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/pqpath.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/pqpath.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/psycopg.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/psycopgmodule.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/python.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/typecast.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/typecast.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/typecast_array.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/typecast_basic.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/typecast_binary.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/typecast_binary.h" subtype="Code" buildaction="Nothing" />
<File name="./psycopg/typecast_builtins.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/typecast_datetime.c" subtype="Code" buildaction="Compile" />
<File name="./psycopg/typecast_mxdatetime.c" subtype="Code" buildaction="Compile" />
<File name="./scripts/buildtypes.py" subtype="Code" buildaction="Nothing" />
<File name="./scripts/ext2html.py" subtype="Code" buildaction="Nothing" />
<File name="./scripts/makedocs.py" subtype="Code" buildaction="Nothing" />
<File name="./scripts/maketypes.sh" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/dtml/add.dtml" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/dtml/browse.dtml" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/dtml/edit.dtml" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/dtml/table_info.dtml" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/bin.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/date.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/datetime.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/field.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/float.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/int.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/stable.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/table.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/text.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/time.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/view.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/icons/what.gif" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/DA.py" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/db.py" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/__init__.py" subtype="Code" buildaction="Nothing" />
<File name="./ZPsycopgDA/pool.py" subtype="Code" buildaction="Nothing" />
<File name="./tests/dbapi20.py" subtype="Code" buildaction="Nothing" />
<File name="./tests/extras_dictcursor.py" subtype="Code" buildaction="Nothing" />
<File name="./tests/__init__.py" subtype="Code" buildaction="Nothing" />
<File name="./tests/test_psycopg2_dbapi20.py" subtype="Code" buildaction="Nothing" />
<File name="./tests/test_transaction.py" subtype="Code" buildaction="Nothing" />
<File name="./tests/types_basic.py" subtype="Code" buildaction="Nothing" />
</Contents>
<compiler ctype="GccCompiler" />
<MonoDevelop.ChangeLogAddIn.ChangeLogInfo policy="UpdateNearestChangeLog" />
</Project>

16
psycopg2.mds Normal file
View File

@ -0,0 +1,16 @@
<Combine name="psycopg2" fileversion="2.0">
<Configurations active="Debug">
<Configuration name="Debug" ctype="CombineConfiguration">
<Entry build="True" name="psycopg2" configuration="Debug" />
</Configuration>
<Configuration name="Release" ctype="CombineConfiguration">
<Entry build="True" name="psycopg2" configuration="Release" />
</Configuration>
</Configurations>
<StartMode startupentry="psycopg2" single="True">
<Execute type="None" entry="psycopg2" />
</StartMode>
<Entries>
<Entry filename="./psycopg2.mdp" />
</Entries>
</Combine>

2
psycopg2.usertasks Normal file
View File

@ -0,0 +1,2 @@
<?xml version="1.0" encoding="utf-8"?>
<ArrayOfUserTask xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" />

View File

@ -1,5 +1,5 @@
[build_ext]
define=PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3
define=PSYCOPG_EXTENSIONS,PSYCOPG_DISPLAY_SIZE,PSYCOPG_NEW_BOOLEAN,HAVE_PQFREEMEM,HAVE_PQPROTOCOL3,PSYCOPG_DEBUG
# PSYCOPG_EXTENSIONS enables extensions to PEP-249 (you really want this)
# PSYCOPG_DISPLAY_SIZE enable display size calculation (a little slower)
# HAVE_PQFREEMEM should be defined on PostgreSQL >= 7.4