psycopg/psycopg2
1
1
Fork 0
mirror of https://github.com/psycopg/psycopg2.git synced 2025-01-31 09:24:07 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fixed problem with lots of columns in COPY operations

Browse Source
This commit is contained in:
Federico Di Gregorio 2009-08-09 15:13:42 +02:00
parent e5a696ad58
commit 4c3e2ad94b
4 changed files with 19 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
ChangeLog
View File

@ -1,3 +1,9 @@
2009-08-08 Federico Di Gregorio <fog@initd.org>
* psycopg/cursor_type.c: unified size macro definitions in COPY TO
and COPY FROM operations: now the buffer for column names is 8192
bytes that should be enough even for very large tables.
2009-05-19 Federico Di Gregorio <fog@initd.org> 2009-05-19 Federico Di Gregorio <fog@initd.org>
* Applied patch from Robert Munro to fix version check * Applied patch from Robert Munro to fix version check

2
psycopg/cursor.h
View File

@ -64,7 +64,7 @@ typedef struct {
PyObject *copyfile; /* file-like used during COPY TO/FROM ops */ PyObject *copyfile; /* file-like used during COPY TO/FROM ops */
Py_ssize_t copysize; /* size of the copy buffer during COPY TO/FROM ops */ Py_ssize_t copysize; /* size of the copy buffer during COPY TO/FROM ops */
#define DEFAULT_COPYSIZE 16384 #define DEFAULT_COPYSIZE 16384
#define DEFAULT_COPYBUFF 1024 #define DEFAULT_COPYBUFF 8132
PyObject *tuple_factory; /* factory for result tuples */ PyObject *tuple_factory; /* factory for result tuples */
PyObject *tzinfo_factory; /* factory for tzinfo objects */ PyObject *tzinfo_factory; /* factory for tzinfo objects */

22
psycopg/cursor_type.c
View File

@ -1127,8 +1127,6 @@ psyco_curs_scroll(cursorObject *self, PyObject *args, PyObject *kwargs)
#ifdef PSYCOPG_EXTENSIONS #ifdef PSYCOPG_EXTENSIONS
#define COPY_BUFFER_SIZE 8192
static int _psyco_curs_copy_columns(PyObject *columns, char *columnlist) static int _psyco_curs_copy_columns(PyObject *columns, char *columnlist)
{ {
PyObject *col, *coliter; PyObject *col, *coliter;
@ -1201,7 +1199,7 @@ _psyco_curs_has_read_check(PyObject* o, void* var)
static PyObject * static PyObject *
psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs) psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs)
{ {
char query_buffer[COPY_BUFFER_SIZE]; char query_buffer[DEFAULT_COPYBUFF];
Py_ssize_t query_size; Py_ssize_t query_size;
char *query; char *query;
const char *table_name; const char *table_name;
@ -1241,10 +1239,10 @@ psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs)
PyErr_NoMemory(); PyErr_NoMemory();
return NULL; return NULL;
} }
query_size = PyOS_snprintf(query, COPY_BUFFER_SIZE, query_size = PyOS_snprintf(query, DEFAULT_COPYBUFF,
"COPY %s%s FROM stdin WITH DELIMITER AS %s NULL AS %s", "COPY %s%s FROM stdin WITH DELIMITER AS %s NULL AS %s",
table_name, columnlist, quoted_delimiter, quoted_null); table_name, columnlist, quoted_delimiter, quoted_null);
if (query_size >= COPY_BUFFER_SIZE) { if (query_size >= DEFAULT_COPYBUFF) {
/* Got truncated, allocate dynamically */ /* Got truncated, allocate dynamically */
query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char)); query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char));
PyOS_snprintf(query, query_size + 1, PyOS_snprintf(query, query_size + 1,
@ -1254,10 +1252,10 @@ psyco_curs_copy_from(cursorObject *self, PyObject *args, PyObject *kwargs)
PyMem_Free(quoted_null); PyMem_Free(quoted_null);
} }
else { else {
query_size = PyOS_snprintf(query, COPY_BUFFER_SIZE, query_size = PyOS_snprintf(query, DEFAULT_COPYBUFF,
"COPY %s%s FROM stdin WITH DELIMITER AS %s", "COPY %s%s FROM stdin WITH DELIMITER AS %s",
table_name, columnlist, quoted_delimiter); table_name, columnlist, quoted_delimiter);
if (query_size >= COPY_BUFFER_SIZE) { if (query_size >= DEFAULT_COPYBUFF) {
/* Got truncated, allocate dynamically */ /* Got truncated, allocate dynamically */
query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char)); query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char));
PyOS_snprintf(query, query_size + 1, PyOS_snprintf(query, query_size + 1,
@ -1310,7 +1308,7 @@ static PyObject *
psyco_curs_copy_to(cursorObject *self, PyObject *args, PyObject *kwargs) psyco_curs_copy_to(cursorObject *self, PyObject *args, PyObject *kwargs)
{ {
char *query = NULL; char *query = NULL;
char query_buffer[COPY_BUFFER_SIZE]; char query_buffer[DEFAULT_COPYBUFF];
size_t query_size; size_t query_size;
char columnlist[DEFAULT_COPYBUFF]; char columnlist[DEFAULT_COPYBUFF];
const char *table_name; const char *table_name;
@ -1344,10 +1342,10 @@ psyco_curs_copy_to(cursorObject *self, PyObject *args, PyObject *kwargs)
PyErr_NoMemory(); PyErr_NoMemory();
return NULL; return NULL;
} }
query_size = PyOS_snprintf(query, COPY_BUFFER_SIZE, query_size = PyOS_snprintf(query, DEFAULT_COPYBUFF,
"COPY %s%s TO stdout WITH DELIMITER AS %s" "COPY %s%s TO stdout WITH DELIMITER AS %s"
" NULL AS %s", table_name, columnlist, quoted_delimiter, quoted_null); " NULL AS %s", table_name, columnlist, quoted_delimiter, quoted_null);
if (query_size >= COPY_BUFFER_SIZE) { if (query_size >= DEFAULT_COPYBUFF) {
/* Got truncated, allocate dynamically */ /* Got truncated, allocate dynamically */
query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char)); query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char));
PyOS_snprintf(query, query_size + 1, PyOS_snprintf(query, query_size + 1,
@ -1357,10 +1355,10 @@ psyco_curs_copy_to(cursorObject *self, PyObject *args, PyObject *kwargs)
PyMem_Free(quoted_null); PyMem_Free(quoted_null);
} }
else { else {
query_size = PyOS_snprintf(query, COPY_BUFFER_SIZE, query_size = PyOS_snprintf(query, DEFAULT_COPYBUFF,
"COPY %s%s TO stdout WITH DELIMITER AS %s", "COPY %s%s TO stdout WITH DELIMITER AS %s",
table_name, columnlist, quoted_delimiter); table_name, columnlist, quoted_delimiter);
if (query_size >= COPY_BUFFER_SIZE) { if (query_size >= DEFAULT_COPYBUFF) {
/* Got truncated, allocate dynamically */ /* Got truncated, allocate dynamically */
query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char)); query = (char *)PyMem_Malloc((query_size + 1) * sizeof(char));
PyOS_snprintf(query, query_size + 1, PyOS_snprintf(query, query_size + 1,

3
psycopg2.cproj
View File

@ -175,7 +175,8 @@
<ChangeLogPolicy UpdateMode="Nearest" VcsIntegration="None" inheritsSet="Mono"> <ChangeLogPolicy UpdateMode="Nearest" VcsIntegration="None" inheritsSet="Mono">
<MessageStyle FileSeparator=", " LineAlign="0" /> <MessageStyle FileSeparator=", " LineAlign="0" />
</ChangeLogPolicy> </ChangeLogPolicy>
<TextStylePolicy FileWidth="144" TabWidth="4" TabsToSpaces="True" NoTabsAfterNonTabs="False" RemoveTrailingWhitespace="True" EolMarker="Native" /> <TextStylePolicy FileWidth="144" RemoveTrailingWhitespace="True" inheritsSet="VisualStudio" inheritsScope="text/plain" />
<TextStylePolicy NoTabsAfterNonTabs="True" RemoveTrailingWhitespace="True" inheritsSet="Mono" inheritsScope="text/plain" scope="text/x-changelog" />
</Policies> </Policies>
</Properties> </Properties>
</MonoDevelop> </MonoDevelop>