Array tokenization seems working.

ChangeLog

@@ -17,8 +17,17 @@
 
 	* scripts/buildtypes.py: new version to include array data.
 
+2005-03-15  Federico Di Gregorio  <fog@debian.org>
+
+	* lib/extensions.py: Added AsIs import.
+
 2005-03-12  Federico Di Gregorio  <fog@debian.org>
 
+	* psycopg/cursor.h: removed "qattr", not used anymore and added
+	"cast", holding the typecaster currently in use.
+
+	* Release 1.99.13.
+
 	* psycopg/cursor_type.c (psyco_curs_executemany): implemented as a
 	wrapper to extract python arguments and then call
 	_psyco_curs_execute().

MANIFEST.in

@@ -1,8 +1,8 @@
 recursive-include psycopg *.c *.h
 recursive-include lib *.py
+recursive-include tests *.py
 recursive-include ZPsycopgDA *.py *.gif *.dtml
 recursive-include examples *.py somehackers.jpg whereareyou.jpg
-#recursive-include tests *.py
 recursive-include doc TODO HACKING SUCCESS ChangeLog-1.x
 include scripts/maketypes.sh scripts/buildtypes.py
 include AUTHORS README INSTALL ChangeLog setup.py setup.cfg

NEWS

@@ -1,3 +1,11 @@
+What's new in psycopg 1.99.13
+-----------------------------
+
+* Added missing .executemany() method.
+
+* Optimized type cast from PostgreSQL to Python (psycopg should be even
+  faster than before.)
+
 What's new in psycopg 1.99.12
 -----------------------------
 

lib/extensions.py

@@ -25,7 +25,7 @@ This module holds all the extensions to the DBAPI-2.0 provided by psycopg:
 from _psycopg import UNICODE, INTEGER, LONGINTEGER, BOOLEAN, FLOAT
 from _psycopg import TIME, DATE, INTERVAL
 
-from _psycopg import Boolean, QuotedString
+from _psycopg import Boolean, QuotedString, AsIs
 try:
     from _psycopg import DateFromMx, TimeFromMx, TimestampFromMx
     from _psycopg import IntervalFromMx

psycopg/typecast.c

@@ -468,11 +468,9 @@ typecast_cast(PyObject *obj, unsigned char *str, int len, PyObject *curs)
     ((cursorObject*)curs)->caster = obj;
     
     if (self->ccast) {
-        Dprintf("typecast_call: calling C cast function");
         res = self->ccast(str, len, curs);
     }
     else if (self->pcast) {
-        Dprintf("typecast_call: calling python callable");
         res = PyObject_CallFunction(self->pcast, "s#O", str, len, curs);
     }
     else {

psycopg/typecast_array.c

@@ -19,9 +19,12 @@
  * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  */
 
+#define MAX_DIMENSIONS 16
+
 
 /** typecast_array_scan - scan a string looking for array items **/
 
+#define ASCAN_ERROR -1
 #define ASCAN_EOF    0
 #define ASCAN_BEGIN  1
 #define ASCAN_END    2
@@ -32,16 +35,20 @@ static int
 typecast_array_tokenize(unsigned char *str, int strlength,
                         int *pos, unsigned char** token, int *length)
 {
-    int i;
+    int i, l, res = ASCAN_TOKEN;
-    int quoted = 0;
+    int qs = 0; /* 2 = in quotes, 1 = quotes closed */
     
     /* first we check for quotes, used when the content of the item contains
        special or quoted characters */
+    
     if (str[*pos] == '"') {
-        quoted = 1;
+        qs = 2;
         *pos += 1;
     }
 
+    Dprintf("typecast_array_tokenize: '%s'; %d/%d",
+            &str[*pos], *pos, strlength);
+
     for (i = *pos ; i < strlength ; i++) {
         switch (str[i]) {
         case '{':
@@ -49,49 +56,116 @@ typecast_array_tokenize(unsigned char *str, int strlength,
             return ASCAN_BEGIN;
 
         case '}':
-            *pos = i+1;
+            /* we tokenize the last item in the array and then return it to
-            return ASCAN_END;
+               the user togheter with the closing bracket marker */
+            res = ASCAN_END;
+            goto tokenize;
+
+        case '"':
+            /* this will close the quoting only if the previous character was
+               NOT a backslash */
+            if (qs == 2 && str[i-1] != '\\') qs = 1;
+            continue;
+               
+        case '\\':
+            /* something has been quoted, sigh, we'll need a copy buffer */
+            res = ASCAN_QUOTED;
+            continue;
             
         case ',':
+            /* if we're inside quotes we use the comma as a normal char */
+            if (qs == 2)
+                continue;
+            else
+                goto tokenize;
+        }
+    }
+
+    res = ASCAN_EOF;
+    
+ tokenize:
+    l = i - *pos - qs;
+    
+    /* if res is ASCAN_QUOTED we need to copy the string to a newly allocated
+       buffer and return it */
+    if (res == ASCAN_QUOTED) {
+        unsigned char *buffer = PyMem_Malloc(l+1);
+        if (buffer == NULL) return ASCAN_ERROR;
+
+        *token = buffer;
+        
+        for (i = *pos; i < l+*pos; i++) {
+            if (str[i] != '\\')
+                *(buffer++) = str[i];
+        }
+        *buffer = '\0';
+        *length = (int)buffer - (int)*token;
+        *pos = i+2;
+    }
+    else {
         *token = &str[*pos];
-            *length = i - *pos;
+        *length = l;
-            if (quoted == 1)
-                *length -= 1;
         *pos = i+1;
-            return ASCAN_TOKEN;
+        if (res == ASCAN_END && str[*pos] == ',')
-
+            *pos += 1; /* skip both the bracket and the comma */
-        default:
-            /* nothing to do right now */
-            break;
-        }
     }
     
-    *token = &str[*pos];
+    return res;
-    *length = i - *pos;
-    if (quoted == 1)
-        *length -= 1;
-    
-    return ASCAN_EOF;
 }
 
 static int
 typecast_array_scan(unsigned char *str, int strlength,
                     PyObject *curs, PyObject *base, PyObject *array)
 {
-    int state, length, pos = 0;
+    int state, length, bracket = 0, pos = 0;
     unsigned char *token;
 
+    PyObject *stack[MAX_DIMENSIONS];
+    int stack_index = 0;
+    
     while (1) {
         state = typecast_array_tokenize(str, strlength, &pos, &token, &length);
+        if (state == ASCAN_TOKEN
+            || state == ASCAN_QUOTED 
+            || (state ==  ASCAN_EOF && bracket == 0)
+            || (state == ASCAN_END && bracket == 0)) {
             
-        if (state == ASCAN_TOKEN || state ==  ASCAN_EOF) {
             PyObject *obj = typecast_cast(base, token, length, curs);
+
+            /* before anything else we free the memory */
+            if (state == ASCAN_QUOTED) PyMem_Free(token);
             if (obj == NULL) return 0;
+
             PyList_Append(array, obj);
             Py_DECREF(obj);
         }
-        else {
+        else if (state == ASCAN_BEGIN) {
-            Dprintf("** RECURSION (not supported right now)!!");
+            PyObject *sub = PyList_New(0);            
+            if (sub == NULL) return 0;
+
+            PyList_Append(array, sub);
+            Py_DECREF(sub);
+
+            if (stack_index == MAX_DIMENSIONS)
+                return 0;
+
+            stack[stack_index++] = array;
+            array = sub;
+        }
+        else if (state == ASCAN_ERROR) {
+            return 0;
+        }
+
+        /* reset the closing bracket marker just before cheking for ASCAN_END:
+           this is to make sure we don't mistake two closing brackets for an
+           empty item */
+        bracket = 0;
+        
+        if (state == ASCAN_END) {
+            if (--stack_index < 0)
+                return 0;
+            array = stack[stack_index];
+            bracket = 1;
         }
 
         if (state ==  ASCAN_EOF) break;
@@ -116,6 +190,8 @@ typecast_GENERIC_ARRAY_cast(unsigned char *str, int len, PyObject *curs)
         return NULL;
     }
 
+    Dprintf("typecast_GENERIC_ARRAY_cast: scanning %s", str);
+    
     obj = PyList_New(0);
 
     /* scan the array skipping the first level of {} */

sandbox/array.py

@@ -9,7 +9,9 @@ print curs.fetchone()
 curs.execute("SELECT ARRAY['1','2','3'] AS foo")
 print curs.fetchone()
 
-curs.execute("""SELECT ARRAY['','"',''] AS foo""")
+curs.execute("""SELECT ARRAY[',','"','\\\\'] AS foo""")
 d = curs.fetchone()
 print d, '->', d[0][0], d[0][1], d[0][2]
 
+curs.execute("SELECT ARRAY[ARRAY[1,2],ARRAY[3,4]] AS foo")
+print curs.fetchone()

setup.py

@@ -47,7 +47,7 @@ from distutils.core import setup, Extension
 from distutils.sysconfig import get_python_inc
 import distutils.ccompiler
 
-PSYCOPG_VERSION = '1.99.13/devel'
+PSYCOPG_VERSION = '1.99.13'
 version_flags   = []
 
 have_pydatetime = False