py3k: Add module initialization and unicode/bytes int/long thunks

This commit:

* Adds Python 3 module initialization functions. I split out the main init
  of each module into a static setup_module function.
* Adds a py3.h which unifies int/long in Python 3 and unicode/bytes in
  Python 2. _imagingft.c unfortunately looks a little kludgy after this
  because it was already using PyUnicode functions, and I had to mix and
  match there manually.

With this commit, the modules all build successfully under Python 3.

What this commit does NOT do is patch all of the uses of PyArg_ParseTuple
and Py_BuildValue, which all need to be checked for proper use of bytes
and unicode codes. It also does not let selftest.py run yet, because there
are probably hundreds of issues to fix in the Python code itself.

PIL/Image.py

@@ -907,12 +907,12 @@ class Image:
         return self.im.getextrema()
 
     ##
-    # Returns a PyCObject that points to the internal image memory.
+    # Returns a capsule that points to the internal image memory.
     #
-    # @return A PyCObject object.
+    # @return A capsule object.
 
     def getim(self):
-        "Get PyCObject pointer to internal image memory"
+        "Get capsule pointer to internal image memory"
 
         self.load()
         return self.im.ptr

_imaging.c

@@ -76,6 +76,7 @@
 
 #include "Imaging.h"
 
+#include "py3.h"
 
 /* Configuration stuff. Feel free to undef things you don't need. */
 #define WITH_IMAGECHOPS /* ImageChops support */
@@ -896,11 +897,11 @@ _getpalette(ImagingObject* self, PyObject* args)
 	return NULL;
     }
 
-    palette = PyString_FromStringAndSize(NULL, palettesize * bits / 8);
+    palette = PyBytes_FromStringAndSize(NULL, palettesize * bits / 8);
     if (!palette)
 	return NULL;
 
-    pack((UINT8*) PyString_AsString(palette),
+    pack((UINT8*) PyBytes_AsString(palette),
 	 self->image->palette->palette, palettesize);
 
     return palette;
@@ -1221,9 +1222,9 @@ _putdata(ImagingObject* self, PyObject* args)
     }
 
     if (image->image8) {
-        if (PyString_Check(data)) {
+        if (PyBytes_Check(data)) {
             unsigned char* p;
-            p = (unsigned char*) PyString_AS_STRING((PyStringObject*) data);
+            p = (unsigned char*) PyBytes_AS_STRING(data);
             if (scale == 1.0 && offset == 0.0)
                 /* Plain string data */
                 for (i = y = 0; i < n; i += image->xsize, y++) {
@@ -2817,11 +2818,10 @@ _getcodecstatus(PyObject* self, PyObject* args)
     case IMAGING_CODEC_MEMORY:
 	msg = "out of memory"; break;
     default:
-	Py_INCREF(Py_None);
-	return Py_None;
+        Py_RETURN_NONE;
     }
 
-    return PyString_FromString(msg);
+    return PyUnicode_FromString(msg);
 }
 
 /* -------------------------------------------------------------------- */
@@ -2955,7 +2955,7 @@ static struct PyMethodDef methods[] = {
 static PyObject*
 _getattr_mode(ImagingObject* self, void* closure)
 {
-    return PyString_FromString(self->image->mode);
+    return PyUnicode_FromString(self->image->mode);
 }
 
 static PyObject*
@@ -2979,7 +2979,11 @@ _getattr_id(ImagingObject* self, void* closure)
 static PyObject*
 _getattr_ptr(ImagingObject* self, void* closure)
 {
+#if PY_VERSION_HEX >= 0x03020000
+    return PyCapsule_New(self->image, IMAGING_MAGIC, NULL);
+#else
     return PyCObject_FromVoidPtrAndDesc(self->image, IMAGING_MAGIC, NULL);
+#endif
 }
 
 static struct PyGetSetDef getsetters[] = {
@@ -3309,36 +3313,69 @@ static PyMethodDef functions[] = {
     {NULL, NULL} /* sentinel */
 };
 
-PyMODINIT_FUNC
-init_imaging(void)
-{
-    PyObject* m;
-    PyObject* d;
+static int
+setup_module(PyObject* m) {
+    PyObject* d = PyModule_GetDict(m);
 
     /* Ready object types */
-    PyType_Ready(&Imaging_Type);
+    if (PyType_Ready(&Imaging_Type) < 0)
+        return -1;
+
 #ifdef WITH_IMAGEDRAW
-    PyType_Ready(&ImagingFont_Type);
-    PyType_Ready(&ImagingDraw_Type);
+    if (PyType_Ready(&ImagingFont_Type) < 0)
+        return -1;
+
+    if (PyType_Ready(&ImagingDraw_Type) < 0)
+        return -1;
 #endif
-    PyType_Ready(&PixelAccess_Type);
+    if (PyType_Ready(&PixelAccess_Type) < 0)
+        return -1;
 
     ImagingAccessInit();
 
-    m = Py_InitModule("_imaging", functions);
-    d = PyModule_GetDict(m);
-
 #ifdef HAVE_LIBJPEG
   {
     extern const char* ImagingJpegVersion(void);
-    PyDict_SetItemString(d, "jpeglib_version", PyString_FromString(ImagingJpegVersion()));
+    PyDict_SetItemString(d, "jpeglib_version", PyUnicode_FromString(ImagingJpegVersion()));
   }
 #endif
 
 #ifdef HAVE_LIBZ
   {
     extern const char* ImagingZipVersion(void);
-    PyDict_SetItemString(d, "zlib_version", PyString_FromString(ImagingZipVersion()));
+    PyDict_SetItemString(d, "zlib_version", PyUnicode_FromString(ImagingZipVersion()));
   }
 #endif
+
+    return 0;
 }
+
+#if PY_VERSION_HEX >= 0x03000000
+PyMODINIT_FUNC
+PyInit__imaging(void) {
+    PyObject* m;
+
+    static PyModuleDef module_def = {
+        PyModuleDef_HEAD_INIT,
+        "_imaging",         /* m_name */
+        NULL,               /* m_doc */
+        -1,                 /* m_size */
+        functions,          /* m_methods */
+    };
+
+    m = PyModule_Create(&module_def);
+
+    if (setup_module(m) < 0)
+        return NULL;
+
+    return m;
+}
+#else
+PyMODINIT_FUNC
+init_imaging(void)
+{
+    PyObject* m = Py_InitModule("_imaging", functions);
+    setup_module(m);
+}
+#endif
+

_imagingcms.c

@@ -26,6 +26,7 @@ http://www.cazabon.com\n\
 #include "Python.h"
 #include "lcms.h"
 #include "Imaging.h"
+#include "py3.h"
 
 #if PY_VERSION_HEX < 0x01060000
 #define PyObject_New PyObject_NEW
@@ -518,19 +519,19 @@ static struct PyMethodDef cms_profile_methods[] = {
 static PyObject*
 cms_profile_getattr_product_name(CmsProfileObject* self, void* closure)
 {
-    return PyString_FromString(cmsTakeProductName(self->profile));
+    return PyUnicode_FromString(cmsTakeProductName(self->profile));
 }
 
 static PyObject*
 cms_profile_getattr_product_desc(CmsProfileObject* self, void* closure)
 {
-    return PyString_FromString(cmsTakeProductDesc(self->profile));
+    return PyUnicode_FromString(cmsTakeProductDesc(self->profile));
 }
 
 static PyObject*
 cms_profile_getattr_product_info(CmsProfileObject* self, void* closure)
 {
-    return PyString_FromString(cmsTakeProductInfo(self->profile));
+    return PyUnicode_FromString(cmsTakeProductInfo(self->profile));
 }
 
 static PyObject*
@@ -542,13 +543,13 @@ cms_profile_getattr_rendering_intent(CmsProfileObject* self, void* closure)
 static PyObject*
 cms_profile_getattr_pcs(CmsProfileObject* self, void* closure)
 {
-    return PyString_FromString(findICmode(cmsGetPCS(self->profile)));
+    return PyUnicode_FromString(findICmode(cmsGetPCS(self->profile)));
 }
 
 static PyObject*
 cms_profile_getattr_color_space(CmsProfileObject* self, void* closure)
 {
-    return PyString_FromString(findICmode(cmsGetColorSpace(self->profile)));
+    return PyUnicode_FromString(findICmode(cmsGetColorSpace(self->profile)));
 }
 
 /* FIXME: add more properties (creation_datetime etc) */
@@ -602,13 +603,13 @@ static struct PyMethodDef cms_transform_methods[] = {
 static PyObject*
 cms_transform_getattr_inputMode(CmsTransformObject* self, void* closure)
 {
-    return PyString_FromString(self->mode_in);
+    return PyUnicode_FromString(self->mode_in);
 }
 
 static PyObject*
 cms_transform_getattr_outputMode(CmsTransformObject* self, void* closure)
 {
-    return PyString_FromString(self->mode_out);
+    return PyUnicode_FromString(self->mode_out);
 }
 
 static struct PyGetSetDef cms_transform_getsetters[] = {
@@ -649,28 +650,51 @@ static PyTypeObject CmsTransform_Type = {
     cms_transform_getsetters,   /*tp_getset*/
 };
 
-PyMODINIT_FUNC
-init_imagingcms(void)
-{
-    PyObject *m;
+static int
+setup_module(PyObject* m) {
     PyObject *d;
     PyObject *v;
 
+    d = PyModule_GetDict(m);
+
     /* Ready object types */
     PyType_Ready(&CmsProfile_Type);
     PyType_Ready(&CmsTransform_Type);
 
-    m = Py_InitModule("_imagingcms", pyCMSdll_methods);
     d = PyModule_GetDict(m);
 
-#if PY_VERSION_HEX >= 0x02020000
-    v = PyString_FromFormat("%d.%d", LCMS_VERSION / 100, LCMS_VERSION % 100);
-#else
-    {
-        char buffer[100];
-        sprintf(buffer, "%d.%d", LCMS_VERSION / 100, LCMS_VERSION % 100);
-        v = PyString_FromString(buffer);
-    }
-#endif
+    v = PyUnicode_FromFormat("%d.%d", LCMS_VERSION / 100, LCMS_VERSION % 100);
     PyDict_SetItemString(d, "littlecms_version", v);
+
+    return 0;
 }
+
+#if PY_VERSION_HEX >= 0x03000000
+PyMODINIT_FUNC
+PyInit__imagingcms(void) {
+    PyObject* m;
+
+    static PyModuleDef module_def = {
+        PyModuleDef_HEAD_INIT,
+        "_imagingcms",      /* m_name */
+        NULL,               /* m_doc */
+        -1,                 /* m_size */
+        pyCMSdll_methods,   /* m_methods */
+    };
+
+    m = PyModule_Create(&module_def);
+
+    if (setup_module(m) < 0)
+        return NULL;
+
+    return m;
+}
+#else
+PyMODINIT_FUNC
+init_imagingcms(void)
+{
+    PyObject *m = Py_InitModule("_imagingcms", pyCMSdll_methods);
+    setup_module(m);
+}
+#endif
+

_imagingft.c

@@ -44,6 +44,9 @@
 #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
 #endif
 
+#define KEEP_PY_UNICODE
+#include "py3.h"
+
 #if !defined(FT_LOAD_TARGET_MONO)
 #define FT_LOAD_TARGET_MONO  FT_LOAD_MONOCHROME
 #endif
@@ -412,16 +415,26 @@ static PyMethodDef font_methods[] = {
 static PyObject*
 font_getattr_family(FontObject* self, void* closure)
 {
+#if PY_VERSION_HEX >= 0x03000000
+    if (self->face->family_name)
+        return PyUnicode_FromString(self->face->family_name);
+#else
     if (self->face->family_name)
         return PyString_FromString(self->face->family_name);
+#endif
     Py_RETURN_NONE;
 }
 
 static PyObject*
 font_getattr_style(FontObject* self, void* closure)
 {
+#if PY_VERSION_HEX >= 0x03000000
+    if (self->face->style_name)
+        return PyUnicode_FromString(self->face->style_name);
+#else
     if (self->face->style_name)
         return PyString_FromString(self->face->style_name);
+#endif
     Py_RETURN_NONE;
 }
 
@@ -489,33 +502,58 @@ static PyMethodDef _functions[] = {
     {NULL, NULL}
 };
 
-PyMODINIT_FUNC
-init_imagingft(void)
-{
-    PyObject* m;
+static int
+setup_module(PyObject* m) {
     PyObject* d;
     PyObject* v;
     int major, minor, patch;
 
+    d = PyModule_GetDict(m);
+
     /* Ready object type */
     PyType_Ready(&Font_Type);
 
-    m = Py_InitModule("_imagingft", _functions);
-    d = PyModule_GetDict(m);
-
     if (FT_Init_FreeType(&library))
-        return; /* leave it uninitalized */
+        return 0; /* leave it uninitalized */
 
     FT_Library_Version(library, &major, &minor, &patch);
 
-#if PY_VERSION_HEX >= 0x02020000
-    v = PyString_FromFormat("%d.%d.%d", major, minor, patch);
+#if PY_VERSION_HEX >= 0x03000000
+    v = PyUnicode_FromFormat("%d.%d.%d", major, minor, patch);
 #else
-    {
-        char buffer[100];
-        sprintf(buffer, "%d.%d.%d", major, minor, patch);
-        v = PyString_FromString(buffer);
-    }
+    v = PyString_FromFormat("%d.%d.%d", major, minor, patch);
 #endif
     PyDict_SetItemString(d, "freetype2_version", v);
+
+    return 0;
 }
+
+#if PY_VERSION_HEX >= 0x03000000
+PyMODINIT_FUNC
+PyInit__imagingft(void) {
+    PyObject* m;
+
+    static PyModuleDef module_def = {
+        PyModuleDef_HEAD_INIT,
+        "_imagingft",       /* m_name */
+        NULL,               /* m_doc */
+        -1,                 /* m_size */
+        _functions,         /* m_methods */
+    };
+
+    m = PyModule_Create(&module_def);
+
+    if (setup_module(m) < 0)
+        return NULL;
+
+    return m;
+}
+#else
+PyMODINIT_FUNC
+init_imagingft(void)
+{
+    PyObject* m = Py_InitModule("_imagingft", _functions);
+    setup_module(m);
+}
+#endif
+

_imagingmath.c

@@ -16,6 +16,7 @@
 #include "Python.h"
 
 #include "Imaging.h"
+#include "py3.h"
 
 #include "math.h"
 #include "float.h"
@@ -227,14 +228,9 @@ install(PyObject *d, char* name, void* value)
     Py_XDECREF(v);
 }
 
-PyMODINIT_FUNC
-init_imagingmath(void)
-{
-    PyObject* m;
-    PyObject* d;
-
-    m = Py_InitModule("_imagingmath", _functions);
-    d = PyModule_GetDict(m);
+static int
+setup_module(PyObject* m) {
+    PyObject* d = PyModule_GetDict(m);
 
     install(d, "abs_I", abs_I);
     install(d, "neg_I", neg_I);
@@ -281,4 +277,35 @@ init_imagingmath(void)
     install(d, "gt_F", gt_F);
     install(d, "ge_F", ge_F);
 
+    return 0;
 }
+
+#if PY_VERSION_HEX >= 0x03000000
+PyMODINIT_FUNC
+PyInit__imagingmath(void) {
+    PyObject* m;
+
+    static PyModuleDef module_def = {
+        PyModuleDef_HEAD_INIT,
+        "_imagingmath",     /* m_name */
+        NULL,               /* m_doc */
+        -1,                 /* m_size */
+        _functions,         /* m_methods */
+    };
+
+    m = PyModule_Create(&module_def);
+
+    if (setup_module(m) < 0)
+        return NULL;
+
+    return m;
+}
+#else
+PyMODINIT_FUNC
+init_imagingmath(void)
+{
+    PyObject* m = Py_InitModule("_imagingmath", _functions);
+    setup_module(m);
+}
+#endif
+

_imagingtk.c

@@ -63,8 +63,24 @@ static PyMethodDef functions[] = {
     {NULL, NULL} /* sentinel */
 };
 
-DL_EXPORT(void)
+#if PY_VERSION_HEX >= 0x03000000
+PyMODINIT_FUNC
+PyInit__imagingtk(void) {
+    static PyModuleDef module_def = {
+        PyModuleDef_HEAD_INIT,
+        "_imagingtk",       /* m_name */
+        NULL,               /* m_doc */
+        -1,                 /* m_size */
+        functions,          /* m_methods */
+    };
+
+    return PyModule_Create(&module_def);
+}
+#else
+PyMODINIT_FUNC
 init_imagingtk(void)
 {
     Py_InitModule("_imagingtk", functions);
 }
+#endif
+

encode.c

@@ -111,15 +111,15 @@ _encode(ImagingEncoderObject* encoder, PyObject* args)
     if (!PyArg_ParseTuple(args, "|i", &bufsize))
 	return NULL;
 
-    buf = PyString_FromStringAndSize(NULL, bufsize);
+    buf = PyBytes_FromStringAndSize(NULL, bufsize);
     if (!buf)
 	return NULL;
 
     status = encoder->encode(encoder->im, &encoder->state,
-			     (UINT8*) PyString_AsString(buf), bufsize);
+			     (UINT8*) PyBytes_AsString(buf), bufsize);
 
     /* adjust string length to avoid slicing in encoder */
-    if (_PyString_Resize(&buf, (status > 0) ? status : 0) < 0)
+    if (_PyBytes_Resize(&buf, (status > 0) ? status : 0) < 0)
         return NULL;
 
     result = Py_BuildValue("iiO", status, encoder->state.errcode, buf);

map.c

@@ -39,6 +39,8 @@
 #include "windows.h"
 #endif
 
+#include "py3.h"
+
 /* compatibility wrappers (defined in _imaging.c) */
 extern int PyImaging_CheckBuffer(PyObject* buffer);
 extern int PyImaging_GetBuffer(PyObject* buffer, Py_buffer *view);
@@ -148,12 +150,12 @@ mapping_read(ImagingMapperObject* mapper, PyObject* args)
     if (size < 0)
         size = 0;
 
-    buf = PyString_FromStringAndSize(NULL, size);
+    buf = PyBytes_FromStringAndSize(NULL, size);
     if (!buf)
 	return NULL;
 
     if (size > 0) {
-        memcpy(PyString_AsString(buf), mapper->base + mapper->offset, size);
+        memcpy(PyBytes_AsString(buf), mapper->base + mapper->offset, size);
         mapper->offset += size;
     }
 

path.c

@@ -44,6 +44,8 @@
 #define ssizessizeobjargproc intintobjargproc
 #endif
 
+#include "py3.h"
+
 /* compatibility wrappers (defined in _imaging.c) */
 extern int PyImaging_CheckBuffer(PyObject* buffer);
 extern int PyImaging_GetBuffer(PyObject* buffer, Py_buffer *view);

py3.h

@@ -0,0 +1,48 @@
+/*
+    Python3 definition file to consistently map the code to Python 2.6 or
+    Python 3.
+
+    PyInt and PyLong were merged into PyLong in Python 3, so all PyInt functions
+    are mapped to PyLong.
+
+    PyString, on the other hand, was split into PyBytes and PyUnicode. We map
+    both back onto PyString, so use PyBytes or PyUnicode where appropriate. The
+    only exception to this is _imagingft.c, where PyUnicode is left alone.
+*/
+
+#if PY_VERSION_HEX >= 0x03000000
+/* Map PyInt -> PyLong */
+#define PyInt_AsLong                PyLong_AsLong
+#define PyInt_Check                 PyLong_Check
+#define PyInt_FromLong              PyLong_FromLong
+#define PyInt_AS_LONG               PyLong_AS_LONG
+
+#else
+
+#if !defined(KEEP_PY_UNICODE)
+/* Map PyUnicode -> PyString */
+#undef PyUnicode_AsString
+#undef PyUnicode_AS_STRING
+#undef PyUnicode_Check
+#undef PyUnicode_FromStringAndSize
+#undef PyUnicode_FromString
+#undef PyUnicode_FromFormat
+
+#define PyUnicode_AsString          PyString_AsString
+#define PyUnicode_AS_STRING         PyString_AS_STRING
+#define PyUnicode_Check             PyString_Check
+#define PyUnicode_FromStringAndSize PyString_FromStringAndSize
+#define PyUnicode_FromString        PyString_FromString
+#define PyUnicode_FromFormat        PyString_FromFormat
+#endif
+
+/* Map PyBytes -> PyString */
+#define PyBytes_AsString            PyString_AsString
+#define PyBytes_AS_STRING           PyString_AS_STRING
+#define PyBytes_Check               PyString_Check
+#define PyBytes_FromStringAndSize   PyString_FromStringAndSize
+#define PyBytes_FromString          PyString_FromString
+#define _PyBytes_Resize             _PyString_Resize
+
+#endif  /* PY_VERSION_HEX < 0x03000000 */
+