Merge pull request #721 from wiredfool/multiprocessing_build

Multiprocessing build

.travis.yml

@@ -3,6 +3,8 @@ language: python
 notifications:
   irc: "chat.freenode.net#pil"
 
+env: MAX_CONCURRENCY=4
+
 python:
   - "pypy"
   - 2.6

mp_compile.py

@@ -0,0 +1,50 @@
+# A monkey patch of the base distutils.ccompiler to use parallel builds
+# Tested on 2.7, looks to be identical to 3.3.
+
+from multiprocessing import Pool, cpu_count
+from distutils.ccompiler import CCompiler
+import os
+
+# hideous monkeypatching.  but. but. but.
+def _mp_compile_one(tp):
+    (self, obj, build, cc_args, extra_postargs, pp_opts) = tp
+    try:
+        src, ext = build[obj]
+    except KeyError:
+        return
+    self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+    return
+
+def _mp_compile(self, sources, output_dir=None, macros=None,
+                include_dirs=None, debug=0, extra_preargs=None,
+                extra_postargs=None, depends=None):
+    """Compile one or more source files.
+    
+    see distutils.ccompiler.CCompiler.compile for comments.
+    """
+    # A concrete compiler class can either override this method
+    # entirely or implement _compile().
+    
+    macros, objects, extra_postargs, pp_opts, build = \
+            self._setup_compile(output_dir, macros, include_dirs, sources,
+                                depends, extra_postargs)
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+
+    try:
+        max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count()))
+    except:
+        max_procs = None
+    pool = Pool(max_procs)
+    try:
+        print ("Building using %d processes" % pool._processes)
+    except: pass
+    arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects]
+    results = pool.map_async(_mp_compile_one,arr)
+    
+    pool.close()
+    pool.join()
+    # Return *all* object filenames, not just the ones we just built.
+    return objects
+
+CCompiler.compile = _mp_compile

setup.py

@@ -14,6 +14,8 @@ import re
 import struct
 import sys
 
+import mp_compile
+
 from distutils.command.build_ext import build_ext
 from distutils import sysconfig
 from setuptools import Extension, setup, find_packages
@@ -705,39 +707,40 @@ class pil_build_ext(build_ext):
         finally:
             os.unlink(tmpfile)
 
-setup(
-    name=NAME,
-    version=VERSION,
-    description='Python Imaging Library (Fork)',
-    long_description=(
-        _read('README.rst') + b'\n' +
-        _read('CHANGES.rst')).decode('utf-8'),
-    author='Alex Clark (fork author)',
-    author_email='aclark@aclark.net',
-    url='http://python-pillow.github.io/',
-    classifiers=[
-        "Development Status :: 6 - Mature",
-        "Topic :: Multimedia :: Graphics",
-        "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
-        "Topic :: Multimedia :: Graphics :: Capture :: Scanners",
-        "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
-        "Topic :: Multimedia :: Graphics :: Graphics Conversion",
-        "Topic :: Multimedia :: Graphics :: Viewers",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.6",
-        "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.2",
-        "Programming Language :: Python :: 3.3", ],
-    cmdclass={"build_ext": pil_build_ext},
-    ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
-    include_package_data=True,
-    packages=find_packages(),
-    scripts=glob.glob("Scripts/pil*.py"),
-    test_suite='PIL.tests',
-    keywords=["Imaging", ],
-    license='Standard PIL License',
-    zip_safe=True,
+if __name__=='__main__':
+    setup(
+        name=NAME,
+        version=VERSION,
+        description='Python Imaging Library (Fork)',
+        long_description=(
+            _read('README.rst') + b'\n' +
+            _read('CHANGES.rst')).decode('utf-8'),
+        author='Alex Clark (fork author)',
+        author_email='aclark@aclark.net',
+        url='http://python-pillow.github.io/',
+        classifiers=[
+            "Development Status :: 6 - Mature",
+            "Topic :: Multimedia :: Graphics",
+            "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
+            "Topic :: Multimedia :: Graphics :: Capture :: Scanners",
+            "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
+            "Topic :: Multimedia :: Graphics :: Graphics Conversion",
+            "Topic :: Multimedia :: Graphics :: Viewers",
+            "Programming Language :: Python :: 2",
+            "Programming Language :: Python :: 2.6",
+            "Programming Language :: Python :: 2.7",
+            "Programming Language :: Python :: 3",
+            "Programming Language :: Python :: 3.2",
+            "Programming Language :: Python :: 3.3", ],
+        cmdclass={"build_ext": pil_build_ext},
+        ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
+        include_package_data=True,
+        packages=find_packages(),
+        scripts=glob.glob("Scripts/pil*.py"),
+        test_suite='PIL.tests',
+        keywords=["Imaging", ],
+        license='Standard PIL License',
+        zip_safe=True,
     )
-
+ 
 # End of file