multithreaded build on multiproc machines

mp_compile.py

@@ -0,0 +1,42 @@
+# A monkey patch of the base distutils.ccompiler to use parallel builds
+# Tested on 2.7, looks to be identical to 3.3.
+
+from multiprocessing import Pool
+from distutils.ccompiler import CCompiler
+
+# hideous monkeypatching.  but. but. but.
+def _mp_compile_one(tp):
+    (self, obj, build, cc_args, extra_postargs, pp_opts) = tp
+    try:
+        src, ext = build[obj]
+    except KeyError:
+        return
+    self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+    return
+
+def _mp_compile(self, sources, output_dir=None, macros=None,
+                include_dirs=None, debug=0, extra_preargs=None,
+                extra_postargs=None, depends=None):
+        """Compile one or more source files.
+
+        see distutils.ccompiler.CCompiler.compile for comments.
+        """
+        # A concrete compiler class can either override this method
+        # entirely or implement _compile().
+
+        macros, objects, extra_postargs, pp_opts, build = \
+                self._setup_compile(output_dir, macros, include_dirs, sources,
+                                    depends, extra_postargs)
+        cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+        pool = Pool()
+        print ("Building using %d processes" % pool._processes)
+        arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects]
+        results = pool.map_async(_mp_compile_one,arr)
+                           
+        pool.close()
+        pool.join()
+        # Return *all* object filenames, not just the ones we just built.
+        return objects
+
+CCompiler.compile = _mp_compile

setup.py

@@ -14,6 +14,8 @@ import re
 import struct
 import sys
 
+import mp_compile
+
 from distutils.command.build_ext import build_ext
 from distutils import sysconfig
 from setuptools import Extension, setup, find_packages
@@ -589,6 +591,9 @@ class pil_build_ext(build_ext):
         finally:
             os.unlink(tmpfile)
 
+    # monkeypatch the compiler
+
+if __name__=='__main__':
     setup(
         name=NAME,
         version=VERSION,