Merge branch 'multiprocessing' of github.com:wiredfool/Pillow into multiprocessing

Tests/run.py

@@ -3,6 +3,7 @@ from __future__ import print_function
 # minimal test runner
 
 import glob, os, os.path, sys, tempfile
+from multiprocessing import Pool
 
 try:
     root = os.path.dirname(__file__)
@@ -14,10 +15,36 @@ if not os.path.isfile("PIL/Image.py"):
     print("***", "$ python Tests/run.py")
     sys.exit(1)
 
-print("-"*68)
-
 python_options = []
 tester_options = []
+include = [x for x in sys.argv[1:] if x[:2] != "--"]
+
+def test_one(f):
+    test, ext = os.path.splitext(os.path.basename(f))
+    print("running", test, "...")
+    # 2>&1 works on unix and on modern windowses.  we might care about
+    # very old Python versions, but not ancient microsoft products :-)
+    out = os.popen("%s %s -u %s %s 2>&1" % (
+        sys.executable, python_options, f, tester_options
+            ))
+    result = out.read().strip()
+    status = out.close()
+
+    return (result, status)
+
+def filter_tests(files):
+    ret = []
+    for f in files:
+        test, ext = os.path.splitext(os.path.basename(f))
+        if include and test not in include:
+            continue
+        ret.append(f)
+    return ret
+
+def main():
+    global python_options, tester_options
+    
+    print("-"*68)
 
     if "--installed" not in sys.argv:
         os.environ["PYTHONPATH"] = "."
@@ -32,23 +59,20 @@ files = glob.glob(os.path.join(root, "test_*.py"))
     files.sort()
 
     success = failure = 0
-include = [x for x in sys.argv[1:] if x[:2] != "--"]
     skipped = []
 
     python_options = " ".join(python_options)
     tester_options = " ".join(tester_options)
 
-for file in files:
+
-    test, ext = os.path.splitext(os.path.basename(file))
+    files = filter_tests(files)
-    if include and test not in include:
+
-        continue
+    pool = Pool()
-    print("running", test, "...")
+    results = pool.map(test_one, files)
-    # 2>&1 works on unix and on modern windowses.  we might care about
+    pool.close()
-    # very old Python versions, but not ancient microsoft products :-)
+    pool.join()
-    out = os.popen("%s %s -u %s %s 2>&1" % (
+    
-            sys.executable, python_options, file, tester_options
+    for test,(result, status) in zip(files,results):
-            ))
-    result = out.read().strip()
         if result == "ok":
             result = None
         elif result == "skip":
@@ -57,7 +81,6 @@ for file in files:
             continue
         elif not result:
             result = "(no output)"
-    status = out.close()
         if status or result:
             if status:
                 print("=== error", status)
@@ -94,3 +117,8 @@ if failure:
         sys.exit(1)
     else:
         print(tests(success), "passed.")
+
+    return 0
+
+if __name__=='__main__':
+    sys.exit(main())

mp_compile.py

@@ -0,0 +1,47 @@
+# A monkey patch of the base distutils.ccompiler to use parallel builds
+# Tested on 2.7, looks to be identical to 3.3.
+
+from multiprocessing import Pool
+from distutils.ccompiler import CCompiler
+
+# hideous monkeypatching.  but. but. but.
+def _mp_compile_one(tp):
+    (self, obj, build, cc_args, extra_postargs, pp_opts) = tp
+    try:
+        src, ext = build[obj]
+    except KeyError:
+        return
+    self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+    return
+
+def _mp_compile(self, sources, output_dir=None, macros=None,
+                include_dirs=None, debug=0, extra_preargs=None,
+                extra_postargs=None, depends=None):
+    """Compile one or more source files.
+    
+    see distutils.ccompiler.CCompiler.compile for comments.
+    """
+    # A concrete compiler class can either override this method
+    # entirely or implement _compile().
+    
+    macros, objects, extra_postargs, pp_opts, build = \
+            self._setup_compile(output_dir, macros, include_dirs, sources,
+                                depends, extra_postargs)
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+    
+    try:
+        processes = cpu_count() * 2
+    except:
+        processes = 2
+        
+    pool = Pool(processes)
+    print ("Building using %d processes" % pool._processes)
+    arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects]
+    results = pool.map_async(_mp_compile_one,arr)
+    
+    pool.close()
+    pool.join()
+    # Return *all* object filenames, not just the ones we just built.
+    return objects
+
+CCompiler.compile = _mp_compile

setup.py

@@ -14,6 +14,8 @@ import re
 import struct
 import sys
 
+import mp_compile
+
 from distutils.command.build_ext import build_ext
 from distutils import sysconfig
 from setuptools import Extension, setup, find_packages
@@ -615,6 +617,9 @@ class pil_build_ext(build_ext):
         finally:
             os.unlink(tmpfile)
 
+    # monkeypatch the compiler
+
+if __name__=='__main__':
     setup(
         name=NAME,
         version=VERSION,