Merge c39de6d299 into f6262505cd

.travis.yml

@@ -7,6 +7,8 @@ virtualenv:
 notifications:
   irc: "chat.freenode.net#pil"
 
+env: MAX_CONCURRENCY=4
+
 python:
   - "pypy"
   - 2.6
@@ -33,6 +35,7 @@ script:
   - python Tests/run.py --coverage
 
 after_success:
+  - coverage combine
   - coverage report
   - coveralls
   - pip install pep8 pyflakes

Tests/run.py

@@ -2,6 +2,8 @@ from __future__ import print_function
 
 # minimal test runner
 
+
+from multiprocessing import Pool
 import glob
 import os
 import os.path
@@ -19,43 +21,27 @@ if not os.path.isfile("PIL/Image.py"):
     print("***", "$ python Tests/run.py")
     sys.exit(1)
 
-print("-"*68)
-
 python_options = []
 tester_options = []
-
-if "--installed" not in sys.argv:
-    os.environ["PYTHONPATH"] = "."
-
-if "--coverage" in sys.argv:
-    tester_options.append("--coverage")
-
-if "--log" in sys.argv:
-    tester_options.append("--log")
-
-files = glob.glob(os.path.join(root, "test_*.py"))
-files.sort()
-
-success = failure = 0
 include = [x for x in sys.argv[1:] if x[:2] != "--"]
 skipped = []
 failed = []
 
-python_options = " ".join(python_options)
+_temproot = tempfile.mkdtemp(prefix='pillow-tests')
-tester_options = " ".join(tester_options)
 
 ignore_re = re.compile('^ignore: (.*)$', re.MULTILINE)
 
-for file in files:
+def test_one(params):
-    test, ext = os.path.splitext(os.path.basename(file))
+    f, python_options, tester_options = params
-    if include and test not in include:
+    test, ext = os.path.splitext(os.path.basename(f))
-        continue
+
     print("running", test, "...")
     # 2>&1 works on unix and on modern windowses.  we might care about
     # very old Python versions, but not ancient microsoft products :-)
     out = os.popen("%s %s -u %s %s 2>&1" % (
-        sys.executable, python_options, file, tester_options
+        sys.executable, python_options, f, tester_options
-        ))
+            ))
+
     result = out.read()
 
     result_lines = result.splitlines()
@@ -83,53 +69,106 @@ for file in files:
     for r in ignore_res:
         result = r.sub('', result)
 
-    result = result.strip()
+    result = result.strip()    
-
-    if result == "ok":
-        result = None
-    elif result == "skip":
-        print("---", "skipped")  # FIXME: driver should include a reason
-        skipped.append(test)
-        continue
-    elif not result:
-        result = "(no output)"
     status = out.close()
-    if status or result:
-        if status:
-            print("=== error", status)
-        if result:
-            if result[-3:] == "\nok":
-                # if there's an ok at the end, it's not really ok
-                result = result[:-3]
-            print(result)
-        failed.append(test)
-    else:
-        success = success + 1
 
-print("-"*68)
+    return (result, status)
 
-temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests')
+def filter_tests(files, python_options, tester_options):
-tempfiles = glob.glob(os.path.join(temp_root, "temp_*"))
+    ret = []
-if tempfiles:
+    for f in files:
-    print("===", "remaining temporary files")
+        test, ext = os.path.splitext(os.path.basename(f))
-    for file in tempfiles:
+        if include and test not in include:
-        print(file)
+            continue
+        ret.append((f, python_options, tester_options))
+    return ret
+
+def main():
+    global python_options, tester_options
+    
     print("-"*68)
 
+    if "--installed" not in sys.argv:
+        os.environ["PYTHONPATH"] = "."
 
-def tests(n):
+    if "--coverage" in sys.argv:
-    if n == 1:
+        tester_options.append("--coverage")
-        return "1 test"
+
+    if "--log" in sys.argv:
+        tester_options.append("--log")
+
+    files = glob.glob(os.path.join(root, "test_*.py"))
+    files.sort()
+
+    success = failure = 0
+    skipped = []
+
+    tester_options.append(_temproot)
+
+    python_options = " ".join(python_options)
+    tester_options = " ".join(tester_options)
+
+
+    files = filter_tests(files, python_options, tester_options)
+
+    try:
+        max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count()))
+    except:
+        max_procs = None
+    pool = Pool(max_procs)
+    results = pool.map(test_one, files)
+    pool.close()
+    pool.join()
+    
+    for (test,pyop, top), (result, status) in zip(files,results):
+        if result == "ok":
+            result = None
+        elif result == "skip":
+            #print("---", "skipped") # FIXME: driver should include a reason
+            skipped.append(test)
+            continue
+        elif not result:
+            result = "(no output)"
+        if status or result:
+            if status:
+                print("=== error", status)
+            if result:
+                if result[-3:] == "\nok":
+                    # if there's an ok at the end, it's not really ok
+                    result = result[:-3]
+                print(result)
+            failed.append(test)
+        else:
+            success = success + 1
+
+    print("-"*68)
+
+    tempfiles = glob.glob(os.path.join(_temproot, "temp_*"))
+    if tempfiles:
+        print("===", "remaining temporary files")
+        for file in tempfiles:
+            print(file)
+        print("-"*68)
+
+    def tests(n):
+        if n == 1:
+            return "1 test"
+        else:
+            return "%d tests" % n
+
+    if skipped:
+        print("---", tests(len(skipped)), "skipped.")
+        print(", ".join(skipped))
+    if failed:
+        failure = len(failed)
+        print("***", tests(failure), "of", (success + failure), "failed:")
+        print(", ".join(failed))
+        sys.exit(1)
     else:
-        return "%d tests" % n
+        print(tests(success), "passed.")
+
+    return 0
+
+if __name__=='__main__':
+    sys.exit(main())
 
-if skipped:
-    print("---", tests(len(skipped)), "skipped:")
-    print(", ".join(skipped))
-if failed:
-    failure = len(failed)
-    print("***", tests(failure), "of", (success + failure), "failed:")
-    print(", ".join(failed))
-    sys.exit(1)
-else:
-    print(tests(success), "passed.")

Tests/tester.py

@@ -1,4 +1,6 @@
 from __future__ import print_function
+import tempfile
+import os
 
 # require that deprecation warnings are triggered
 import warnings
@@ -18,6 +20,12 @@ py3 = (sys.version_info >= (3, 0))
 
 _target = None
 _tempfiles = []
+if 'pillow-tests' in sys.argv[-1] and os.path.exists(sys.argv[-1]):
+    _temproot = sys.argv[-1]
+    _rmtempdir = False
+else:
+    _temproot = tempfile.mkdtemp(prefix='pillow-tests')
+    _rmtempdir = True
 _logfile = None
 
 
@@ -274,19 +282,14 @@ def tempfile(template, *extra):
     import os
     import os.path
     import sys
-    import tempfile
+    
     files = []
-    root = os.path.join(tempfile.gettempdir(), 'pillow-tests')
-    try:
-        os.mkdir(root)
-    except OSError:
-        pass
     for temp in (template,) + extra:
         assert temp[:5] in ("temp.", "temp_")
         name = os.path.basename(sys.argv[0])
         name = temp[:4] + os.path.splitext(name)[0][4:]
-        name = name + "_%d" % len(_tempfiles) + temp[4:]
+        name = name + "_%d_%d" % (os.getpid(), len(_tempfiles)) + temp[4:]
-        name = os.path.join(root, name)
+        name = os.path.join(_temproot, name)
         files.append(name)
     _tempfiles.extend(files)
     return files[0]
@@ -346,7 +349,7 @@ def _setup():
     import sys
     if "--coverage" in sys.argv:
         import coverage
-        cov = coverage.coverage(auto_data=True, include="PIL/*")
+        cov = coverage.coverage(auto_data=True, data_suffix=True, include="PIL/*")
         cov.start()
 
     def report():
@@ -363,11 +366,11 @@ def _setup():
                     os.remove(file)
                 except OSError:
                     pass  # report?
-            temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests')
+            if _rmtempdir:
-            try:
+                try:
-                os.rmdir(temp_root)
+                    os.rmdir(_temproot)
-            except OSError:
+                except OSError:
-                pass
+                    pass
 
     import atexit
     atexit.register(report)

mp_compile.py

@@ -0,0 +1,50 @@
+# A monkey patch of the base distutils.ccompiler to use parallel builds
+# Tested on 2.7, looks to be identical to 3.3.
+
+from multiprocessing import Pool, cpu_count
+from distutils.ccompiler import CCompiler
+import os
+
+# hideous monkeypatching.  but. but. but.
+def _mp_compile_one(tp):
+    (self, obj, build, cc_args, extra_postargs, pp_opts) = tp
+    try:
+        src, ext = build[obj]
+    except KeyError:
+        return
+    self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+    return
+
+def _mp_compile(self, sources, output_dir=None, macros=None,
+                include_dirs=None, debug=0, extra_preargs=None,
+                extra_postargs=None, depends=None):
+    """Compile one or more source files.
+    
+    see distutils.ccompiler.CCompiler.compile for comments.
+    """
+    # A concrete compiler class can either override this method
+    # entirely or implement _compile().
+    
+    macros, objects, extra_postargs, pp_opts, build = \
+            self._setup_compile(output_dir, macros, include_dirs, sources,
+                                depends, extra_postargs)
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+
+    try:
+        max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count()))
+    except:
+        max_procs = None
+    pool = Pool(max_procs)
+    try:
+        print ("Building using %d processes" % pool._processes)
+    except: pass
+    arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects]
+    results = pool.map_async(_mp_compile_one,arr)
+    
+    pool.close()
+    pool.join()
+    # Return *all* object filenames, not just the ones we just built.
+    return objects
+
+CCompiler.compile = _mp_compile

setup.py

@@ -14,6 +14,8 @@ import re
 import struct
 import sys
 
+import mp_compile
+
 from distutils.command.build_ext import build_ext
 from distutils import sysconfig
 from setuptools import Extension, setup, find_packages
@@ -645,38 +647,41 @@ class pil_build_ext(build_ext):
         finally:
             os.unlink(tmpfile)
 
-setup(
+    # monkeypatch the compiler
-    name=NAME,
+
-    version=VERSION,
+if __name__=='__main__':
-    description='Python Imaging Library (Fork)',
+    setup(
-    long_description=(
+        name=NAME,
+        version=VERSION,
+        description='Python Imaging Library (Fork)',
+        long_description=(
         _read('README.rst') + b'\n' +
         _read('CHANGES.rst')).decode('utf-8'),
-    author='Alex Clark (fork author)',
+        author='Alex Clark (fork author)',
-    author_email='aclark@aclark.net',
+        author_email='aclark@aclark.net',
-    url='http://python-imaging.github.io/',
+        url='http://python-imaging.github.io/',
-    classifiers=[
+        classifiers=[
-        "Development Status :: 6 - Mature",
+            "Development Status :: 6 - Mature",
-        "Topic :: Multimedia :: Graphics",
+            "Topic :: Multimedia :: Graphics",
-        "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
+            "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
-        "Topic :: Multimedia :: Graphics :: Capture :: Scanners",
+            "Topic :: Multimedia :: Graphics :: Capture :: Scanners",
-        "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
+            "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
-        "Topic :: Multimedia :: Graphics :: Graphics Conversion",
+            "Topic :: Multimedia :: Graphics :: Graphics Conversion",
-        "Topic :: Multimedia :: Graphics :: Viewers",
+            "Topic :: Multimedia :: Graphics :: Viewers",
-        "Programming Language :: Python :: 2",
+            "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.6",
+            "Programming Language :: Python :: 2.6",
-        "Programming Language :: Python :: 2.7",
+            "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3",
+            "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.2",
+            "Programming Language :: Python :: 3.2",
-        "Programming Language :: Python :: 3.3", ],
+            "Programming Language :: Python :: 3.3", ],
-    cmdclass={"build_ext": pil_build_ext},
+        cmdclass={"build_ext": pil_build_ext},
-    ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
+        ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
-    include_package_data=True,
+        include_package_data=True,
-    packages=find_packages(),
+        packages=find_packages(),
-    scripts=glob.glob("Scripts/pil*.py"),
+        scripts=glob.glob("Scripts/pil*.py"),
-    test_suite='PIL.tests',
+        test_suite='PIL.tests',
-    keywords=["Imaging",],
+        keywords=["Imaging",],
-    license='Standard PIL License',
+        license='Standard PIL License',
-    zip_safe=True,
+        zip_safe=True,
-    )
+        )