diff --git a/.travis.yml b/.travis.yml index 07862a6c1..2570b9c6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ virtualenv: notifications: irc: "chat.freenode.net#pil" +env: MAX_CONCURRENCY=4 + python: - "pypy" - 2.6 @@ -33,6 +35,7 @@ script: - python Tests/run.py --coverage after_success: + - coverage combine - coverage report - coveralls - pip install pep8 pyflakes diff --git a/Tests/run.py b/Tests/run.py index 758923f0f..42461b85d 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -2,6 +2,8 @@ from __future__ import print_function # minimal test runner + +from multiprocessing import Pool import glob import os import os.path @@ -19,43 +21,27 @@ if not os.path.isfile("PIL/Image.py"): print("***", "$ python Tests/run.py") sys.exit(1) -print("-"*68) - python_options = [] tester_options = [] - -if "--installed" not in sys.argv: - os.environ["PYTHONPATH"] = "." - -if "--coverage" in sys.argv: - tester_options.append("--coverage") - -if "--log" in sys.argv: - tester_options.append("--log") - -files = glob.glob(os.path.join(root, "test_*.py")) -files.sort() - -success = failure = 0 include = [x for x in sys.argv[1:] if x[:2] != "--"] skipped = [] failed = [] -python_options = " ".join(python_options) -tester_options = " ".join(tester_options) +_temproot = tempfile.mkdtemp(prefix='pillow-tests') ignore_re = re.compile('^ignore: (.*)$', re.MULTILINE) -for file in files: - test, ext = os.path.splitext(os.path.basename(file)) - if include and test not in include: - continue +def test_one(params): + f, python_options, tester_options = params + test, ext = os.path.splitext(os.path.basename(f)) + print("running", test, "...") # 2>&1 works on unix and on modern windowses. we might care about # very old Python versions, but not ancient microsoft products :-) out = os.popen("%s %s -u %s %s 2>&1" % ( - sys.executable, python_options, file, tester_options - )) + sys.executable, python_options, f, tester_options + )) + result = out.read() result_lines = result.splitlines() @@ -83,53 +69,106 @@ for file in files: for r in ignore_res: result = r.sub('', result) - result = result.strip() - - if result == "ok": - result = None - elif result == "skip": - print("---", "skipped") # FIXME: driver should include a reason - skipped.append(test) - continue - elif not result: - result = "(no output)" + result = result.strip() status = out.close() - if status or result: - if status: - print("=== error", status) - if result: - if result[-3:] == "\nok": - # if there's an ok at the end, it's not really ok - result = result[:-3] - print(result) - failed.append(test) - else: - success = success + 1 -print("-"*68) + return (result, status) -temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') -tempfiles = glob.glob(os.path.join(temp_root, "temp_*")) -if tempfiles: - print("===", "remaining temporary files") - for file in tempfiles: - print(file) +def filter_tests(files, python_options, tester_options): + ret = [] + for f in files: + test, ext = os.path.splitext(os.path.basename(f)) + if include and test not in include: + continue + ret.append((f, python_options, tester_options)) + return ret + +def main(): + global python_options, tester_options + print("-"*68) + if "--installed" not in sys.argv: + os.environ["PYTHONPATH"] = "." -def tests(n): - if n == 1: - return "1 test" + if "--coverage" in sys.argv: + tester_options.append("--coverage") + + if "--log" in sys.argv: + tester_options.append("--log") + + files = glob.glob(os.path.join(root, "test_*.py")) + files.sort() + + success = failure = 0 + skipped = [] + + tester_options.append(_temproot) + + python_options = " ".join(python_options) + tester_options = " ".join(tester_options) + + + files = filter_tests(files, python_options, tester_options) + + try: + max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count())) + except: + max_procs = None + pool = Pool(max_procs) + results = pool.map(test_one, files) + pool.close() + pool.join() + + for (test,pyop, top), (result, status) in zip(files,results): + if result == "ok": + result = None + elif result == "skip": + #print("---", "skipped") # FIXME: driver should include a reason + skipped.append(test) + continue + elif not result: + result = "(no output)" + if status or result: + if status: + print("=== error", status) + if result: + if result[-3:] == "\nok": + # if there's an ok at the end, it's not really ok + result = result[:-3] + print(result) + failed.append(test) + else: + success = success + 1 + + print("-"*68) + + tempfiles = glob.glob(os.path.join(_temproot, "temp_*")) + if tempfiles: + print("===", "remaining temporary files") + for file in tempfiles: + print(file) + print("-"*68) + + def tests(n): + if n == 1: + return "1 test" + else: + return "%d tests" % n + + if skipped: + print("---", tests(len(skipped)), "skipped.") + print(", ".join(skipped)) + if failed: + failure = len(failed) + print("***", tests(failure), "of", (success + failure), "failed:") + print(", ".join(failed)) + sys.exit(1) else: - return "%d tests" % n + print(tests(success), "passed.") + + return 0 + +if __name__=='__main__': + sys.exit(main()) -if skipped: - print("---", tests(len(skipped)), "skipped:") - print(", ".join(skipped)) -if failed: - failure = len(failed) - print("***", tests(failure), "of", (success + failure), "failed:") - print(", ".join(failed)) - sys.exit(1) -else: - print(tests(success), "passed.") diff --git a/Tests/tester.py b/Tests/tester.py index 32da48e98..b7666cfc6 100644 --- a/Tests/tester.py +++ b/Tests/tester.py @@ -1,4 +1,6 @@ from __future__ import print_function +import tempfile +import os # require that deprecation warnings are triggered import warnings @@ -18,6 +20,12 @@ py3 = (sys.version_info >= (3, 0)) _target = None _tempfiles = [] +if 'pillow-tests' in sys.argv[-1] and os.path.exists(sys.argv[-1]): + _temproot = sys.argv[-1] + _rmtempdir = False +else: + _temproot = tempfile.mkdtemp(prefix='pillow-tests') + _rmtempdir = True _logfile = None @@ -274,19 +282,14 @@ def tempfile(template, *extra): import os import os.path import sys - import tempfile + files = [] - root = os.path.join(tempfile.gettempdir(), 'pillow-tests') - try: - os.mkdir(root) - except OSError: - pass for temp in (template,) + extra: assert temp[:5] in ("temp.", "temp_") name = os.path.basename(sys.argv[0]) name = temp[:4] + os.path.splitext(name)[0][4:] - name = name + "_%d" % len(_tempfiles) + temp[4:] - name = os.path.join(root, name) + name = name + "_%d_%d" % (os.getpid(), len(_tempfiles)) + temp[4:] + name = os.path.join(_temproot, name) files.append(name) _tempfiles.extend(files) return files[0] @@ -346,7 +349,7 @@ def _setup(): import sys if "--coverage" in sys.argv: import coverage - cov = coverage.coverage(auto_data=True, include="PIL/*") + cov = coverage.coverage(auto_data=True, data_suffix=True, include="PIL/*") cov.start() def report(): @@ -363,11 +366,11 @@ def _setup(): os.remove(file) except OSError: pass # report? - temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') - try: - os.rmdir(temp_root) - except OSError: - pass + if _rmtempdir: + try: + os.rmdir(_temproot) + except OSError: + pass import atexit atexit.register(report) diff --git a/mp_compile.py b/mp_compile.py new file mode 100644 index 000000000..0ff5b4b62 --- /dev/null +++ b/mp_compile.py @@ -0,0 +1,50 @@ +# A monkey patch of the base distutils.ccompiler to use parallel builds +# Tested on 2.7, looks to be identical to 3.3. + +from multiprocessing import Pool, cpu_count +from distutils.ccompiler import CCompiler +import os + +# hideous monkeypatching. but. but. but. +def _mp_compile_one(tp): + (self, obj, build, cc_args, extra_postargs, pp_opts) = tp + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + return + +def _mp_compile(self, sources, output_dir=None, macros=None, + include_dirs=None, debug=0, extra_preargs=None, + extra_postargs=None, depends=None): + """Compile one or more source files. + + see distutils.ccompiler.CCompiler.compile for comments. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + + try: + max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count())) + except: + max_procs = None + pool = Pool(max_procs) + try: + print ("Building using %d processes" % pool._processes) + except: pass + arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] + results = pool.map_async(_mp_compile_one,arr) + + pool.close() + pool.join() + # Return *all* object filenames, not just the ones we just built. + return objects + +CCompiler.compile = _mp_compile diff --git a/setup.py b/setup.py index 93918debd..b3d6195cd 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ import re import struct import sys +import mp_compile + from distutils.command.build_ext import build_ext from distutils import sysconfig from setuptools import Extension, setup, find_packages @@ -645,38 +647,41 @@ class pil_build_ext(build_ext): finally: os.unlink(tmpfile) -setup( - name=NAME, - version=VERSION, - description='Python Imaging Library (Fork)', - long_description=( + # monkeypatch the compiler + +if __name__=='__main__': + setup( + name=NAME, + version=VERSION, + description='Python Imaging Library (Fork)', + long_description=( _read('README.rst') + b'\n' + _read('CHANGES.rst')).decode('utf-8'), - author='Alex Clark (fork author)', - author_email='aclark@aclark.net', - url='http://python-imaging.github.io/', - classifiers=[ - "Development Status :: 6 - Mature", - "Topic :: Multimedia :: Graphics", - "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", - "Topic :: Multimedia :: Graphics :: Capture :: Scanners", - "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", - "Topic :: Multimedia :: Graphics :: Graphics Conversion", - "Topic :: Multimedia :: Graphics :: Viewers", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", ], - cmdclass={"build_ext": pil_build_ext}, - ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], - include_package_data=True, - packages=find_packages(), - scripts=glob.glob("Scripts/pil*.py"), - test_suite='PIL.tests', - keywords=["Imaging",], - license='Standard PIL License', - zip_safe=True, - ) + author='Alex Clark (fork author)', + author_email='aclark@aclark.net', + url='http://python-imaging.github.io/', + classifiers=[ + "Development Status :: 6 - Mature", + "Topic :: Multimedia :: Graphics", + "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", + "Topic :: Multimedia :: Graphics :: Capture :: Scanners", + "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", + "Topic :: Multimedia :: Graphics :: Graphics Conversion", + "Topic :: Multimedia :: Graphics :: Viewers", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", ], + cmdclass={"build_ext": pil_build_ext}, + ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], + include_package_data=True, + packages=find_packages(), + scripts=glob.glob("Scripts/pil*.py"), + test_suite='PIL.tests', + keywords=["Imaging",], + license='Standard PIL License', + zip_safe=True, + )