From 4386c4edbb4c312bad14712ebf0a766d35be4c15 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Tue, 28 Jan 2014 12:21:05 -0800 Subject: [PATCH 1/4] multiprocess --in progress --- Tests/run.py | 155 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 64 deletions(-) diff --git a/Tests/run.py b/Tests/run.py index 02b633c90..173536a01 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -3,6 +3,7 @@ from __future__ import print_function # minimal test runner import glob, os, os.path, sys, tempfile +from multiprocessing import Pool try: root = os.path.dirname(__file__) @@ -14,83 +15,109 @@ if not os.path.isfile("PIL/Image.py"): print("***", "$ python Tests/run.py") sys.exit(1) -print("-"*68) - python_options = [] tester_options = [] - -if "--installed" not in sys.argv: - os.environ["PYTHONPATH"] = "." - -if "--coverage" in sys.argv: - tester_options.append("--coverage") - -if "--log" in sys.argv: - tester_options.append("--log") - -files = glob.glob(os.path.join(root, "test_*.py")) -files.sort() - -success = failure = 0 include = [x for x in sys.argv[1:] if x[:2] != "--"] -skipped = [] -python_options = " ".join(python_options) -tester_options = " ".join(tester_options) - -for file in files: - test, ext = os.path.splitext(os.path.basename(file)) - if include and test not in include: - continue +def test_one(f): + test, ext = os.path.splitext(os.path.basename(f)) print("running", test, "...") # 2>&1 works on unix and on modern windowses. we might care about # very old Python versions, but not ancient microsoft products :-) out = os.popen("%s %s -u %s %s 2>&1" % ( - sys.executable, python_options, file, tester_options + sys.executable, python_options, f, tester_options )) result = out.read().strip() - if result == "ok": - result = None - elif result == "skip": - print("---", "skipped") # FIXME: driver should include a reason - skipped.append(test) - continue - elif not result: - result = "(no output)" status = out.close() - if status or result: - if status: - print("=== error", status) - if result: - if result[-3:] == "\nok": - # if there's an ok at the end, it's not really ok - result = result[:-3] - print(result) - failure = failure + 1 - else: - success = success + 1 -print("-"*68) + return (result, status) -temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') -tempfiles = glob.glob(os.path.join(temp_root, "temp_*")) -if tempfiles: - print("===", "remaining temporary files") - for file in tempfiles: - print(file) +def filter_tests(files): + ret = [] + for f in files: + test, ext = os.path.splitext(os.path.basename(f)) + if include and test not in include: + continue + ret.append(f) + return ret + +def main(): + global python_options, tester_options + print("-"*68) -def tests(n): - if n == 1: - return "1 test" - else: - return "%d tests" % n + if "--installed" not in sys.argv: + os.environ["PYTHONPATH"] = "." -if skipped: - print("---", tests(len(skipped)), "skipped.") - print(skipped) -if failure: - print("***", tests(failure), "of", (success + failure), "failed.") - sys.exit(1) -else: - print(tests(success), "passed.") + if "--coverage" in sys.argv: + tester_options.append("--coverage") + + if "--log" in sys.argv: + tester_options.append("--log") + + files = glob.glob(os.path.join(root, "test_*.py")) + files.sort() + + success = failure = 0 + skipped = [] + + python_options = " ".join(python_options) + tester_options = " ".join(tester_options) + + + files = filter_tests(files) + + print (files) + pool = Pool() + results = pool.map(test_one, files) + + for test,(result, status) in zip(files,results): + if result == "ok": + result = None + elif result == "skip": + print("---", "skipped") # FIXME: driver should include a reason + skipped.append(test) + continue + elif not result: + result = "(no output)" + if status or result: + if status: + print("=== error", status) + if result: + if result[-3:] == "\nok": + # if there's an ok at the end, it's not really ok + result = result[:-3] + print(result) + failure = failure + 1 + else: + success = success + 1 + + print("-"*68) + + temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') + tempfiles = glob.glob(os.path.join(temp_root, "temp_*")) + if tempfiles: + print("===", "remaining temporary files") + for file in tempfiles: + print(file) + print("-"*68) + + def tests(n): + if n == 1: + return "1 test" + else: + return "%d tests" % n + + if skipped: + print("---", tests(len(skipped)), "skipped.") + print(skipped) + if failure: + print("***", tests(failure), "of", (success + failure), "failed.") + sys.exit(1) + else: + print(tests(success), "passed.") + + return 0 + +if __name__=='__main__': + sys.exit(main()) From 7cc9af6f83c41377157c75ef23b841735978e2a4 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 28 Jan 2014 12:53:02 -0800 Subject: [PATCH 2/4] added close/join --- Tests/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/run.py b/Tests/run.py index 173536a01..521e66af8 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -67,9 +67,10 @@ def main(): files = filter_tests(files) - print (files) pool = Pool() results = pool.map(test_one, files) + pool.close() + pool.join() for test,(result, status) in zip(files,results): if result == "ok": From 7f9de4a54d7aca29e5cfff31fd843f4c9b15836f Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 5 Mar 2014 15:46:44 -0800 Subject: [PATCH 3/4] multithreaded build on multiproc machines --- mp_compile.py | 42 +++++++++++++++++++++++++++++++ setup.py | 69 +++++++++++++++++++++++++++------------------------ 2 files changed, 79 insertions(+), 32 deletions(-) create mode 100644 mp_compile.py diff --git a/mp_compile.py b/mp_compile.py new file mode 100644 index 000000000..b5b18f960 --- /dev/null +++ b/mp_compile.py @@ -0,0 +1,42 @@ +# A monkey patch of the base distutils.ccompiler to use parallel builds +# Tested on 2.7, looks to be identical to 3.3. + +from multiprocessing import Pool +from distutils.ccompiler import CCompiler + +# hideous monkeypatching. but. but. but. +def _mp_compile_one(tp): + (self, obj, build, cc_args, extra_postargs, pp_opts) = tp + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + return + +def _mp_compile(self, sources, output_dir=None, macros=None, + include_dirs=None, debug=0, extra_preargs=None, + extra_postargs=None, depends=None): + """Compile one or more source files. + + see distutils.ccompiler.CCompiler.compile for comments. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + pool = Pool() + print ("Building using %d processes" % pool._processes) + arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] + results = pool.map_async(_mp_compile_one,arr) + + pool.close() + pool.join() + # Return *all* object filenames, not just the ones we just built. + return objects + +CCompiler.compile = _mp_compile diff --git a/setup.py b/setup.py index 5100aa0f2..81607c544 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ import re import struct import sys +import mp_compile + from distutils.command.build_ext import build_ext from distutils import sysconfig from setuptools import Extension, setup, find_packages @@ -589,38 +591,41 @@ class pil_build_ext(build_ext): finally: os.unlink(tmpfile) -setup( - name=NAME, - version=VERSION, - description='Python Imaging Library (Fork)', - long_description=( + # monkeypatch the compiler + +if __name__=='__main__': + setup( + name=NAME, + version=VERSION, + description='Python Imaging Library (Fork)', + long_description=( _read('README.rst') + b'\n' + _read('CHANGES.rst')).decode('utf-8'), - author='Alex Clark (fork author)', - author_email='aclark@aclark.net', - url='http://python-imaging.github.io/', - classifiers=[ - "Development Status :: 6 - Mature", - "Topic :: Multimedia :: Graphics", - "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", - "Topic :: Multimedia :: Graphics :: Capture :: Scanners", - "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", - "Topic :: Multimedia :: Graphics :: Graphics Conversion", - "Topic :: Multimedia :: Graphics :: Viewers", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", ], - cmdclass={"build_ext": pil_build_ext}, - ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], - include_package_data=True, - packages=find_packages(), - scripts=glob.glob("Scripts/pil*.py"), - test_suite='PIL.tests', - keywords=["Imaging",], - license='Standard PIL License', - zip_safe=True, - ) + author='Alex Clark (fork author)', + author_email='aclark@aclark.net', + url='http://python-imaging.github.io/', + classifiers=[ + "Development Status :: 6 - Mature", + "Topic :: Multimedia :: Graphics", + "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", + "Topic :: Multimedia :: Graphics :: Capture :: Scanners", + "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", + "Topic :: Multimedia :: Graphics :: Graphics Conversion", + "Topic :: Multimedia :: Graphics :: Viewers", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", ], + cmdclass={"build_ext": pil_build_ext}, + ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], + include_package_data=True, + packages=find_packages(), + scripts=glob.glob("Scripts/pil*.py"), + test_suite='PIL.tests', + keywords=["Imaging",], + license='Standard PIL License', + zip_safe=True, + ) From 7ce1c0206599a8cad38b938bbed53b402d2d398a Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 5 Mar 2014 15:57:10 -0800 Subject: [PATCH 4/4] added parallelisim --- mp_compile.py | 47 ++++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/mp_compile.py b/mp_compile.py index b5b18f960..a0a59faa4 100644 --- a/mp_compile.py +++ b/mp_compile.py @@ -17,26 +17,31 @@ def _mp_compile_one(tp): def _mp_compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): - """Compile one or more source files. - - see distutils.ccompiler.CCompiler.compile for comments. - """ - # A concrete compiler class can either override this method - # entirely or implement _compile(). - - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - - pool = Pool() - print ("Building using %d processes" % pool._processes) - arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] - results = pool.map_async(_mp_compile_one,arr) - - pool.close() - pool.join() - # Return *all* object filenames, not just the ones we just built. - return objects + """Compile one or more source files. + + see distutils.ccompiler.CCompiler.compile for comments. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + try: + processes = cpu_count() * 2 + except: + processes = 2 + + pool = Pool(processes) + print ("Building using %d processes" % pool._processes) + arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] + results = pool.map_async(_mp_compile_one,arr) + + pool.close() + pool.join() + # Return *all* object filenames, not just the ones we just built. + return objects CCompiler.compile = _mp_compile