From 4386c4edbb4c312bad14712ebf0a766d35be4c15 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Tue, 28 Jan 2014 12:21:05 -0800 Subject: [PATCH 01/14] multiprocess --in progress --- Tests/run.py | 155 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 64 deletions(-) diff --git a/Tests/run.py b/Tests/run.py index 02b633c90..173536a01 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -3,6 +3,7 @@ from __future__ import print_function # minimal test runner import glob, os, os.path, sys, tempfile +from multiprocessing import Pool try: root = os.path.dirname(__file__) @@ -14,83 +15,109 @@ if not os.path.isfile("PIL/Image.py"): print("***", "$ python Tests/run.py") sys.exit(1) -print("-"*68) - python_options = [] tester_options = [] - -if "--installed" not in sys.argv: - os.environ["PYTHONPATH"] = "." - -if "--coverage" in sys.argv: - tester_options.append("--coverage") - -if "--log" in sys.argv: - tester_options.append("--log") - -files = glob.glob(os.path.join(root, "test_*.py")) -files.sort() - -success = failure = 0 include = [x for x in sys.argv[1:] if x[:2] != "--"] -skipped = [] -python_options = " ".join(python_options) -tester_options = " ".join(tester_options) - -for file in files: - test, ext = os.path.splitext(os.path.basename(file)) - if include and test not in include: - continue +def test_one(f): + test, ext = os.path.splitext(os.path.basename(f)) print("running", test, "...") # 2>&1 works on unix and on modern windowses. we might care about # very old Python versions, but not ancient microsoft products :-) out = os.popen("%s %s -u %s %s 2>&1" % ( - sys.executable, python_options, file, tester_options + sys.executable, python_options, f, tester_options )) result = out.read().strip() - if result == "ok": - result = None - elif result == "skip": - print("---", "skipped") # FIXME: driver should include a reason - skipped.append(test) - continue - elif not result: - result = "(no output)" status = out.close() - if status or result: - if status: - print("=== error", status) - if result: - if result[-3:] == "\nok": - # if there's an ok at the end, it's not really ok - result = result[:-3] - print(result) - failure = failure + 1 - else: - success = success + 1 -print("-"*68) + return (result, status) -temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') -tempfiles = glob.glob(os.path.join(temp_root, "temp_*")) -if tempfiles: - print("===", "remaining temporary files") - for file in tempfiles: - print(file) +def filter_tests(files): + ret = [] + for f in files: + test, ext = os.path.splitext(os.path.basename(f)) + if include and test not in include: + continue + ret.append(f) + return ret + +def main(): + global python_options, tester_options + print("-"*68) -def tests(n): - if n == 1: - return "1 test" - else: - return "%d tests" % n + if "--installed" not in sys.argv: + os.environ["PYTHONPATH"] = "." -if skipped: - print("---", tests(len(skipped)), "skipped.") - print(skipped) -if failure: - print("***", tests(failure), "of", (success + failure), "failed.") - sys.exit(1) -else: - print(tests(success), "passed.") + if "--coverage" in sys.argv: + tester_options.append("--coverage") + + if "--log" in sys.argv: + tester_options.append("--log") + + files = glob.glob(os.path.join(root, "test_*.py")) + files.sort() + + success = failure = 0 + skipped = [] + + python_options = " ".join(python_options) + tester_options = " ".join(tester_options) + + + files = filter_tests(files) + + print (files) + pool = Pool() + results = pool.map(test_one, files) + + for test,(result, status) in zip(files,results): + if result == "ok": + result = None + elif result == "skip": + print("---", "skipped") # FIXME: driver should include a reason + skipped.append(test) + continue + elif not result: + result = "(no output)" + if status or result: + if status: + print("=== error", status) + if result: + if result[-3:] == "\nok": + # if there's an ok at the end, it's not really ok + result = result[:-3] + print(result) + failure = failure + 1 + else: + success = success + 1 + + print("-"*68) + + temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') + tempfiles = glob.glob(os.path.join(temp_root, "temp_*")) + if tempfiles: + print("===", "remaining temporary files") + for file in tempfiles: + print(file) + print("-"*68) + + def tests(n): + if n == 1: + return "1 test" + else: + return "%d tests" % n + + if skipped: + print("---", tests(len(skipped)), "skipped.") + print(skipped) + if failure: + print("***", tests(failure), "of", (success + failure), "failed.") + sys.exit(1) + else: + print(tests(success), "passed.") + + return 0 + +if __name__=='__main__': + sys.exit(main()) From 7cc9af6f83c41377157c75ef23b841735978e2a4 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 28 Jan 2014 12:53:02 -0800 Subject: [PATCH 02/14] added close/join --- Tests/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/run.py b/Tests/run.py index 173536a01..521e66af8 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -67,9 +67,10 @@ def main(): files = filter_tests(files) - print (files) pool = Pool() results = pool.map(test_one, files) + pool.close() + pool.join() for test,(result, status) in zip(files,results): if result == "ok": From 7f9de4a54d7aca29e5cfff31fd843f4c9b15836f Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 5 Mar 2014 15:46:44 -0800 Subject: [PATCH 03/14] multithreaded build on multiproc machines --- mp_compile.py | 42 +++++++++++++++++++++++++++++++ setup.py | 69 +++++++++++++++++++++++++++------------------------ 2 files changed, 79 insertions(+), 32 deletions(-) create mode 100644 mp_compile.py diff --git a/mp_compile.py b/mp_compile.py new file mode 100644 index 000000000..b5b18f960 --- /dev/null +++ b/mp_compile.py @@ -0,0 +1,42 @@ +# A monkey patch of the base distutils.ccompiler to use parallel builds +# Tested on 2.7, looks to be identical to 3.3. + +from multiprocessing import Pool +from distutils.ccompiler import CCompiler + +# hideous monkeypatching. but. but. but. +def _mp_compile_one(tp): + (self, obj, build, cc_args, extra_postargs, pp_opts) = tp + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + return + +def _mp_compile(self, sources, output_dir=None, macros=None, + include_dirs=None, debug=0, extra_preargs=None, + extra_postargs=None, depends=None): + """Compile one or more source files. + + see distutils.ccompiler.CCompiler.compile for comments. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + pool = Pool() + print ("Building using %d processes" % pool._processes) + arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] + results = pool.map_async(_mp_compile_one,arr) + + pool.close() + pool.join() + # Return *all* object filenames, not just the ones we just built. + return objects + +CCompiler.compile = _mp_compile diff --git a/setup.py b/setup.py index 5100aa0f2..81607c544 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ import re import struct import sys +import mp_compile + from distutils.command.build_ext import build_ext from distutils import sysconfig from setuptools import Extension, setup, find_packages @@ -589,38 +591,41 @@ class pil_build_ext(build_ext): finally: os.unlink(tmpfile) -setup( - name=NAME, - version=VERSION, - description='Python Imaging Library (Fork)', - long_description=( + # monkeypatch the compiler + +if __name__=='__main__': + setup( + name=NAME, + version=VERSION, + description='Python Imaging Library (Fork)', + long_description=( _read('README.rst') + b'\n' + _read('CHANGES.rst')).decode('utf-8'), - author='Alex Clark (fork author)', - author_email='aclark@aclark.net', - url='http://python-imaging.github.io/', - classifiers=[ - "Development Status :: 6 - Mature", - "Topic :: Multimedia :: Graphics", - "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", - "Topic :: Multimedia :: Graphics :: Capture :: Scanners", - "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", - "Topic :: Multimedia :: Graphics :: Graphics Conversion", - "Topic :: Multimedia :: Graphics :: Viewers", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", ], - cmdclass={"build_ext": pil_build_ext}, - ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], - include_package_data=True, - packages=find_packages(), - scripts=glob.glob("Scripts/pil*.py"), - test_suite='PIL.tests', - keywords=["Imaging",], - license='Standard PIL License', - zip_safe=True, - ) + author='Alex Clark (fork author)', + author_email='aclark@aclark.net', + url='http://python-imaging.github.io/', + classifiers=[ + "Development Status :: 6 - Mature", + "Topic :: Multimedia :: Graphics", + "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", + "Topic :: Multimedia :: Graphics :: Capture :: Scanners", + "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", + "Topic :: Multimedia :: Graphics :: Graphics Conversion", + "Topic :: Multimedia :: Graphics :: Viewers", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", ], + cmdclass={"build_ext": pil_build_ext}, + ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], + include_package_data=True, + packages=find_packages(), + scripts=glob.glob("Scripts/pil*.py"), + test_suite='PIL.tests', + keywords=["Imaging",], + license='Standard PIL License', + zip_safe=True, + ) From 7ce1c0206599a8cad38b938bbed53b402d2d398a Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 5 Mar 2014 15:57:10 -0800 Subject: [PATCH 04/14] added parallelisim --- mp_compile.py | 47 ++++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/mp_compile.py b/mp_compile.py index b5b18f960..a0a59faa4 100644 --- a/mp_compile.py +++ b/mp_compile.py @@ -17,26 +17,31 @@ def _mp_compile_one(tp): def _mp_compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): - """Compile one or more source files. - - see distutils.ccompiler.CCompiler.compile for comments. - """ - # A concrete compiler class can either override this method - # entirely or implement _compile(). - - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - - pool = Pool() - print ("Building using %d processes" % pool._processes) - arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] - results = pool.map_async(_mp_compile_one,arr) - - pool.close() - pool.join() - # Return *all* object filenames, not just the ones we just built. - return objects + """Compile one or more source files. + + see distutils.ccompiler.CCompiler.compile for comments. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + try: + processes = cpu_count() * 2 + except: + processes = 2 + + pool = Pool(processes) + print ("Building using %d processes" % pool._processes) + arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] + results = pool.map_async(_mp_compile_one,arr) + + pool.close() + pool.join() + # Return *all* object filenames, not just the ones we just built. + return objects CCompiler.compile = _mp_compile From 187e26524f917344aa269a13bf372c08316e8bb4 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 5 Mar 2014 21:15:27 -0800 Subject: [PATCH 05/14] cpu_count *2 is a step too far, the default is faster --- mp_compile.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mp_compile.py b/mp_compile.py index a0a59faa4..947a4086e 100644 --- a/mp_compile.py +++ b/mp_compile.py @@ -1,7 +1,7 @@ # A monkey patch of the base distutils.ccompiler to use parallel builds # Tested on 2.7, looks to be identical to 3.3. -from multiprocessing import Pool +from multiprocessing import Pool, cpu_count from distutils.ccompiler import CCompiler # hideous monkeypatching. but. but. but. @@ -29,12 +29,7 @@ def _mp_compile(self, sources, output_dir=None, macros=None, depends, extra_postargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - try: - processes = cpu_count() * 2 - except: - processes = 2 - - pool = Pool(processes) + pool = Pool() print ("Building using %d processes" % pool._processes) arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] results = pool.map_async(_mp_compile_one,arr) From 8ad2d53483df0f9e183385cf8065b08770eeebc9 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 5 Mar 2014 21:36:28 -0800 Subject: [PATCH 06/14] _processes isn't in py 2.6 --- mp_compile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mp_compile.py b/mp_compile.py index 947a4086e..e6cb5c07e 100644 --- a/mp_compile.py +++ b/mp_compile.py @@ -30,7 +30,9 @@ def _mp_compile(self, sources, output_dir=None, macros=None, cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) pool = Pool() - print ("Building using %d processes" % pool._processes) + try: + print ("Building using %d processes" % pool._processes) + except: pass arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] results = pool.map_async(_mp_compile_one,arr) From 84cfe379d32712921d03d853ab9e223dacc57b6c Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 9 Apr 2014 14:44:55 -0700 Subject: [PATCH 07/14] Fixes race condition on multiprocess testing --- Tests/tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/tester.py b/Tests/tester.py index 5900a7f3a..b722e012a 100644 --- a/Tests/tester.py +++ b/Tests/tester.py @@ -241,7 +241,7 @@ def tempfile(template, *extra): assert temp[:5] in ("temp.", "temp_") name = os.path.basename(sys.argv[0]) name = temp[:4] + os.path.splitext(name)[0][4:] - name = name + "_%d" % len(_tempfiles) + temp[4:] + name = name + "_%d_%d" % (os.getpid(), len(_tempfiles)) + temp[4:] name = os.path.join(root, name) files.append(name) _tempfiles.extend(files) From 1b091e27a20060eb8ebe793c35733180ab1758dc Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 10 Apr 2014 15:28:40 -0700 Subject: [PATCH 08/14] secure tempdir creation and deletion, fixes race condition --- Tests/tester.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Tests/tester.py b/Tests/tester.py index b722e012a..e2cf00f5e 100644 --- a/Tests/tester.py +++ b/Tests/tester.py @@ -1,4 +1,5 @@ from __future__ import print_function +import tempfile # require that deprecation warnings are triggered import warnings @@ -18,6 +19,7 @@ py3 = (sys.version_info >= (3, 0)) _target = None _tempfiles = [] +_temproot = tempfile.mkdtemp(prefix='pillow-tests') _logfile = None @@ -230,19 +232,14 @@ def tempfile(template, *extra): import os import os.path import sys - import tempfile + files = [] - root = os.path.join(tempfile.gettempdir(), 'pillow-tests') - try: - os.mkdir(root) - except OSError: - pass for temp in (template,) + extra: assert temp[:5] in ("temp.", "temp_") name = os.path.basename(sys.argv[0]) name = temp[:4] + os.path.splitext(name)[0][4:] name = name + "_%d_%d" % (os.getpid(), len(_tempfiles)) + temp[4:] - name = os.path.join(root, name) + name = os.path.join(_temproot, name) files.append(name) _tempfiles.extend(files) return files[0] @@ -319,9 +316,8 @@ def _setup(): os.remove(file) except OSError: pass # report? - temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') try: - os.rmdir(temp_root) + os.rmdir(_temproot) except OSError: pass From a2428f24bd5a8851463b03dbf51e95a4adbc1f6b Mon Sep 17 00:00:00 2001 From: wiredfool Date: Wed, 16 Apr 2014 14:16:58 -0700 Subject: [PATCH 09/14] fixed borked merge --- Tests/run.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Tests/run.py b/Tests/run.py index 639df5d7a..1576efa9d 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -27,12 +27,10 @@ include = [x for x in sys.argv[1:] if x[:2] != "--"] skipped = [] failed = [] -python_options = " ".join(python_options) -tester_options = " ".join(tester_options) - ignore_re = re.compile('^ignore: (.*)$', re.MULTILINE) -def test_one(f): +def test_one(params): + f, python_options, tester_options = params test, ext = os.path.splitext(os.path.basename(f)) print("running", test, "...") @@ -69,13 +67,13 @@ def test_one(f): return (result, status) -def filter_tests(files): +def filter_tests(files, python_options, tester_options): ret = [] for f in files: test, ext = os.path.splitext(os.path.basename(f)) if include and test not in include: continue - ret.append(f) + ret.append((f, python_options, tester_options)) return ret def main(): @@ -102,14 +100,14 @@ def main(): tester_options = " ".join(tester_options) - files = filter_tests(files) + files = filter_tests(files, python_options, tester_options) pool = Pool() results = pool.map(test_one, files) pool.close() pool.join() - for test,(result, status) in zip(files,results): + for (test,pyop, top), (result, status) in zip(files,results): if result == "ok": result = None elif result == "skip": From 5e077e0414e51818f817c69e9f6d83c70462ecbb Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 17 Apr 2014 08:55:38 -0700 Subject: [PATCH 10/14] Pass the tempdir through to the tester from the bulk runner --- Tests/run.py | 8 +++++--- Tests/tester.py | 6 +++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Tests/run.py b/Tests/run.py index 1576efa9d..dae5bc5ba 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -27,6 +27,8 @@ include = [x for x in sys.argv[1:] if x[:2] != "--"] skipped = [] failed = [] +_temproot = tempfile.mkdtemp(prefix='pillow-tests') + ignore_re = re.compile('^ignore: (.*)$', re.MULTILINE) def test_one(params): @@ -96,6 +98,8 @@ def main(): success = failure = 0 skipped = [] + tester_options.append(_temproot) + python_options = " ".join(python_options) tester_options = " ".join(tester_options) @@ -130,9 +134,7 @@ def main(): print("-"*68) - #UNDONE -- this is wrong - temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') - tempfiles = glob.glob(os.path.join(temp_root, "temp_*")) + tempfiles = glob.glob(os.path.join(_temproot, "temp_*")) if tempfiles: print("===", "remaining temporary files") for file in tempfiles: diff --git a/Tests/tester.py b/Tests/tester.py index e2cf00f5e..058e8305b 100644 --- a/Tests/tester.py +++ b/Tests/tester.py @@ -1,5 +1,6 @@ from __future__ import print_function import tempfile +import os # require that deprecation warnings are triggered import warnings @@ -19,7 +20,10 @@ py3 = (sys.version_info >= (3, 0)) _target = None _tempfiles = [] -_temproot = tempfile.mkdtemp(prefix='pillow-tests') +if 'pillow-tests' in sys.argv[-1] and os.path.exists(sys.argv[-1]): + _temproot = sys.argv[-1] +else: + _temproot = tempfile.mkdtemp(prefix='pillow-tests') _logfile = None From fdd5900d322eeb6568f938748e6058f1bb552600 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 17 Apr 2014 08:58:41 -0700 Subject: [PATCH 11/14] skipped tests are printed later --- Tests/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/run.py b/Tests/run.py index dae5bc5ba..719f8b904 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -115,7 +115,7 @@ def main(): if result == "ok": result = None elif result == "skip": - print("---", "skipped") # FIXME: driver should include a reason + #print("---", "skipped") # FIXME: driver should include a reason skipped.append(test) continue elif not result: From 7ecda007e04e14c0bdc47c6c241c2915c9ceafdb Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 17 Apr 2014 09:48:16 -0700 Subject: [PATCH 12/14] multiprocess coverage --- .travis.yml | 1 + Tests/tester.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 34ffcfe1a..2f636f223 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,7 @@ script: - python Tests/run.py --coverage after_success: + - coverage combine - coverage report - coveralls - pip install pep8 pyflakes diff --git a/Tests/tester.py b/Tests/tester.py index 058e8305b..690acbafb 100644 --- a/Tests/tester.py +++ b/Tests/tester.py @@ -303,7 +303,7 @@ def _setup(): import sys if "--coverage" in sys.argv: import coverage - cov = coverage.coverage(auto_data=True, include="PIL/*") + cov = coverage.coverage(auto_data=True, data_suffix=True, include="PIL/*") cov.start() def report(): From 955bfd47de82eb50a55a777cd39fc9170b049d04 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 17 Apr 2014 10:05:07 -0700 Subject: [PATCH 13/14] Don't delete the tempdir if it's passed in from the runner --- Tests/tester.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Tests/tester.py b/Tests/tester.py index 690acbafb..3005b45aa 100644 --- a/Tests/tester.py +++ b/Tests/tester.py @@ -22,8 +22,10 @@ _target = None _tempfiles = [] if 'pillow-tests' in sys.argv[-1] and os.path.exists(sys.argv[-1]): _temproot = sys.argv[-1] + _rmtempdir = False else: _temproot = tempfile.mkdtemp(prefix='pillow-tests') + _rmtempdir = True _logfile = None @@ -320,10 +322,11 @@ def _setup(): os.remove(file) except OSError: pass # report? - try: - os.rmdir(_temproot) - except OSError: - pass + if _rmtempdir: + try: + os.rmdir(_temproot) + except OSError: + pass import atexit atexit.register(report) From c39de6d299f1d05b602ba38975e3980167838707 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 18 Apr 2014 13:22:47 -0700 Subject: [PATCH 14/14] environment based concurrency limit --- .travis.yml | 2 ++ Tests/run.py | 6 +++++- mp_compile.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f636f223..d2a82e3b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ virtualenv: notifications: irc: "chat.freenode.net#pil" +env: MAX_CONCURRENCY=4 + python: - 2.6 - 2.7 diff --git a/Tests/run.py b/Tests/run.py index 719f8b904..5b4807530 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -106,7 +106,11 @@ def main(): files = filter_tests(files, python_options, tester_options) - pool = Pool() + try: + max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count())) + except: + max_procs = None + pool = Pool(max_procs) results = pool.map(test_one, files) pool.close() pool.join() diff --git a/mp_compile.py b/mp_compile.py index e6cb5c07e..0ff5b4b62 100644 --- a/mp_compile.py +++ b/mp_compile.py @@ -3,6 +3,7 @@ from multiprocessing import Pool, cpu_count from distutils.ccompiler import CCompiler +import os # hideous monkeypatching. but. but. but. def _mp_compile_one(tp): @@ -28,8 +29,13 @@ def _mp_compile(self, sources, output_dir=None, macros=None, self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - - pool = Pool() + + + try: + max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count())) + except: + max_procs = None + pool = Pool(max_procs) try: print ("Building using %d processes" % pool._processes) except: pass