From 2be4e9f3e5f433aa34e73ed225837ca0ebd06a5c Mon Sep 17 00:00:00 2001 From: wiredfool Date: Tue, 24 Jun 2014 15:57:24 -0700 Subject: [PATCH 1/2] Multithreaded build --- mp_compile.py | 50 ++++++++++++++++++++++++++++++++++++ setup.py | 71 +++++++++++++++++++++++++++------------------------ 2 files changed, 87 insertions(+), 34 deletions(-) create mode 100644 mp_compile.py diff --git a/mp_compile.py b/mp_compile.py new file mode 100644 index 000000000..0ff5b4b62 --- /dev/null +++ b/mp_compile.py @@ -0,0 +1,50 @@ +# A monkey patch of the base distutils.ccompiler to use parallel builds +# Tested on 2.7, looks to be identical to 3.3. + +from multiprocessing import Pool, cpu_count +from distutils.ccompiler import CCompiler +import os + +# hideous monkeypatching. but. but. but. +def _mp_compile_one(tp): + (self, obj, build, cc_args, extra_postargs, pp_opts) = tp + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + return + +def _mp_compile(self, sources, output_dir=None, macros=None, + include_dirs=None, debug=0, extra_preargs=None, + extra_postargs=None, depends=None): + """Compile one or more source files. + + see distutils.ccompiler.CCompiler.compile for comments. + """ + # A concrete compiler class can either override this method + # entirely or implement _compile(). + + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + + try: + max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count())) + except: + max_procs = None + pool = Pool(max_procs) + try: + print ("Building using %d processes" % pool._processes) + except: pass + arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects] + results = pool.map_async(_mp_compile_one,arr) + + pool.close() + pool.join() + # Return *all* object filenames, not just the ones we just built. + return objects + +CCompiler.compile = _mp_compile diff --git a/setup.py b/setup.py index 2fbcc2959..1a31a8981 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ import re import struct import sys +import mp_compile + from distutils.command.build_ext import build_ext from distutils import sysconfig from setuptools import Extension, setup, find_packages @@ -705,39 +707,40 @@ class pil_build_ext(build_ext): finally: os.unlink(tmpfile) -setup( - name=NAME, - version=VERSION, - description='Python Imaging Library (Fork)', - long_description=( - _read('README.rst') + b'\n' + - _read('CHANGES.rst')).decode('utf-8'), - author='Alex Clark (fork author)', - author_email='aclark@aclark.net', - url='http://python-pillow.github.io/', - classifiers=[ - "Development Status :: 6 - Mature", - "Topic :: Multimedia :: Graphics", - "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", - "Topic :: Multimedia :: Graphics :: Capture :: Scanners", - "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", - "Topic :: Multimedia :: Graphics :: Graphics Conversion", - "Topic :: Multimedia :: Graphics :: Viewers", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", ], - cmdclass={"build_ext": pil_build_ext}, - ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], - include_package_data=True, - packages=find_packages(), - scripts=glob.glob("Scripts/pil*.py"), - test_suite='PIL.tests', - keywords=["Imaging", ], - license='Standard PIL License', - zip_safe=True, +if __name__=='__main__': + setup( + name=NAME, + version=VERSION, + description='Python Imaging Library (Fork)', + long_description=( + _read('README.rst') + b'\n' + + _read('CHANGES.rst')).decode('utf-8'), + author='Alex Clark (fork author)', + author_email='aclark@aclark.net', + url='http://python-pillow.github.io/', + classifiers=[ + "Development Status :: 6 - Mature", + "Topic :: Multimedia :: Graphics", + "Topic :: Multimedia :: Graphics :: Capture :: Digital Camera", + "Topic :: Multimedia :: Graphics :: Capture :: Scanners", + "Topic :: Multimedia :: Graphics :: Capture :: Screen Capture", + "Topic :: Multimedia :: Graphics :: Graphics Conversion", + "Topic :: Multimedia :: Graphics :: Viewers", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", ], + cmdclass={"build_ext": pil_build_ext}, + ext_modules=[Extension("PIL._imaging", ["_imaging.c"])], + include_package_data=True, + packages=find_packages(), + scripts=glob.glob("Scripts/pil*.py"), + test_suite='PIL.tests', + keywords=["Imaging", ], + license='Standard PIL License', + zip_safe=True, ) - + # End of file From ff95b8bc513d75926eeffc800465cd2402ec48d9 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Tue, 24 Jun 2014 16:01:05 -0700 Subject: [PATCH 2/2] Limit buildprocs on travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 389051358..4de1fde99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ language: python notifications: irc: "chat.freenode.net#pil" +env: MAX_CONCURRENCY=4 + python: - "pypy" - 2.6