Multithreaded build

This commit is contained in:
wiredfool 2014-06-24 15:57:24 -07:00
parent 964935fb75
commit 2be4e9f3e5
2 changed files with 87 additions and 34 deletions

50
mp_compile.py Normal file
View File

@ -0,0 +1,50 @@
# A monkey patch of the base distutils.ccompiler to use parallel builds
# Tested on 2.7, looks to be identical to 3.3.
from multiprocessing import Pool, cpu_count
from distutils.ccompiler import CCompiler
import os
# hideous monkeypatching. but. but. but.
def _mp_compile_one(tp):
(self, obj, build, cc_args, extra_postargs, pp_opts) = tp
try:
src, ext = build[obj]
except KeyError:
return
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
return
def _mp_compile(self, sources, output_dir=None, macros=None,
include_dirs=None, debug=0, extra_preargs=None,
extra_postargs=None, depends=None):
"""Compile one or more source files.
see distutils.ccompiler.CCompiler.compile for comments.
"""
# A concrete compiler class can either override this method
# entirely or implement _compile().
macros, objects, extra_postargs, pp_opts, build = \
self._setup_compile(output_dir, macros, include_dirs, sources,
depends, extra_postargs)
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
try:
max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count()))
except:
max_procs = None
pool = Pool(max_procs)
try:
print ("Building using %d processes" % pool._processes)
except: pass
arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects]
results = pool.map_async(_mp_compile_one,arr)
pool.close()
pool.join()
# Return *all* object filenames, not just the ones we just built.
return objects
CCompiler.compile = _mp_compile

View File

@ -14,6 +14,8 @@ import re
import struct
import sys
import mp_compile
from distutils.command.build_ext import build_ext
from distutils import sysconfig
from setuptools import Extension, setup, find_packages
@ -705,39 +707,40 @@ class pil_build_ext(build_ext):
finally:
os.unlink(tmpfile)
setup(
name=NAME,
version=VERSION,
description='Python Imaging Library (Fork)',
long_description=(
_read('README.rst') + b'\n' +
_read('CHANGES.rst')).decode('utf-8'),
author='Alex Clark (fork author)',
author_email='aclark@aclark.net',
url='http://python-pillow.github.io/',
classifiers=[
"Development Status :: 6 - Mature",
"Topic :: Multimedia :: Graphics",
"Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
"Topic :: Multimedia :: Graphics :: Capture :: Scanners",
"Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
"Topic :: Multimedia :: Graphics :: Graphics Conversion",
"Topic :: Multimedia :: Graphics :: Viewers",
"Programming Language :: Python :: 2",
"Programming Language :: Python :: 2.6",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.2",
"Programming Language :: Python :: 3.3", ],
cmdclass={"build_ext": pil_build_ext},
ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
include_package_data=True,
packages=find_packages(),
scripts=glob.glob("Scripts/pil*.py"),
test_suite='PIL.tests',
keywords=["Imaging", ],
license='Standard PIL License',
zip_safe=True,
if __name__=='__main__':
setup(
name=NAME,
version=VERSION,
description='Python Imaging Library (Fork)',
long_description=(
_read('README.rst') + b'\n' +
_read('CHANGES.rst')).decode('utf-8'),
author='Alex Clark (fork author)',
author_email='aclark@aclark.net',
url='http://python-pillow.github.io/',
classifiers=[
"Development Status :: 6 - Mature",
"Topic :: Multimedia :: Graphics",
"Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
"Topic :: Multimedia :: Graphics :: Capture :: Scanners",
"Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
"Topic :: Multimedia :: Graphics :: Graphics Conversion",
"Topic :: Multimedia :: Graphics :: Viewers",
"Programming Language :: Python :: 2",
"Programming Language :: Python :: 2.6",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.2",
"Programming Language :: Python :: 3.3", ],
cmdclass={"build_ext": pil_build_ext},
ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
include_package_data=True,
packages=find_packages(),
scripts=glob.glob("Scripts/pil*.py"),
test_suite='PIL.tests',
keywords=["Imaging", ],
license='Standard PIL License',
zip_safe=True,
)
# End of file