python-pillow/Pillow
1
1
Fork 0
mirror of https://github.com/python-pillow/Pillow.git synced 2025-07-22 14:10:06 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #791 from wiredfool/multiprocessing_fail

Browse Source 
Disable mp_compile when Multiprocess.Pool fails
This commit is contained in:
Alex Clark ☺ 2014-07-09 13:29:50 -04:00
commit 878a4e194a
2 changed files with 22 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -15,7 +15,9 @@ pre:
clean: clean:
python setup.py clean python setup.py clean
rm PIL/*.so || true rm PIL/*.so || true
find . -name __pycache__ | xargs rm -r rm -r build || true
find . -name __pycache__ | xargs rm -r || true
install: install:
python setup.py install python setup.py install

28
mp_compile.py
View File

@ -5,6 +5,11 @@ from multiprocessing import Pool, cpu_count
from distutils.ccompiler import CCompiler from distutils.ccompiler import CCompiler
import os import os
try:
MAX_PROCS = int(os.environ.get('MAX_CONCURRENCY', cpu_count()))
except:
MAX_PROCS = None
# hideous monkeypatching. but. but. but. # hideous monkeypatching. but. but. but.
def _mp_compile_one(tp): def _mp_compile_one(tp):
@ -31,22 +36,27 @@ def _mp_compile(self, sources, output_dir=None, macros=None,
output_dir, macros, include_dirs, sources, depends, extra_postargs) output_dir, macros, include_dirs, sources, depends, extra_postargs)
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
try: pool = Pool(MAX_PROCS)
max_procs = int(os.environ.get('MAX_CONCURRENCY', cpu_count()))
except:
max_procs = None
pool = Pool(max_procs)
try: try:
print ("Building using %d processes" % pool._processes) print ("Building using %d processes" % pool._processes)
except: except:
pass pass
arr = [ arr = [(self, obj, build, cc_args, extra_postargs, pp_opts)
(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects for obj in objects]
]
pool.map_async(_mp_compile_one, arr) pool.map_async(_mp_compile_one, arr)
pool.close() pool.close()
pool.join() pool.join()
# Return *all* object filenames, not just the ones we just built. # Return *all* object filenames, not just the ones we just built.
return objects return objects
CCompiler.compile = _mp_compile # explicitly don't enable if environment says 1 processor
if MAX_PROCS != 1:
try:
# bug, only enable if we can make a Pool. see issue #790 and
# http://stackoverflow.com/questions/6033599/oserror-38-errno-38-with-multiprocessing
pool = Pool(2)
CCompiler.compile = _mp_compile
except Exception as msg:
print("Exception installing mp_compile, proceeding without: %s" %msg)
else:
print("Single threaded build, not installing mp_compile: %s processes" %MAX_PROCS)