Pillow/mp_compile.py

94 lines
2.7 KiB
Python
Raw Normal View History

2014-06-25 02:57:24 +04:00
# A monkey patch of the base distutils.ccompiler to use parallel builds
# Tested on 2.7, looks to be identical to 3.3.
2019-01-13 22:55:07 +03:00
# Only applied on Python 2.7 because otherwise, it conflicts with Python's
# own newly-added support for parallel builds.
2014-06-25 02:57:24 +04:00
from __future__ import print_function
2015-04-24 03:15:14 +03:00
import os
import sys
from distutils.ccompiler import CCompiler
from multiprocessing import Pool, cpu_count
2014-06-25 02:57:24 +04:00
try:
2019-06-13 18:54:57 +03:00
MAX_PROCS = int(os.environ.get("MAX_CONCURRENCY", min(4, cpu_count())))
2014-12-08 22:15:44 +03:00
except NotImplementedError:
MAX_PROCS = None
2014-08-28 15:44:19 +04:00
2014-06-28 03:13:00 +04:00
2014-06-25 02:57:24 +04:00
# hideous monkeypatching. but. but. but.
def _mp_compile_one(tp):
(self, obj, build, cc_args, extra_postargs, pp_opts) = tp
try:
src, ext = build[obj]
except KeyError:
return
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
return
2014-06-28 03:13:00 +04:00
2019-06-13 18:54:57 +03:00
def _mp_compile(
self,
sources,
output_dir=None,
macros=None,
include_dirs=None,
debug=0,
extra_preargs=None,
extra_postargs=None,
depends=None,
):
2014-06-25 02:57:24 +04:00
"""Compile one or more source files.
2014-06-28 03:13:00 +04:00
2014-06-25 02:57:24 +04:00
see distutils.ccompiler.CCompiler.compile for comments.
"""
# A concrete compiler class can either override this method
# entirely or implement _compile().
2014-06-28 03:13:00 +04:00
macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
2019-06-13 18:54:57 +03:00
output_dir, macros, include_dirs, sources, depends, extra_postargs
)
2014-06-28 03:13:00 +04:00
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
2014-06-25 02:57:24 +04:00
pool = Pool(MAX_PROCS)
2014-06-25 02:57:24 +04:00
try:
2015-04-24 03:15:14 +03:00
print("Building using %d processes" % pool._processes)
except Exception:
2014-06-28 03:13:00 +04:00
pass
2019-06-13 18:54:57 +03:00
arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects]
2014-06-28 03:13:00 +04:00
pool.map_async(_mp_compile_one, arr)
2014-06-25 02:57:24 +04:00
pool.close()
pool.join()
# Return *all* object filenames, not just the ones we just built.
return objects
def install():
2019-06-13 18:54:57 +03:00
fl_win = sys.platform.startswith("win")
fl_cygwin = sys.platform.startswith("cygwin")
2015-09-11 12:28:19 +03:00
if fl_win or fl_cygwin:
# Windows barfs on multiprocessing installs
print("Single threaded build for Windows")
return
2015-09-11 12:28:19 +03:00
if MAX_PROCS != 1:
# explicitly don't enable if environment says 1 processor
try:
# bug, only enable if we can make a Pool. see issue #790 and
2017-02-14 12:27:02 +03:00
# https://stackoverflow.com/questions/6033599/oserror-38-errno-38-with-multiprocessing
2017-03-24 12:43:03 +03:00
Pool(2)
CCompiler.compile = _mp_compile
except Exception as msg:
2019-06-13 18:54:57 +03:00
print("Exception installing mp_compile, proceeding without: %s" % msg)
else:
2019-06-13 18:54:57 +03:00
print(
"Single threaded build, not installing mp_compile: %s processes" % MAX_PROCS
)
2019-01-13 22:55:07 +03:00
# We monkeypatch Python 2.7
if sys.version_info.major < 3:
install()