mirror of
https://github.com/python-pillow/Pillow.git
synced 2025-08-05 04:50:09 +03:00
Merge branch 'multiprocessing' of github.com:wiredfool/Pillow into multiprocessing
This commit is contained in:
wiredfool
commit
bbd68ea9d4
156
Tests/run.py
156
Tests/run.py
|
|
@ -3,6 +3,7 @@ from __future__ import print_function
|
|||
# minimal test runner
|
||||
|
||||
import glob, os, os.path, sys, tempfile
|
||||
from multiprocessing import Pool
|
||||
|
||||
try:
|
||||
root = os.path.dirname(__file__)
|
||||
|
|
@ -14,83 +15,110 @@ if not os.path.isfile("PIL/Image.py"):
|
|||
print("***", "$ python Tests/run.py")
|
||||
sys.exit(1)
|
||||
|
||||
print("-"*68)
|
||||
|
||||
python_options = []
|
||||
tester_options = []
|
||||
|
||||
if "--installed" not in sys.argv:
|
||||
os.environ["PYTHONPATH"] = "."
|
||||
|
||||
if "--coverage" in sys.argv:
|
||||
tester_options.append("--coverage")
|
||||
|
||||
if "--log" in sys.argv:
|
||||
tester_options.append("--log")
|
||||
|
||||
files = glob.glob(os.path.join(root, "test_*.py"))
|
||||
files.sort()
|
||||
|
||||
success = failure = 0
|
||||
include = [x for x in sys.argv[1:] if x[:2] != "--"]
|
||||
skipped = []
|
||||
|
||||
python_options = " ".join(python_options)
|
||||
tester_options = " ".join(tester_options)
|
||||
|
||||
for file in files:
|
||||
test, ext = os.path.splitext(os.path.basename(file))
|
||||
if include and test not in include:
|
||||
continue
|
||||
def test_one(f):
|
||||
test, ext = os.path.splitext(os.path.basename(f))
|
||||
print("running", test, "...")
|
||||
# 2>&1 works on unix and on modern windowses. we might care about
|
||||
# very old Python versions, but not ancient microsoft products :-)
|
||||
out = os.popen("%s %s -u %s %s 2>&1" % (
|
||||
sys.executable, python_options, file, tester_options
|
||||
sys.executable, python_options, f, tester_options
|
||||
))
|
||||
result = out.read().strip()
|
||||
if result == "ok":
|
||||
result = None
|
||||
elif result == "skip":
|
||||
print("---", "skipped") # FIXME: driver should include a reason
|
||||
skipped.append(test)
|
||||
continue
|
||||
elif not result:
|
||||
result = "(no output)"
|
||||
status = out.close()
|
||||
if status or result:
|
||||
if status:
|
||||
print("=== error", status)
|
||||
if result:
|
||||
if result[-3:] == "\nok":
|
||||
# if there's an ok at the end, it's not really ok
|
||||
result = result[:-3]
|
||||
print(result)
|
||||
failure = failure + 1
|
||||
else:
|
||||
success = success + 1
|
||||
|
||||
print("-"*68)
|
||||
return (result, status)
|
||||
|
||||
temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests')
|
||||
tempfiles = glob.glob(os.path.join(temp_root, "temp_*"))
|
||||
if tempfiles:
|
||||
print("===", "remaining temporary files")
|
||||
for file in tempfiles:
|
||||
print(file)
|
||||
def filter_tests(files):
|
||||
ret = []
|
||||
for f in files:
|
||||
test, ext = os.path.splitext(os.path.basename(f))
|
||||
if include and test not in include:
|
||||
continue
|
||||
ret.append(f)
|
||||
return ret
|
||||
|
||||
def main():
|
||||
global python_options, tester_options
|
||||
|
||||
print("-"*68)
|
||||
|
||||
def tests(n):
|
||||
if n == 1:
|
||||
return "1 test"
|
||||
else:
|
||||
return "%d tests" % n
|
||||
if "--installed" not in sys.argv:
|
||||
os.environ["PYTHONPATH"] = "."
|
||||
|
||||
if skipped:
|
||||
print("---", tests(len(skipped)), "skipped.")
|
||||
print(skipped)
|
||||
if failure:
|
||||
print("***", tests(failure), "of", (success + failure), "failed.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(tests(success), "passed.")
|
||||
if "--coverage" in sys.argv:
|
||||
tester_options.append("--coverage")
|
||||
|
||||
if "--log" in sys.argv:
|
||||
tester_options.append("--log")
|
||||
|
||||
files = glob.glob(os.path.join(root, "test_*.py"))
|
||||
files.sort()
|
||||
|
||||
success = failure = 0
|
||||
skipped = []
|
||||
|
||||
python_options = " ".join(python_options)
|
||||
tester_options = " ".join(tester_options)
|
||||
|
||||
|
||||
files = filter_tests(files)
|
||||
|
||||
pool = Pool()
|
||||
results = pool.map(test_one, files)
|
||||
pool.close()
|
||||
pool.join()
|
||||
|
||||
for test,(result, status) in zip(files,results):
|
||||
if result == "ok":
|
||||
result = None
|
||||
elif result == "skip":
|
||||
print("---", "skipped") # FIXME: driver should include a reason
|
||||
skipped.append(test)
|
||||
continue
|
||||
elif not result:
|
||||
result = "(no output)"
|
||||
if status or result:
|
||||
if status:
|
||||
print("=== error", status)
|
||||
if result:
|
||||
if result[-3:] == "\nok":
|
||||
# if there's an ok at the end, it's not really ok
|
||||
result = result[:-3]
|
||||
print(result)
|
||||
failure = failure + 1
|
||||
else:
|
||||
success = success + 1
|
||||
|
||||
print("-"*68)
|
||||
|
||||
temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests')
|
||||
tempfiles = glob.glob(os.path.join(temp_root, "temp_*"))
|
||||
if tempfiles:
|
||||
print("===", "remaining temporary files")
|
||||
for file in tempfiles:
|
||||
print(file)
|
||||
print("-"*68)
|
||||
|
||||
def tests(n):
|
||||
if n == 1:
|
||||
return "1 test"
|
||||
else:
|
||||
return "%d tests" % n
|
||||
|
||||
if skipped:
|
||||
print("---", tests(len(skipped)), "skipped.")
|
||||
print(skipped)
|
||||
if failure:
|
||||
print("***", tests(failure), "of", (success + failure), "failed.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(tests(success), "passed.")
|
||||
|
||||
return 0
|
||||
|
||||
if __name__=='__main__':
|
||||
sys.exit(main())
|
||||
|
|
|
|||
47
mp_compile.py
Normal file
47
mp_compile.py
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# A monkey patch of the base distutils.ccompiler to use parallel builds
|
||||
# Tested on 2.7, looks to be identical to 3.3.
|
||||
|
||||
from multiprocessing import Pool
|
||||
from distutils.ccompiler import CCompiler
|
||||
|
||||
# hideous monkeypatching. but. but. but.
|
||||
def _mp_compile_one(tp):
|
||||
(self, obj, build, cc_args, extra_postargs, pp_opts) = tp
|
||||
try:
|
||||
src, ext = build[obj]
|
||||
except KeyError:
|
||||
return
|
||||
self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
|
||||
return
|
||||
|
||||
def _mp_compile(self, sources, output_dir=None, macros=None,
|
||||
include_dirs=None, debug=0, extra_preargs=None,
|
||||
extra_postargs=None, depends=None):
|
||||
"""Compile one or more source files.
|
||||
|
||||
see distutils.ccompiler.CCompiler.compile for comments.
|
||||
"""
|
||||
# A concrete compiler class can either override this method
|
||||
# entirely or implement _compile().
|
||||
|
||||
macros, objects, extra_postargs, pp_opts, build = \
|
||||
self._setup_compile(output_dir, macros, include_dirs, sources,
|
||||
depends, extra_postargs)
|
||||
cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
|
||||
|
||||
try:
|
||||
processes = cpu_count() * 2
|
||||
except:
|
||||
processes = 2
|
||||
|
||||
pool = Pool(processes)
|
||||
print ("Building using %d processes" % pool._processes)
|
||||
arr = [(self, obj, build, cc_args, extra_postargs, pp_opts) for obj in objects]
|
||||
results = pool.map_async(_mp_compile_one,arr)
|
||||
|
||||
pool.close()
|
||||
pool.join()
|
||||
# Return *all* object filenames, not just the ones we just built.
|
||||
return objects
|
||||
|
||||
CCompiler.compile = _mp_compile
|
||||
69
setup.py
69
setup.py
|
|
@ -14,6 +14,8 @@ import re
|
|||
import struct
|
||||
import sys
|
||||
|
||||
import mp_compile
|
||||
|
||||
from distutils.command.build_ext import build_ext
|
||||
from distutils import sysconfig
|
||||
from setuptools import Extension, setup, find_packages
|
||||
|
|
@ -615,38 +617,41 @@ class pil_build_ext(build_ext):
|
|||
finally:
|
||||
os.unlink(tmpfile)
|
||||
|
||||
setup(
|
||||
name=NAME,
|
||||
version=VERSION,
|
||||
description='Python Imaging Library (Fork)',
|
||||
long_description=(
|
||||
# monkeypatch the compiler
|
||||
|
||||
if __name__=='__main__':
|
||||
setup(
|
||||
name=NAME,
|
||||
version=VERSION,
|
||||
description='Python Imaging Library (Fork)',
|
||||
long_description=(
|
||||
_read('README.rst') + b'\n' +
|
||||
_read('CHANGES.rst')).decode('utf-8'),
|
||||
author='Alex Clark (fork author)',
|
||||
author_email='aclark@aclark.net',
|
||||
url='http://python-imaging.github.io/',
|
||||
classifiers=[
|
||||
"Development Status :: 6 - Mature",
|
||||
"Topic :: Multimedia :: Graphics",
|
||||
"Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
|
||||
"Topic :: Multimedia :: Graphics :: Capture :: Scanners",
|
||||
"Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
|
||||
"Topic :: Multimedia :: Graphics :: Graphics Conversion",
|
||||
"Topic :: Multimedia :: Graphics :: Viewers",
|
||||
"Programming Language :: Python :: 2",
|
||||
"Programming Language :: Python :: 2.6",
|
||||
"Programming Language :: Python :: 2.7",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.2",
|
||||
"Programming Language :: Python :: 3.3", ],
|
||||
cmdclass={"build_ext": pil_build_ext},
|
||||
ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
|
||||
include_package_data=True,
|
||||
packages=find_packages(),
|
||||
scripts=glob.glob("Scripts/pil*.py"),
|
||||
test_suite='PIL.tests',
|
||||
keywords=["Imaging",],
|
||||
license='Standard PIL License',
|
||||
zip_safe=True,
|
||||
)
|
||||
author='Alex Clark (fork author)',
|
||||
author_email='aclark@aclark.net',
|
||||
url='http://python-imaging.github.io/',
|
||||
classifiers=[
|
||||
"Development Status :: 6 - Mature",
|
||||
"Topic :: Multimedia :: Graphics",
|
||||
"Topic :: Multimedia :: Graphics :: Capture :: Digital Camera",
|
||||
"Topic :: Multimedia :: Graphics :: Capture :: Scanners",
|
||||
"Topic :: Multimedia :: Graphics :: Capture :: Screen Capture",
|
||||
"Topic :: Multimedia :: Graphics :: Graphics Conversion",
|
||||
"Topic :: Multimedia :: Graphics :: Viewers",
|
||||
"Programming Language :: Python :: 2",
|
||||
"Programming Language :: Python :: 2.6",
|
||||
"Programming Language :: Python :: 2.7",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.2",
|
||||
"Programming Language :: Python :: 3.3", ],
|
||||
cmdclass={"build_ext": pil_build_ext},
|
||||
ext_modules=[Extension("PIL._imaging", ["_imaging.c"])],
|
||||
include_package_data=True,
|
||||
packages=find_packages(),
|
||||
scripts=glob.glob("Scripts/pil*.py"),
|
||||
test_suite='PIL.tests',
|
||||
keywords=["Imaging",],
|
||||
license='Standard PIL License',
|
||||
zip_safe=True,
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user