Merge pull request #1291 from radarhere/winbuild_health

Style/health fixes
This commit is contained in:
Hugo van Kemenade 2015-06-21 17:42:01 +03:00
commit 05c482655c
11 changed files with 190 additions and 171 deletions

View File

@ -105,7 +105,6 @@ post-patch:
test_output = BytesIO()
im.save(test_output, "JPEG", qtables=qtables)
def test_exif_leak(self):
"""
pre patch:

View File

@ -233,6 +233,7 @@ def netpbm_available():
def imagemagick_available():
return IMCONVERT and command_succeeds([IMCONVERT, '-version'])
def on_appveyor():
return 'APPVEYOR' in os.environ

View File

@ -758,4 +758,3 @@ setup(
zip_safe=not debug_build(),
)
# End of file

View File

@ -2,11 +2,13 @@
import subprocess
import shutil
import sys, getopt
import sys
import getopt
import os
from config import *
def setup_vms():
ret = []
for py in pythons.keys():
@ -20,50 +22,53 @@ def setup_vms():
(VIRT_BASE, py, arch))
return "\n".join(ret)
def run_script(params):
(version, script) = params
try:
print ("Running %s" %version)
print("Running %s" % version)
filename = 'build_pillow_%s.cmd' % version
with open(filename, 'w') as f:
f.write(script)
command = ['powershell', "./%s" %filename]
proc = subprocess.Popen(command,
command = ['powershell', "./%s" % filename]
proc = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
(trace, stderr) = proc.communicate()
status = proc.returncode
print (stderr)
print ("Done with %s: %s" % (version, status))
print(stderr)
print("Done with %s: %s" % (version, status))
return (version, status, trace, stderr)
except Exception as msg:
print ("Error with %s: %s" % (version, str(msg)))
print("Error with %s: %s" % (version, str(msg)))
return (version, -1, "", str(msg))
def header(op):
return r"""
return r"""
setlocal
set MPLSRC=%%~dp0\..
set INCLIB=%%~dp0\depends
set BLDOPT=%s
cd /D %%MPLSRC%%
""" % (op)
""" % (op)
def footer():
return """endlocal
exit
"""
def build_one(py_ver, compiler):
# UNDONE virtual envs if we're not running on appveyor
args = {}
args.update(compiler)
if 'PYTHON' in os.environ:
args['python_path'] = "%PYTHON%"
args['python_path'] = "%PYTHON%"
else:
args['python_path'] = "%s%s\\Scripts" % (VIRT_BASE, py_ver)
args['py_ver'] = py_ver
@ -71,7 +76,7 @@ def build_one(py_ver, compiler):
args['tcl_ver'] = '86'
else:
args['tcl_ver'] = '85'
return r"""
setlocal EnableDelayedExpansion
call "%%ProgramFiles%%\Microsoft SDKs\Windows\%(env_version)s\Bin\SetEnv.Cmd" /Release %(env_flags)s
@ -87,6 +92,7 @@ endlocal
endlocal
""" % args
def clean():
try:
shutil.rmtree('../build')
@ -95,49 +101,48 @@ def clean():
pass
run_script(('virtualenvs', setup_vms()))
def main(op):
scripts = []
for py_version, compiler_version in pythons.items():
scripts.append((py_version,
scripts.append((py_version,
"\n".join([header(op),
build_one(py_version,
build_one(py_version,
compilers[(compiler_version, 32)]),
footer()])))
scripts.append(("%s%s" % (py_version, X64_EXT),
"\n".join([header(op),
build_one("%sx64" %py_version,
"\n".join([header(op),
build_one("%sx64" % py_version,
compilers[(compiler_version, 64)]),
footer()])))
results = map(run_script, scripts)
for (version, status, trace, err) in results:
print ("Compiled %s: %s" % (version, status and 'ERR' or 'OK'))
print("Compiled %s: %s" % (version, status and 'ERR' or 'OK'))
def run_one(op):
compiler = compiler_fromEnv()
py_version = pyversion_fromEnv()
run_script((py_version,
"\n".join([header(op),
build_one(py_version, compiler),
footer()])
))
))
if __name__=='__main__':
if __name__ == '__main__':
opts, args = getopt.getopt(sys.argv[1:], '', ['clean', 'dist'])
opts = dict(opts)
opts = dict(opts)
if '--clean' in opts:
clean()
op = 'install'
if '--dist' in opts:
op = "bdist_wininst --user-access-control=auto"

View File

@ -1,13 +1,15 @@
from fetch import fetch
from unzip import unzip
from untar import untar
import os, hashlib
import shutil
import os
import hashlib
from config import *
def _relpath(*args):
return os.path.join(os.getcwd(),*args)
return os.path.join(os.getcwd(), *args)
def _relbuild(*args):
return _relpath('build', *args)
@ -15,66 +17,65 @@ def _relbuild(*args):
build_dir = _relpath('build')
inc_dir = _relpath('depends')
def check_hash(filename, checksum):
if not checksum: return filename
if not checksum:
return filename
(algo, value) = checksum.split(':')
h = hashlib.new(algo)
with open(filename, 'rb') as f:
h.update(f.read())
if not(h.hexdigest().lower() == value):
raise ValueError('Checksum Mismatch for %s' %filename)
raise ValueError('Checksum Mismatch for %s' % filename)
return filename
def check_sig(filename, signame):
#UNDONE -- need gpg
# UNDONE -- need gpg
return filename
def mkdirs():
try:
os.mkdir(build_dir)
except:
except OSError:
pass
try:
os.mkdir(inc_dir)
except:
except OSError:
pass
for compiler in compilers.values():
try:
os.mkdir(os.path.join(inc_dir, compiler['inc_dir']))
except:
except OSError:
pass
def extract(src, dest):
if '.zip' in src:
return unzip(src, dest)
if '.tar.gz' in src or '.tgz' in src:
return untar(src, dest)
def fetch_libs():
for name,lib in libs.items():
for name, lib in libs.items():
if name == 'openjpeg':
filename = check_hash(fetch(lib['url']),lib['hash'])
filename = check_hash(fetch(lib['url']), lib['hash'])
for compiler in compilers.values():
if not os.path.exists(os.path.join(build_dir,lib['dir']+compiler['inc_dir'])):
if not os.path.exists(os.path.join(build_dir, lib['dir']+compiler['inc_dir'])):
extract(filename, build_dir)
os.rename(os.path.join(build_dir,lib['dir']),
os.path.join(build_dir,lib['dir']+compiler['inc_dir']))
os.rename(os.path.join(build_dir, lib['dir']),
os.path.join(build_dir, lib['dir']+compiler['inc_dir']))
else:
extract(check_hash(fetch(lib['url']),lib['hash']),build_dir)
extract(check_hash(fetch(lib['url']), lib['hash']), build_dir)
def extract_binlib():
lib = bin_libs['openjpeg']
extract(lib['filename'], build_dir)
return
base = os.path.splitext(lib['filename'])[0]
for compiler in compilers.values():
shutil.copy(os.path.join(inc_dir, base, 'include', 'openjpeg-%s' % lib['version']),
os.path.join(inc_dir, compiler['inc_dir']))
shutil.copy(os.path.join(inc_dir, base, 'bin', 'openjp2.dll'),
os.path.join(inc_dir, compiler['inc_dir']))
shutil.copy(os.path.join(inc_dir, base, 'lib', 'openjp2.lib'),
os.path.join(inc_dir, compiler['inc_dir']))
def extract_openjpeg(compiler):
return r"""
@ -88,7 +89,8 @@ copy /Y /B openjpeg-2.0.0-win32-x86\bin\ %%INCLIB%%
copy /Y /B openjpeg-2.0.0-win32-x86\lib\ %%INCLIB%%
endlocal
""" % compiler
def cp_tk():
return r"""
mkdir %INCLIB%\tcl85\include\X11
@ -102,26 +104,30 @@ copy /Y /B %BUILD%\tk8.6.4\generic\*.h %INCLIB%\tcl86\include\
copy /Y /B %BUILD%\tk8.6.4\xlib\X11\* %INCLIB%\tcl86\include\X11\
"""
def header():
return r"""setlocal
set MSBUILD=C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe
set CMAKE="cmake.exe"
set INCLIB=%~dp0\depends
set BUILD=%~dp0\build
""" + "\n".join('set %s=%%BUILD%%\%s' %(k.upper(), v['dir'])
for (k,v) in libs.items() if v['dir'])
""" + "\n".join('set %s=%%BUILD%%\%s' % (k.upper(), v['dir'])
for (k, v) in libs.items() if v['dir'])
def setup_compiler(compiler):
return r"""setlocal EnableDelayedExpansion
call "%%ProgramFiles%%\Microsoft SDKs\Windows\%(env_version)s\Bin\SetEnv.Cmd" /Release %(env_flags)s
set INCLIB=%%INCLIB%%\%(inc_dir)s
""" % compiler
def end_compiler():
return """
endlocal
"""
def nmake_openjpeg(compiler):
atts = {'op_ver': '2.1'}
atts.update(compiler)
@ -139,6 +145,7 @@ copy /Y /B src\lib\openjp2\*.h %%INCLIB%%\openjpeg-%(op_ver)s
endlocal
""" % atts
def msbuild_openjpeg(compiler):
atts = {'op_ver': '2.1'}
atts.update(compiler)
@ -158,7 +165,6 @@ endlocal
""" % atts
def nmake_libs(compiler):
# undone -- pre, makes, headers, libs
return r"""
@ -210,22 +216,25 @@ endlocal
""" % compiler
def msbuild_freetype(compiler):
if compiler['env_version'] == 'v7.1':
return msbuild_freetype_71(compiler)
return msbuild_freetype_70(compiler)
if compiler['env_version'] == 'v7.1':
return msbuild_freetype_71(compiler)
return msbuild_freetype_70(compiler)
def msbuild_freetype_71(compiler):
return r"""
rem Build freetype
setlocal
rd /S /Q %%FREETYPE%%\objs
%%MSBUILD%% %%FREETYPE%%\builds\windows\vc%(vc_version)s\freetype.sln /t:Clean;Build /p:Configuration="Release" /p:Platform=%(platform)s /m
%%MSBUILD%% %%FREETYPE%%\builds\windows\vc%(vc_version)s\freetype.sln /t:Clean;Build /p:Configuration="Release" /p:Platform=%(platform)s /m
xcopy /Y /E /Q %%FREETYPE%%\include %%INCLIB%%
copy /Y /B %%FREETYPE%%\objs\vc%(vc_version)s\%(platform)s\*.lib %%INCLIB%%\freetype.lib
endlocal
""" %compiler
""" % compiler
def msbuild_freetype_70(compiler):
return r"""
@ -234,22 +243,25 @@ setlocal
py -3 %%~dp0\fixproj.py %%FREETYPE%%\builds\windows\vc%(vc_version)s\freetype.sln %(platform)s
py -3 %%~dp0\fixproj.py %%FREETYPE%%\builds\windows\vc%(vc_version)s\freetype.vcproj %(platform)s
rd /S /Q %%FREETYPE%%\objs
%%MSBUILD%% %%FREETYPE%%\builds\windows\vc%(vc_version)s\freetype.sln /t:Clean;Build /p:Configuration="LIB Release";Platform=%(platform)s /m
%%MSBUILD%% %%FREETYPE%%\builds\windows\vc%(vc_version)s\freetype.sln /t:Clean;Build /p:Configuration="LIB Release";Platform=%(platform)s /m
xcopy /Y /E /Q %%FREETYPE%%\include %%INCLIB%%
xcopy /Y /E /Q %%FREETYPE%%\objs\win32\vc%(vc_version)s %%INCLIB%%
copy /Y /B %%FREETYPE%%\objs\win32\vc%(vc_version)s\*.lib %%INCLIB%%\freetype.lib
endlocal
""" %compiler
""" % compiler
def build_lcms2(compiler):
if compiler['env_version'] == 'v7.1':
return build_lcms_71(compiler)
return build_lcms_70(compiler)
if compiler['env_version'] == 'v7.1':
return build_lcms_71(compiler)
return build_lcms_70(compiler)
def build_lcms_70(compiler):
"""Link error here on x64"""
if compiler['platform'] == 'x64': return ''
if compiler['platform'] == 'x64':
return ''
"""Build LCMS on VC2008. This version is only 32bit/Win32"""
return r"""
rem Build lcms2
@ -263,6 +275,7 @@ copy /Y /B %%LCMS%%\Projects\VC%(vc_version)s\Release\*.lib %%INCLIB%%
endlocal
""" % compiler
def build_lcms_71(compiler):
return r"""
rem Build lcms2
@ -281,33 +294,27 @@ def add_compiler(compiler):
script.append(setup_compiler(compiler))
script.append(nmake_libs(compiler))
#script.append(extract_openjpeg(compiler))
# script.append(extract_openjpeg(compiler))
script.append(msbuild_freetype(compiler))
script.append(build_lcms2(compiler))
#script.append(nmake_openjpeg(compiler))
# script.append(nmake_openjpeg(compiler))
script.append(end_compiler())
mkdirs()
fetch_libs()
#extract_binlib()
# extract_binlib()
script = [header(), cp_tk()]
if 'PYTHON' in os.environ:
add_compiler(compiler_fromEnv())
else:
#for compiler in compilers.values():
#add_compiler(compiler)
add_compiler(compilers[(7.0,32)])
#add_compiler(compilers[(7.1,64)])
with open('build_deps.cmd', 'w') as f:
# for compiler in compilers.values():
# add_compiler(compiler)
add_compiler(compilers[(7.0, 32)])
# add_compiler(compilers[(7.1, 64)])
with open('build_deps.cmd', 'w') as f:
f.write("\n".join(script))

View File

@ -2,109 +2,110 @@ import os
SF_MIRROR = 'http://iweb.dl.sourceforge.net'
pythons = {#'26':7,
'27':7,
#'32':7,
'33':7.1,
'34':7.1}
pythons = {#'26': 7,
'27': 7,
#'32': 7,
'33': 7.1,
'34': 7.1}
VIRT_BASE = "c:/vp/"
X64_EXT = os.environ.get('X64_EXT',"x64")
X64_EXT = os.environ.get('X64_EXT', "x64")
libs = { 'zlib':{
'url':'http://zlib.net/zlib128.zip',
libs = {'zlib': {
'url': 'http://zlib.net/zlib128.zip',
'hash': 'md5:126f8676442ffbd97884eb4d6f32afb4',
'dir': 'zlib-1.2.8',
},
'jpeg':{
'url':'http://www.ijg.org/files/jpegsr9a.zip',
'hash': 'md5:a34f3c82760270ee1e1885b15b90a72e', # not found - generated by wiredfool
'jpeg': {
'url': 'http://www.ijg.org/files/jpegsr9a.zip',
'hash': 'md5:a34f3c82760270ee1e1885b15b90a72e', # not found - generated by wiredfool
'dir': 'jpeg-9a',
},
'tiff':{
'url':'ftp://ftp.remotesensing.org/pub/libtiff/tiff-4.0.3.zip',
'hash': 'md5:dd70349cedb3981371686e1c9b89a7f9', # not found - generated by wiredfool
'tiff': {
'url': 'ftp://ftp.remotesensing.org/pub/libtiff/tiff-4.0.3.zip',
'hash': 'md5:dd70349cedb3981371686e1c9b89a7f9', # not found - generated by wiredfool
'dir': 'tiff-4.0.3',
},
'freetype':{
'url': 'http://download.savannah.gnu.org/releases/freetype/freetype-2.6.tar.gz',
'hash':'md5:1d733ea6c1b7b3df38169fbdbec47d2b',
'dir': 'freetype-2.6',
'freetype': {
'url': 'http://download.savannah.gnu.org/releases/freetype/freetype-2.6.tar.gz',
'hash': 'md5:1d733ea6c1b7b3df38169fbdbec47d2b',
'dir': 'freetype-2.6',
},
'lcms':{
'url':SF_MIRROR+'/project/lcms/lcms/2.7/lcms2-2.7.zip',
'lcms': {
'url': SF_MIRROR+'/project/lcms/lcms/2.7/lcms2-2.7.zip',
'hash': 'sha1:7ff1a5b721ca719760ba6eb4ec6f38d5e65381cf',
'dir': 'lcms2-2.7',
},
'tcl-8.5':{
'url':SF_MIRROR+'/project/tcl/Tcl/8.5.13/tcl8513-src.zip',
'tcl-8.5': {
'url': SF_MIRROR+'/project/tcl/Tcl/8.5.13/tcl8513-src.zip',
'hash': 'sha1:3e01585c91293c532a3cd594ec59deca92153a5e',
'dir': '',
},
'tk-8.5':{
'url':SF_MIRROR+'/project/tcl/Tcl/8.5.13/tk8513-src.zip',
'tk-8.5': {
'url': SF_MIRROR+'/project/tcl/Tcl/8.5.13/tk8513-src.zip',
'hash': 'sha1:23a1d7ddd416e11e06dfdb9f86111d4bab9420b4',
'dir': '',
},
'tcl-8.6':{
'url':SF_MIRROR+'/project/tcl/Tcl/8.6.4/tcl864-src.zip',
'tcl-8.6': {
'url': SF_MIRROR+'/project/tcl/Tcl/8.6.4/tcl864-src.zip',
'hash': 'md5:35748d2fc61e08a2fdb23b85c6f8c4a0',
'dir': '',
},
'tk-8.6':{
'url':SF_MIRROR+'/project/tcl/Tcl/8.6.4/tk864-src.zip',
'tk-8.6': {
'url': SF_MIRROR+'/project/tcl/Tcl/8.6.4/tk864-src.zip',
'hash': 'md5:111d45061a69e7f5250b6ec8ca7c4f35',
'dir': '',
},
'webp':{
'url':'http://downloads.webmproject.org/releases/webp/libwebp-0.4.3.tar.gz',
'hash':'sha1:1c307a61c4d0018620b4ba9a58e8f48a8d6640ef',
'dir':'libwebp-0.4.3',
'webp': {
'url': 'http://downloads.webmproject.org/releases/webp/libwebp-0.4.3.tar.gz',
'hash': 'sha1:1c307a61c4d0018620b4ba9a58e8f48a8d6640ef',
'dir': 'libwebp-0.4.3',
},
'openjpeg':{
'url':SF_MIRROR+'/project/openjpeg/openjpeg/2.1.0/openjpeg-2.1.0.tar.gz',
'hash':'md5:f6419fcc233df84f9a81eb36633c6db6',
'dir':'openjpeg-2.1.0',
'openjpeg': {
'url': SF_MIRROR+'/project/openjpeg/openjpeg/2.1.0/openjpeg-2.1.0.tar.gz',
'hash': 'md5:f6419fcc233df84f9a81eb36633c6db6',
'dir': 'openjpeg-2.1.0',
},
}
bin_libs = {
'openjpeg':{
'filename':'openjpeg-2.0.0-win32-x86.zip',
'hash':'sha1:xxx',
'version':'2.0'
},
'openjpeg': {
'filename': 'openjpeg-2.0.0-win32-x86.zip',
'hash': 'sha1:xxx',
'version': '2.0'
},
}
compilers = { (7,64): {
'env_version':'v7.0',
'vc_version':'2008',
compilers = {(7, 64): {
'env_version': 'v7.0',
'vc_version': '2008',
'env_flags': '/x64 /xp',
'inc_dir': 'msvcr90-x64',
'platform': 'x64',
'webp_platform': 'x64',
},
(7,32): {
'env_version':'v7.0',
'vc_version':'2008',
(7, 32): {
'env_version': 'v7.0',
'vc_version': '2008',
'env_flags': '/x86 /xp',
'inc_dir': 'msvcr90-x32',
'platform': 'Win32',
'webp_platform': 'x86',
},
(7.1,64): {
'env_version':'v7.1',
'vc_version':'2010',
(7.1, 64): {
'env_version': 'v7.1',
'vc_version': '2010',
'env_flags': '/x64 /vista',
'inc_dir': 'msvcr10-x64',
'platform': 'x64',
'webp_platform': 'x64',
},
(7.1,32): {
'env_version':'v7.1',
'vc_version':'2010',
(7.1, 32): {
'env_version': 'v7.1',
'vc_version': '2010',
'env_flags': '/x86 /vista',
'inc_dir': 'msvcr10-x32',
'platform': 'Win32',
@ -118,7 +119,7 @@ def pyversion_fromEnv():
py = os.environ['PYTHON']
py_version = '27'
for k,v in pythons.items():
for k in pythons.keys():
if k in py:
py_version = k
break
@ -128,11 +129,11 @@ def pyversion_fromEnv():
return py_version
def compiler_fromEnv():
py = os.environ['PYTHON']
for k,v in pythons.items():
for k, v in pythons.items():
if k in py:
compiler_version = v
break

View File

@ -1,4 +1,8 @@
import sys, os, urllib.parse, urllib.request
import sys
import os
import urllib.parse
import urllib.request
def fetch(url):
name = urllib.parse.urlsplit(url)[2].split('/')[-1]
@ -10,7 +14,5 @@ def fetch(url):
fd.write(content)
return name
if __name__=='__main__':
if __name__ == '__main__':
fetch(sys.argv[1])

View File

@ -1,11 +1,11 @@
from fetch import fetch
import os
if __name__=='__main__':
if __name__ == '__main__':
for version in ['2.6.5', '2.7.6', '3.2.5', '3.3.5', '3.4.3']:
for platform in ['', '.amd64']:
for extension in ['','.asc']:
fetch('https://www.python.org/ftp/python/%s/python-%s%s.msi%s' %(
for extension in ['', '.asc']:
fetch('https://www.python.org/ftp/python/%s/python-%s%s.msi%s' % (
version, version, platform, extension))
# find pip, if it's not in the path!

View File

@ -7,38 +7,39 @@ import sys
from config import *
def test_one(params):
python, architecture = params
try:
print ("Running: %s, %s" %params)
print("Running: %s, %s" % params)
command = [r'%s\%s%s\Scripts\python.exe' % (VIRT_BASE, python, architecture),
'test-installed.py',
'--processes=-0',
'--process-timeout=30',
]
command.extend(glob.glob('Tests/test*.py'))
proc = subprocess.Popen(command,
proc = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
(trace, stderr) = proc.communicate()
status = proc.returncode
print ("Done with %s, %s -- %s" % (python, architecture, status ))
print("Done with %s, %s -- %s" % (python, architecture, status))
return (python, architecture, status, trace)
except Exception as msg:
print ("Error with %s, %s: %s" % (python, architecture, msg))
print("Error with %s, %s: %s" % (python, architecture, msg))
return (python, architecture, -1, str(msg))
if __name__=='__main__':
if __name__ == '__main__':
os.chdir('..')
matrix = [(python, architecture) for python in pythons
for architecture in ('', X64_EXT)]
matrix = [(python, architecture) for python in pythons
for architecture in ('', X64_EXT)]
results = map(test_one, matrix)
for (python, architecture, status, trace) in results:
print ("%s%s: %s" % (python, architecture, status and 'ERR' or 'PASS'))
print("%s%s: %s" % (python, architecture, status and 'ERR' or 'PASS'))
res = all(status for (python, architecture, status, trace) in results)
sys.exit(res)

View File

@ -1,8 +1,10 @@
import sys, tarfile
import sys
import tarfile
def untar(src, dest):
with tarfile.open(src, 'r:gz') as tgz:
tgz.extractall(dest)
if __name__=='__main__':
untar(sys.argv[1],sys.argv[2])
if __name__ == '__main__':
untar(sys.argv[1], sys.argv[2])

View File

@ -1,8 +1,10 @@
import sys, zipfile
import sys
import zipfile
def unzip(src, dest):
with zipfile.ZipFile(src) as zf:
zf.extractall(dest)
with zipfile.ZipFile(src) as zf:
zf.extractall(dest)
if __name__=='__main__':
unzip(sys.argv[1], sys.argv[2])
if __name__ == '__main__':
unzip(sys.argv[1], sys.argv[2])