From 89e05cdf56fa3dd93df9c4301f8bd88399534080 Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 2 Oct 2019 19:48:55 +0200 Subject: [PATCH 01/27] extract scripts from test-windows.yml (cherry-picked from commit 1a6ee35d0e5fa1bf4081649a6ebf67208d76f322) --- .github/workflows/test-windows.yml | 204 ++---------- setup.py | 2 +- winbuild/build_prepare.py | 494 +++++++++++++++++++++++++++++ winbuild/commands.py | 111 +++++++ 4 files changed, 634 insertions(+), 177 deletions(-) create mode 100644 winbuild/build_prepare.py create mode 100644 winbuild/commands.py diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index adbe97968..6425f3b5c 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -4,7 +4,6 @@ on: [push, pull_request] jobs: build: - runs-on: windows-2019 strategy: fail-fast: false @@ -70,155 +69,28 @@ jobs: Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" $env:PYTHON=$env:pythonLocation - xcopy ..\pillow-depends\*.zip $env:GITHUB_WORKSPACE\winbuild\ - xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\ + mkdir $env:GITHUB_WORKSPACE\winbuild\depends\ + xcopy ..\pillow-depends\*.zip $env:GITHUB_WORKSPACE\winbuild\depends\ + xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\depends\ xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\ cd $env:GITHUB_WORKSPACE/winbuild/ - python.exe $env:GITHUB_WORKSPACE\winbuild\build_dep.py - env: - EXECUTABLE: bin\python.exe + python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py shell: pwsh - name: Build dependencies / libjpeg - if: false - run: | - REM FIXME uses /MT not /MD, see makefile.vc and win32.mak for more info - - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\jpeg-9d - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - nmake -nologo -f makefile.vc setup-vc6 - nmake -nologo -f makefile.vc clean - nmake -nologo -f makefile.vc nodebug=1 libjpeg.lib cjpeg.exe djpeg.exe - copy /Y /B j*.h %INCLIB% - copy /Y /B *.lib %INCLIB% - copy /Y /B *.exe %INCLIB% - shell: cmd - - - name: Build dependencies / libjpeg-turbo - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\libjpeg-turbo-2.0.3 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DENABLE_SHARED:BOOL=OFF -DWITH_JPEG8:BOOL=TRUE -DWITH_CRT_DLL:BOOL=TRUE -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile jpeg-static cjpeg-static djpeg-static - copy /Y /B j*.h %INCLIB% - copy /Y /B jpeg-static.lib %INCLIB%\libjpeg.lib - copy /Y /B cjpeg-static.exe %INCLIB%\cjpeg.exe - copy /Y /B djpeg-static.exe %INCLIB%\djpeg.exe - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libjpeg.cmd" - name: Build dependencies / zlib - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\zlib-1.2.11 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - nmake -nologo -f win32\Makefile.msc clean - nmake -nologo -f win32\Makefile.msc zlib.lib - copy /Y /B z*.h %INCLIB% - copy /Y /B *.lib %INCLIB% - copy /Y /B zlib.lib %INCLIB%\z.lib - shell: cmd - - - name: Build dependencies / LibTIFF - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\tiff-4.1.0 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - copy %GITHUB_WORKSPACE%\winbuild\tiff.opt nmake.opt - nmake -nologo -f makefile.vc clean - nmake -nologo -f makefile.vc lib - copy /Y /B libtiff\tiff*.h %INCLIB% - copy /Y /B libtiff\*.dll %INCLIB% - copy /Y /B libtiff\*.lib %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_zlib.cmd" + - name: Build dependencies / LibTiff + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libtiff.cmd" - name: Build dependencies / WebP - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\libwebp-1.1.0 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - rmdir /S /Q output\release-static - nmake -nologo -f Makefile.vc CFG=release-static OBJDIR=output ARCH=${{ matrix.architecture }} all - mkdir %INCLIB%\webp - copy /Y /B src\webp\*.h %INCLIB%\webp - copy /Y /B output\release-static\${{ matrix.architecture }}\lib\* %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libwebp.cmd" - name: Build dependencies / FreeType - run: | - REM Toolkit v100 not available; missing VCTargetsPath; Clean fails - - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\freetype-2.10.1 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - rmdir /S /Q objs - set DefaultPlatformToolset=v142 - set VCTargetsPath=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\ - set MSBUILD="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe" - powershell -Command "(gc builds\windows\vc2010\freetype.vcxproj) -replace 'MultiThreaded<', 'MultiThreadedDLL<' | Out-File -encoding ASCII builds\windows\vc2010\freetype.vcxproj" - %MSBUILD% builds\windows\vc2010\freetype.sln /t:Build /p:Configuration="Release Static" /p:Platform=${{ matrix.platform-msbuild }} /m - xcopy /Y /E /Q include %INCLIB% - copy /Y /B "objs\${{ matrix.platform-msbuild }}\Release Static\freetype.lib" %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_freetype.cmd" - name: Build dependencies / LCMS2 - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\lcms2-2.8 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - rmdir /S /Q Lib - rmdir /S /Q Projects\VC2015\Release - set VCTargetsPath=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\ - set MSBUILD="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe" - powershell %GITHUB_WORKSPACE%\winbuild\lcms2_patch.ps1 - %MSBUILD% Projects\VC2015\lcms2.sln /t:Clean;lcms2_static /p:Configuration="Release" /p:Platform=${{ matrix.platform-msbuild }} /m - xcopy /Y /E /Q include %INCLIB% - copy /Y /B Lib\MS\*.lib %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_lcms2.cmd" - name: Build dependencies / OpenJPEG - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\openjpeg-2.3.1msvcr10-x32 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DBUILD_THIRDPARTY:BOOL=OFF -DBUILD_SHARED_LIBS:BOOL=OFF - set CMAKE=%CMAKE% -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile - mkdir %INCLIB%\openjpeg-2.3.1 - copy /Y /B src\lib\openjp2\*.h %INCLIB%\openjpeg-2.3.1 - copy /Y /B bin\*.lib %INCLIB% - shell: cmd + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_openjpeg.cmd" # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant @@ -304,22 +176,12 @@ jobs: - name: Build Pillow run: | - set PYTHON=%pythonLocation% - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set MPLSRC=%GITHUB_WORKSPACE% - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - cd /D %GITHUB_WORKSPACE% - set LIB=%INCLIB%;%PYTHON%\tcl - set INCLUDE=%INCLIB%;%GITHUB_WORKSPACE%\depends\tcl86\include;%INCLUDE% - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - set MSSdk=1 - set DISTUTILS_USE_SDK=1 - set py_vcruntime_redist=true - %PYTHON%\python.exe setup.py build_ext install - rem Add libraqm.dll (copied to INCLIB) to PATH. - path %INCLIB%;%PATH% - %PYTHON%\python.exe selftest.py --installed - shell: cmd + cd $env:GITHUB_WORKSPACE + # & winbuild\build_dep_tcl.cmd + # & winbuild\build_dep_tk.cmd + & winbuild\build_pillow.cmd + & $env:pythonLocation\python.exe selftest.py --installed + shell: pwsh # failing with PyPy3 - name: Enable heap verification @@ -330,12 +192,8 @@ jobs: - name: Test Pillow run: | - set PYTHON=%pythonLocation% - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - rem Add libraqm.dll (copied to INCLIB) to PATH. - path %INCLIB%;%PATH% - cd /D %GITHUB_WORKSPACE% - %PYTHON%\python.exe -m pytest -vx -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests + path %GITHUB_WORKSPACE%\\winbuild\\build\\bin;%PATH% + python.exe -m pytest -vx -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests shell: cmd - name: Prepare to upload errors @@ -366,22 +224,16 @@ jobs: - name: Build wheel id: wheel if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" - run: | - for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ##[set-output name=dist;]dist-%%a - for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a - set PYTHON=%pythonLocation% - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set MPLSRC=%GITHUB_WORKSPACE% - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - cd /D %GITHUB_WORKSPACE% - set LIB=%INCLIB%;%PYTHON%\tcl - set INCLUDE=%INCLIB%;%GITHUB_WORKSPACE%\depends\tcl86\include;%INCLUDE% - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - %PYTHON%\python.exe setup.py bdist_wheel - shell: cmd + run: "%GITHUB_WORKSPACE%\\winbuild\\build_pillow_wheel.cmd" - uses: actions/upload-artifact@v1 if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" with: - name: ${{ steps.wheel.outputs.dist }} + name: dist path: dist + +# - uses: actions/upload-artifact@v1 +# if: matrix.architecture == 'x86' +# with: +# name: lib +# path: "winbuild\\build\\3.x\\x86\\lib" diff --git a/setup.py b/setup.py index 1236b5df5..b55b1de62 100755 --- a/setup.py +++ b/setup.py @@ -120,7 +120,7 @@ _LIB_IMAGING = ( "codec_fd", ) -DEBUG = False +DEBUG = True class DependencyException(Exception): diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py new file mode 100644 index 000000000..c17af7b95 --- /dev/null +++ b/winbuild/build_prepare.py @@ -0,0 +1,494 @@ +import os +import shutil +import subprocess +import winreg +from itertools import count + +from commands import * + +SF_MIRROR = "http://iweb.dl.sourceforge.net" + +# use PYTHON to select architecture +architectures = { + "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, + "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, +} + +# use PYTHON + architecture to select config +pythons = { + "pypy3.6": {"config-x86": "3.5"}, + "3.5": {"config-x86": "3.5", "config-x64": "3.5"}, + "3.6": {"config-x86": "3.6", "config-x64": "3.6"}, + "3.7": {"config-x86": "3.6", "config-x64": "3.6"}, +} + +# select deps and libs +configs = { + "3.5": { + "deps": [ + "libjpeg-turbo-2.0.3", + "zlib-1.2.11", + "tiff-4.0.10", + "libwebp-1.0.3", + "freetype-2.10.1", + "lcms2-2.9", + "openjpeg-2.3.1", + "ghostscript-9.27", + # "tcl-8.6", + # "tk-8.6", + ], + "vcvars_ver": "14.0", + "vs_ver": "2015", + }, + "3.6": { + "deps": [ + "libjpeg-turbo-2.0.3", + "zlib-1.2.11", + "tiff-4.0.10", + "libwebp-1.0.3", + "freetype-2.10.1", + "lcms2-2.9", + "openjpeg-2.3.1", + "ghostscript-9.27", + # "tcl-8.6", + # "tk-8.6", + ], + "vs_ver": "2017", + }, +} + +header = [ + cmd_set("BUILD", "{build_dir}"), + cmd_set("INCLUDE", "{inc_dir}"), + cmd_set("INCLIB", "{lib_dir}"), + cmd_set("LIB", "{lib_dir}"), + cmd_append("PATH", "{bin_dir}"), + "@echo on", +] + +# dependencies +deps = { + "jpeg-9c": { + "name": "libjpeg", + # FIXME HTTP 403 + "url": "http://www.ijg.org/files/jpegsr9c.zip", + "filename": "jpegsr9c.zip", + "build": [ + # FIXME builds with -MT, not -MD + cmd_nmake("makefile.vc", "setup-vc6"), + cmd_nmake("makefile.vc", "clean"), + cmd_nmake("makefile.vc", "libjpeg.lib", "nodebug=1"), + ], + "headers": [r"j*.h"], + "libs": [r"*.lib"], + }, + "libjpeg-turbo-2.0.3": { + "name": "libjpeg", + "url": SF_MIRROR + "/project/libjpeg-turbo/2.0.3/libjpeg-turbo-2.0.3.tar.gz", + "filename": "libjpeg-turbo-2.0.3.tar.gz", + "build": [ + cmd_cmake([ + "-DENABLE_SHARED:BOOL=FALSE", + "-DWITH_JPEG8:BOOL=TRUE", + "-DWITH_CRT_DLL:BOOL=TRUE", + ]), + cmd_nmake(target="clean"), + cmd_nmake(target="jpeg-static"), + cmd_copy("jpeg-static.lib", "libjpeg.lib"), + cmd_nmake(target="cjpeg-static"), + cmd_copy("cjpeg-static.exe", "cjpeg.exe"), + cmd_nmake(target="djpeg-static"), + cmd_copy("djpeg-static.exe", "djpeg.exe"), + ], + "headers": ["j*.h"], + "libs": ["libjpeg.lib"], + "bins": ["cjpeg.exe", "djpeg.exe"], + }, + "zlib-1.2.11": { + "name": "zlib", + "url": "http://zlib.net/zlib1211.zip", + "filename": "zlib1211.zip", + "build": [ + cmd_nmake(r"win32\Makefile.msc", "clean"), + cmd_nmake(r"win32\Makefile.msc", "zlib.lib"), + cmd_copy("zlib.lib", "z.lib"), + ], + "headers": [r"z*.h"], + "libs": [r"*.lib"], + }, + "tiff-4.0.10": { + "name": "libtiff", + # FIXME FTP timeout + "url": "ftp://download.osgeo.org/libtiff/tiff-4.0.10.tar.gz", + "filename": "tiff-4.0.10.tar.gz", + "build": [ + cmd_copy(r"{script_dir}\tiff.opt", "nmake.opt"), + cmd_nmake("makefile.vc", "clean"), + cmd_nmake("makefile.vc", "lib", "RuntimeLibrary=-MT"), + ], + "headers": [r"libtiff\tiff*.h"], + "libs": [r"libtiff\*.lib"], + # "bins": [r"libtiff\*.dll"], + }, + "libwebp-1.0.3": { + "name": "libwebp", + "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.0.3.tar.gz", # noqa: E501 + "filename": "libwebp-1.0.3.tar.gz", + "build": [ + cmd_rmdir(r"output\release-static"), # clean + cmd_nmake( + "Makefile.vc", + "all", + ["CFG=release-static", "OBJDIR=output", "ARCH={architecture}"] + ), + cmd_mkdir(r"{inc_dir}\webp"), + cmd_copy(r"src\webp\*.h", r"{inc_dir}\webp"), + ], + "libs": [r"output\release-static\{architecture}\lib\*.lib"], + }, + "freetype-2.10.1": { + "name": "freetype", + "url": "https://download.savannah.gnu.org/releases/freetype/freetype-2.10.1.tar.gz", # noqa: E501 + "filename": "freetype-2.10.1.tar.gz", + "build": [ + cmd_rmdir("objs"), + # freetype setting is /MD for .dll and /MT for .lib, we need /MD + cmd_patch_replace( + r"builds\windows\vc2010\freetype.vcxproj", + "MultiThreaded<", + "MultiThreadedDLL<" + ), + cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Clean"), # TODO failing on GHA # noqa: E501 + cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Build"), + cmd_xcopy("include", "{inc_dir}"), + ], + "libs": [r"objs\{msbuild_arch}\Release Static\freetype.lib"], + # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], + }, + "lcms2-2.9": { + "name": "lcms2", + "url": SF_MIRROR + "/project/lcms/lcms/2.8/lcms2-2.9.tar.gz", + "filename": "lcms2-2.9.tar.gz", + "build": [ + cmd_rmdir("Lib"), + cmd_rmdir(r"Projects\VC{vs_ver}\Release"), + # lcms2-2.8\VC2015 setting is /MD for x86 and /MT for x64, we need /MD always + cmd_patch_replace( + r"Projects\VC2017\lcms2.sln", "MultiThreaded<", "MultiThreadedDLL<" + ), + cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "Clean"), + cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "lcms2_static"), + cmd_xcopy("include", "{inc_dir}"), + ], + "libs": [r"Lib\MS\*.lib"], + }, + "openjpeg-2.3.1": { + "name": "openjpeg", + "url": "https://github.com/uclouvain/openjpeg/archive/v2.3.1.tar.gz", + "filename": "openjpeg-2.3.1.tar.gz", + "build": [ + cmd_cmake(("-DBUILD_THIRDPARTY:BOOL=OFF", "-DBUILD_SHARED_LIBS:BOOL=OFF")), + cmd_nmake(target="clean"), + cmd_nmake(), + cmd_mkdir(r"{inc_dir}\openjpeg-2.3.1"), + cmd_copy(r"src\lib\openjp2\*.h", r"{inc_dir}\openjpeg-2.3.1"), + ], + "libs": [r"bin\*.lib"], + }, + "ghostscript-9.27": { + "name": "ghostscript", + "url": "https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs927/ghostscript-9.27.tar.gz", # noqa: E501 + "filename": "ghostscript-9.27.tar.gz", + "build": [ + cmd_set("MSVC_VERSION", 14), + cmd_if_eq("{architecture}", "x64", cmd_set("WIN64", '""')), + cmd_nmake(r"psi\msvc.mak"), + ], + "bins": [r"bin\*"], + }, + "tcl-8.5": { + "name": "tcl", + "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tcl8519-src.zip", + "filename": "tcl8519-src.zip", + }, + "tk-8.5": { + "name": "tk", + "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tk8519-src.zip", + "filename": "tk8519-src.zip", + }, + "tcl-8.6": { + "name": "tcl", + "url": SF_MIRROR + "/project/tcl/Tcl/8.6.9/tcl869-src.zip", + "filename": "tcl869-src.zip", + "headers": [r"generic\*.h"], + }, + "tk-8.6": { + "name": "tk", + "url": SF_MIRROR + "/project/tcl/Tcl/8.6.9/tk869-src.zip", + "filename": "tk869-src.zip", + "build": [ + r"""mkdir {inc_dir}\X11""", + r"""copy /Y /B xlib\X11\* "{inc_dir}\X11\" """, + ], + "headers": [r"generic\*.h"], + }, +} + + +# based on distutils._msvccompiler from CPython 3.7.4 +def find_vs2017(config): + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if not root: + print("Program Files not found") + return None + + try: + vspath = subprocess.check_output([ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath", + "-products", "*", + ]).decode(encoding="mbcs").strip() + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + print("vswhere not found") + return None + + if not os.path.isdir(os.path.join(vspath, "VC", "Auxiliary", "Build")): + print("Visual Studio seems to be missing C compiler") + return None + + vs = { + "header": [], + # nmake selected by vcvarsall + "nmake": "nmake.exe", + } + + msbuild = os.path.join(vspath, "MSBuild", "15.0", "Bin", "MSBuild.exe") + if os.path.isfile(msbuild): + # default_platform_toolset = "v140" + vs["msbuild"] = '"{}"'.format(msbuild) + # vs["header"].append(cmd_set("DefaultPlatformToolset", default_platform_toolset)) + else: + print("Visual Studio MSBuild not found") + return None + + vcvarsall = os.path.join(vspath, "VC", "Auxiliary", "Build", "vcvarsall.bat") + if not os.path.isfile(vcvarsall): + print("Visual Studio vcvarsall not found") + return None + vcvars_ver = "-vcvars_ver={}".format(config["vcvars_ver"]) if "vcvars_ver" in config else "" + vs["header"].append('call "{}" {{vcvars_arch}} {}'.format(vcvarsall, vcvars_ver)) + + return vs + + +def find_sdk71a(): + try: + key = winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A", + access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY, + ) + except OSError: + return None + with key: + for i in count(): + try: + v_name, v_value, v_type = winreg.EnumValue(key, i) + except OSError: + return None + if v_name == "InstallationFolder" and v_type == winreg.REG_SZ: + sdk_dir = v_value + break + else: + return None + + if not os.path.isdir(sdk_dir): + return None + + sdk = { + "header": [ + # for win32.mak + cmd_append("INCLUDE", os.path.join(sdk_dir, "Include")), + # for ghostscript + cmd_set("RCOMP", '"{}"'.format(os.path.join(sdk_dir, "Bin", "RC.EXE"))), + ] + } + + return sdk + + +def match(values, target): + for key, value in values.items(): + if key in target: + return {"name": key, **value} + + +def extract_dep(url, filename): + import urllib.request + import tarfile + import zipfile + + file = os.path.join(depends_dir, filename) + if not os.path.exists(file): + ex = None + for i in range(3): + try: + print("Fetching %s (attempt %d)..." % (url, i + 1)) + content = urllib.request.urlopen(url).read() + with open(file, "wb") as f: + f.write(content) + break + except urllib.error.URLError as e: + ex = e + else: + raise RuntimeError(ex) + if filename.endswith(".zip"): + with zipfile.ZipFile(file) as zf: + zf.extractall(build_dir) + elif filename.endswith(".tar.gz") or filename.endswith(".tgz"): + with tarfile.open(file, "r:gz") as tgz: + tgz.extractall(build_dir) + else: + raise RuntimeError("Unknown archive type: " + filename) + + +def write_script(name, lines): + name = os.path.join(script_dir, name) + lines = [line.format(**prefs) for line in lines] + print("Writing " + name) + with open(name, "w") as f: + f.write("\n\r".join(lines)) + for line in lines: + print(" " + line) + + +def get_footer(dep): + lines = [] + for out in dep.get("headers", []): + lines.append(cmd_copy(out, "{inc_dir}")) + for out in dep.get("libs", []): + lines.append(cmd_copy(out, "{lib_dir}")) + for out in dep.get("bins", []): + lines.append(cmd_copy(out, "{bin_dir}")) + return lines + + +def build_dep(name): + dep = deps[name] + file = "build_dep_{name}.cmd".format(name=dep["name"]) + + extract_dep(dep["url"], dep["filename"]) + + lines = ["cd /D %s" % os.path.join(build_dir, name)] + lines.extend(prefs["header"]) + + lines.extend(dep.get("build", [])) + + lines.extend(get_footer(dep)) + + write_script(file, lines) + return file + + +def build_dep_all(): + lines = ["cd {script_dir}"] + for dep_name in prefs["deps"]: + lines.append('cmd.exe /c "%s"' % build_dep(dep_name)) + write_script("build_dep_all.ps1", lines) + + +def build_pillow(wheel=False): + if not wheel: + op, filename = "install", "build_pillow.cmd" + else: + op, filename = "bdist_wheel", "build_pillow_wheel.cmd" + + lines = [] + if path_dir is not None and not wheel: + lines.append(cmd_xcopy("{bin_dir}", path_dir)) + lines.extend(prefs["header"]) + lines.extend( + [ + cmd_cd("{pillow_dir}"), + cmd_append("LIB", r"{python_dir}\tcl"), + r'"{{python_dir}}\python.exe" setup.py build_ext {}'.format(op), + # r"""%PYTHON%\python.exe selftest.py --installed""", + ] + ) + + write_script(filename, lines) + + +if __name__ == "__main__": + script_dir = os.path.dirname(os.path.realpath(__file__)) + depends_dir = os.path.join(script_dir, "depends") + python_dir = os.environ["PYTHON"] + + # copy binaries to this directory + path_dir = os.environ.get("PILLOW_BIN") + + # use PYTHON to select architecture + arch_prefs = match(architectures, python_dir) + if arch_prefs is None: + architecture = "x86" + print("WARN: Could not determine architecture, guessing " + architecture) + arch_prefs = architectures[architecture] + else: + architecture = arch_prefs["name"] + + # use PYTHON to select python version + python_prefs = match(pythons, python_dir) + if python_prefs is None: + raise KeyError("Failed to determine Python version from PYTHON: " + python_dir) + + # use python version + architecture to select build config + config_name = python_prefs["config-" + architecture] + config = configs[config_name] + + vs2017 = find_vs2017(config) + if vs2017 is None: + raise RuntimeError("Visual Studio 2017 not found") + + sdk71a = find_sdk71a() + if sdk71a is None: + raise RuntimeError("Windows SDK v7.1A not found") + + build_dir = os.path.join(script_dir, "build", config_name, architecture) + lib_dir = os.path.join(build_dir, "lib") + inc_dir = os.path.join(build_dir, "inc") + bin_dir = os.path.join(build_dir, "bin") + + # for path in [lib_dir, inc_dir, bin_dir]: + # shutil.rmtree(path) + for path in [depends_dir, build_dir, lib_dir, inc_dir, bin_dir]: + os.makedirs(path, exist_ok=True) + + prefs = { + "python_version": python_prefs["name"], + "architecture": architecture, + "script_dir": script_dir, + "depends_dir": depends_dir, + "python_dir": python_dir, + "build_dir": build_dir, + "lib_dir": lib_dir, + "inc_dir": inc_dir, + "bin_dir": bin_dir, + "pillow_dir": os.path.realpath(os.path.join(script_dir, "..")), + # TODO auto find: + "cmake": "cmake.exe", + } + + dicts = [vs2017, sdk71a, arch_prefs, python_prefs, config] + for x in dicts: + prefs.update(x) + prefs["header"] = sum((x.get("header", []) for x in dicts), header) + del prefs["name"] + + print("Target: Python {python_version} {architecture}".format(**prefs)) + + build_dep_all() + build_pillow() + build_pillow(wheel=True) diff --git a/winbuild/commands.py b/winbuild/commands.py new file mode 100644 index 000000000..2fbbc9b1a --- /dev/null +++ b/winbuild/commands.py @@ -0,0 +1,111 @@ +# builtins + + +def cmd_cd(path): + return "cd /D {path}".format(**locals()) + + +def cmd_set(name, value): + return "set {name}={value}".format(**locals()) + + +def cmd_prepend(name, value): + op = "path " if name == "PATH" else "set {name}=" + return (op + "{value};%{name}%").format(**locals()) + + +def cmd_append(name, value): + op = "path " if name == "PATH" else "set {name}=" + return (op + "%{name}%;{value}").format(**locals()) + + +def cmd_copy(src, tgt): + return 'copy /Y /B "{src}" "{tgt}"'.format(**locals()) + + +def cmd_xcopy(src, tgt): + return 'xcopy /Y /E "{src}" "{tgt}"'.format(**locals()) + + +def cmd_mkdir(path): + return 'mkdir "{path}"'.format(**locals()) + + +def cmd_rmdir(path): + return 'rmdir /S /Q "{path}"'.format(**locals()) + + +def cmd_if_eq(a, b, cmd): + return 'if "{a}"=="{b}" {cmd}'.format(**locals()) + + +# tools + + +def cmd_nmake(makefile=None, target="", params=None): + if params is None: + params = "" + elif isinstance(params, list) or isinstance(params, tuple): + params = " ".join(params) + else: + params = str(params) + + return " ".join( + [ + "{{nmake}}", + "-nologo", + '-f "{makefile}"' if makefile is not None else "", + "{params}", + '"{target}"', + ] + ).format(**locals()) + + +def cmd_cmake(params=None, file="."): + if params is None: + params = "" + elif isinstance(params, list) or isinstance(params, tuple): + params = " ".join(params) + else: + params = str(params) + return " ".join( + [ + "{{cmake}}", + "-DCMAKE_VERBOSE_MAKEFILE=ON", + "-DCMAKE_RULE_MESSAGES:BOOL=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "{params}", + '-G "NMake Makefiles"', + '"{file}"', + ] + ).format(**locals()) + + +def cmd_msbuild( + file, configuration="Release", target="Build", platform="{msbuild_arch}" +): + return " ".join( + [ + "{{msbuild}}", + "{file}", + '/t:"{target}"', + '/p:Configuration="{configuration}"', + "/p:Platform={platform}", + "/m", + ] + ).format(**locals()) + + +# patch tools + + +def cmd_patch_replace(file, src, dst): + return " ".join( + [ + "echo", + "(Get-Content '{file}')", + '-replace "{src}", "{dst}"', + "^| Out-File -encoding ASCII '{file}'", + "> temp.ps1\r\npowershell .\\temp.ps1", + ] + ).format(**locals()) From 46f8729e991be208ec98a59c64cb66eb93a1f3e5 Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 01:10:22 +0100 Subject: [PATCH 02/27] add vs2019 support --- winbuild/build_prepare.py | 123 +++++++++++--------------------------- 1 file changed, 35 insertions(+), 88 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index c17af7b95..937bdc886 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -22,41 +22,28 @@ pythons = { "3.7": {"config-x86": "3.6", "config-x64": "3.6"}, } -# select deps and libs +# select preferred compiler configs = { "3.5": { - "deps": [ - "libjpeg-turbo-2.0.3", - "zlib-1.2.11", - "tiff-4.0.10", - "libwebp-1.0.3", - "freetype-2.10.1", - "lcms2-2.9", - "openjpeg-2.3.1", - "ghostscript-9.27", - # "tcl-8.6", - # "tk-8.6", - ], "vcvars_ver": "14.0", "vs_ver": "2015", }, "3.6": { - "deps": [ - "libjpeg-turbo-2.0.3", - "zlib-1.2.11", - "tiff-4.0.10", - "libwebp-1.0.3", - "freetype-2.10.1", - "lcms2-2.9", - "openjpeg-2.3.1", - "ghostscript-9.27", - # "tcl-8.6", - # "tk-8.6", - ], "vs_ver": "2017", }, } +# selected dependencies +deps_list = [ + "libjpeg-turbo-2.0.3", + "zlib-1.2.11", + "tiff-4.0.10", + "libwebp-1.0.3", + "freetype-2.10.1", + "lcms2-2.9", + "openjpeg-2.3.1", +] + header = [ cmd_set("BUILD", "{build_dir}"), cmd_set("INCLUDE", "{inc_dir}"), @@ -68,20 +55,6 @@ header = [ # dependencies deps = { - "jpeg-9c": { - "name": "libjpeg", - # FIXME HTTP 403 - "url": "http://www.ijg.org/files/jpegsr9c.zip", - "filename": "jpegsr9c.zip", - "build": [ - # FIXME builds with -MT, not -MD - cmd_nmake("makefile.vc", "setup-vc6"), - cmd_nmake("makefile.vc", "clean"), - cmd_nmake("makefile.vc", "libjpeg.lib", "nodebug=1"), - ], - "headers": [r"j*.h"], - "libs": [r"*.lib"], - }, "libjpeg-turbo-2.0.3": { "name": "libjpeg", "url": SF_MIRROR + "/project/libjpeg-turbo/2.0.3/libjpeg-turbo-2.0.3.tar.gz", @@ -195,43 +168,6 @@ deps = { ], "libs": [r"bin\*.lib"], }, - "ghostscript-9.27": { - "name": "ghostscript", - "url": "https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs927/ghostscript-9.27.tar.gz", # noqa: E501 - "filename": "ghostscript-9.27.tar.gz", - "build": [ - cmd_set("MSVC_VERSION", 14), - cmd_if_eq("{architecture}", "x64", cmd_set("WIN64", '""')), - cmd_nmake(r"psi\msvc.mak"), - ], - "bins": [r"bin\*"], - }, - "tcl-8.5": { - "name": "tcl", - "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tcl8519-src.zip", - "filename": "tcl8519-src.zip", - }, - "tk-8.5": { - "name": "tk", - "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tk8519-src.zip", - "filename": "tk8519-src.zip", - }, - "tcl-8.6": { - "name": "tcl", - "url": SF_MIRROR + "/project/tcl/Tcl/8.6.9/tcl869-src.zip", - "filename": "tcl869-src.zip", - "headers": [r"generic\*.h"], - }, - "tk-8.6": { - "name": "tk", - "url": SF_MIRROR + "/project/tcl/Tcl/8.6.9/tk869-src.zip", - "filename": "tk869-src.zip", - "build": [ - r"""mkdir {inc_dir}\X11""", - r"""copy /Y /B xlib\X11\* "{inc_dir}\X11\" """, - ], - "headers": [r"generic\*.h"], - }, } @@ -265,14 +201,21 @@ def find_vs2017(config): "nmake": "nmake.exe", } + # vs2017 msbuild = os.path.join(vspath, "MSBuild", "15.0", "Bin", "MSBuild.exe") if os.path.isfile(msbuild): - # default_platform_toolset = "v140" + default_platform_toolset = "v140" vs["msbuild"] = '"{}"'.format(msbuild) - # vs["header"].append(cmd_set("DefaultPlatformToolset", default_platform_toolset)) else: - print("Visual Studio MSBuild not found") - return None + # vs2019 + msbuild = os.path.join(vspath, "MSBuild", "Current", "Bin", "MSBuild.exe") + if os.path.isfile(msbuild): + default_platform_toolset = "v142" + vs["msbuild"] = '"{}"'.format(msbuild) + else: + print("Visual Studio MSBuild not found") + return None + # vs["header"].append(cmd_set("DefaultPlatformToolset", default_platform_toolset)) vcvarsall = os.path.join(vspath, "VC", "Auxiliary", "Build", "vcvarsall.bat") if not os.path.isfile(vcvarsall): @@ -382,20 +325,24 @@ def build_dep(name): extract_dep(dep["url"], dep["filename"]) - lines = ["cd /D %s" % os.path.join(build_dir, name)] - lines.extend(prefs["header"]) - - lines.extend(dep.get("build", [])) - - lines.extend(get_footer(dep)) + lines = [ + "echo Building {name} ({dir})...".format(name=dep["name"], dir=name), + "cd /D %s" % os.path.join(build_dir, name), + *prefs["header"], + *dep.get("build", []), + *get_footer(dep), + ] write_script(file, lines) return file def build_dep_all(): - lines = ["cd {script_dir}"] - for dep_name in prefs["deps"]: + lines = [ + "$ErrorActionPreference = 'stop'", + "cd {script_dir}", + ] + for dep_name in deps_list: lines.append('cmd.exe /c "%s"' % build_dep(dep_name)) write_script("build_dep_all.ps1", lines) From b0885a5ceb46d41a383c4e97aa33a5e7686688bf Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 14:45:11 +0100 Subject: [PATCH 03/27] fix finding PYTHON dir --- .github/workflows/test-windows.yml | 4 ++-- winbuild/build_prepare.py | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 6425f3b5c..c0bd4e6ce 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -179,7 +179,7 @@ jobs: cd $env:GITHUB_WORKSPACE # & winbuild\build_dep_tcl.cmd # & winbuild\build_dep_tk.cmd - & winbuild\build_pillow.cmd + & winbuild\build_pillow.cmd install & $env:pythonLocation\python.exe selftest.py --installed shell: pwsh @@ -224,7 +224,7 @@ jobs: - name: Build wheel id: wheel if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" - run: "%GITHUB_WORKSPACE%\\winbuild\\build_pillow_wheel.cmd" + run: "%GITHUB_WORKSPACE%\\winbuild\\build_pillow.cmd bdist_wheel" - uses: actions/upload-artifact@v1 if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 937bdc886..1492a57ee 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -1,6 +1,7 @@ import os import shutil import subprocess +import sys import winreg from itertools import count @@ -199,6 +200,7 @@ def find_vs2017(config): "header": [], # nmake selected by vcvarsall "nmake": "nmake.exe", + "vs_dir": vspath, } # vs2017 @@ -257,7 +259,8 @@ def find_sdk71a(): cmd_append("INCLUDE", os.path.join(sdk_dir, "Include")), # for ghostscript cmd_set("RCOMP", '"{}"'.format(os.path.join(sdk_dir, "Bin", "RC.EXE"))), - ] + ], + "sdk_dir": sdk_dir, } return sdk @@ -348,11 +351,6 @@ def build_dep_all(): def build_pillow(wheel=False): - if not wheel: - op, filename = "install", "build_pillow.cmd" - else: - op, filename = "bdist_wheel", "build_pillow_wheel.cmd" - lines = [] if path_dir is not None and not wheel: lines.append(cmd_xcopy("{bin_dir}", path_dir)) @@ -361,18 +359,18 @@ def build_pillow(wheel=False): [ cmd_cd("{pillow_dir}"), cmd_append("LIB", r"{python_dir}\tcl"), - r'"{{python_dir}}\python.exe" setup.py build_ext {}'.format(op), + r'"{{python_dir}}\python.exe" setup.py build_ext %*', # r"""%PYTHON%\python.exe selftest.py --installed""", ] ) - write_script(filename, lines) + write_script("build_pillow.cmd", lines) if __name__ == "__main__": script_dir = os.path.dirname(os.path.realpath(__file__)) depends_dir = os.path.join(script_dir, "depends") - python_dir = os.environ["PYTHON"] + python_dir = os.environ.get("PYTHON", os.path.dirname(os.path.realpath(sys.executable))) # copy binaries to this directory path_dir = os.environ.get("PILLOW_BIN") @@ -391,6 +389,8 @@ if __name__ == "__main__": if python_prefs is None: raise KeyError("Failed to determine Python version from PYTHON: " + python_dir) + print("Target: Python {python_version} {architecture} at: {python_dir}".format(python_version=python_prefs["name"], architecture=architecture, python_dir=python_dir)) + # use python version + architecture to select build config config_name = python_prefs["config-" + architecture] config = configs[config_name] @@ -399,10 +399,14 @@ if __name__ == "__main__": if vs2017 is None: raise RuntimeError("Visual Studio 2017 not found") + print("Found Visual Studio at: {}".format(vs2017["vs_dir"])) + sdk71a = find_sdk71a() if sdk71a is None: raise RuntimeError("Windows SDK v7.1A not found") + print("Found Windows SDK 7.1A at: {}".format(sdk71a["sdk_dir"])) + build_dir = os.path.join(script_dir, "build", config_name, architecture) lib_dir = os.path.join(build_dir, "lib") inc_dir = os.path.join(build_dir, "inc") @@ -434,8 +438,5 @@ if __name__ == "__main__": prefs["header"] = sum((x.get("header", []) for x in dicts), header) del prefs["name"] - print("Target: Python {python_version} {architecture}".format(**prefs)) - build_dep_all() build_pillow() - build_pillow(wheel=True) From b7120d56f194c7500ebe52b2235eafbfcdc1a755 Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 15:25:21 +0100 Subject: [PATCH 04/27] add trace statements --- winbuild/build_prepare.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 1492a57ee..b277ae0b9 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -231,28 +231,37 @@ def find_vs2017(config): def find_sdk71a(): try: + print("trace: opening sdk key") key = winreg.OpenKeyEx( winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A", access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY, ) + print("trace: opened sdk key") except OSError: return None + print("trace: enumerating key") with key: for i in count(): try: v_name, v_value, v_type = winreg.EnumValue(key, i) + print("trace: entry: {} {} = {}".format(v_type, v_name, v_value)) except OSError: return None if v_name == "InstallationFolder" and v_type == winreg.REG_SZ: sdk_dir = v_value + print("trace: found dir: {}".format(sdk_dir)) break else: return None + print("trace: closed key") + if not os.path.isdir(sdk_dir): return None + print("trace: creating dict") + sdk = { "header": [ # for win32.mak From e0dd636e2cc047da28d71d36c0062c65510025cc Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 15:36:16 +0100 Subject: [PATCH 05/27] remove 7.1a dependency --- .github/workflows/test-windows.yml | 2 ++ winbuild/build_prepare.py | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index c0bd4e6ce..3ce907931 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -74,6 +74,8 @@ jobs: xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\depends\ xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\ cd $env:GITHUB_WORKSPACE/winbuild/ + # reg query "HKLM\Software\Microsoft\Microsoft SDKs\Windows" /s + ls 'C:\Program Files (x86)\Microsoft SDKs\Windows\' python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py shell: pwsh diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index b277ae0b9..8d0ad9b80 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -410,11 +410,11 @@ if __name__ == "__main__": print("Found Visual Studio at: {}".format(vs2017["vs_dir"])) - sdk71a = find_sdk71a() - if sdk71a is None: - raise RuntimeError("Windows SDK v7.1A not found") - - print("Found Windows SDK 7.1A at: {}".format(sdk71a["sdk_dir"])) + # sdk71a = find_sdk71a() + # if sdk71a is None: + # raise RuntimeError("Windows SDK v7.1A not found") + # + # print("Found Windows SDK 7.1A at: {}".format(sdk71a["sdk_dir"])) build_dir = os.path.join(script_dir, "build", config_name, architecture) lib_dir = os.path.join(build_dir, "lib") @@ -441,7 +441,7 @@ if __name__ == "__main__": "cmake": "cmake.exe", } - dicts = [vs2017, sdk71a, arch_prefs, python_prefs, config] + dicts = [vs2017, arch_prefs, python_prefs, config] for x in dicts: prefs.update(x) prefs["header"] = sum((x.get("header", []) for x in dicts), header) From 85110521ecf7b9d05b952b5ab9a9f800c0ad901b Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 15:53:07 +0100 Subject: [PATCH 06/27] fix default shell --- .github/workflows/test-windows.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 3ce907931..af30b98e9 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -80,19 +80,19 @@ jobs: shell: pwsh - name: Build dependencies / libjpeg - run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libjpeg.cmd" + run: "& .\\winbuild\\build_dep_libjpeg.cmd" - name: Build dependencies / zlib - run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_zlib.cmd" + run: "& .\\winbuild\\build_dep_zlib.cmd" - name: Build dependencies / LibTiff - run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libtiff.cmd" + run: "& .\\winbuild\\build_dep_libtiff.cmd" - name: Build dependencies / WebP - run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libwebp.cmd" + run: "& .\\winbuild\\build_dep_libwebp.cmd" - name: Build dependencies / FreeType - run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_freetype.cmd" + run: "& .\\winbuild\\build_dep_freetype.cmd" - name: Build dependencies / LCMS2 - run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_lcms2.cmd" + run: "& .\\winbuild\\build_dep_lcms2.cmd" - name: Build dependencies / OpenJPEG - run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_openjpeg.cmd" + run: "& .\\winbuild\\build_dep_openjpeg.cmd" # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant From 7b70265e69e9f26a8de24e810ac3a7bef3546ed7 Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 17:34:11 +0100 Subject: [PATCH 07/27] rewrite retargetting --- winbuild/build_prepare.py | 42 ++++++++++++++++++++++++++------------- winbuild/commands.py | 15 -------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 8d0ad9b80..454d80f41 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -51,7 +51,6 @@ header = [ cmd_set("INCLIB", "{lib_dir}"), cmd_set("LIB", "{lib_dir}"), cmd_append("PATH", "{bin_dir}"), - "@echo on", ] # dependencies @@ -124,14 +123,14 @@ deps = { "name": "freetype", "url": "https://download.savannah.gnu.org/releases/freetype/freetype-2.10.1.tar.gz", # noqa: E501 "filename": "freetype-2.10.1.tar.gz", + "patch": { + r"builds\windows\vc2010\freetype.vcxproj": { + # freetype setting is /MD for .dll and /MT for .lib, we need /MD + "MultiThreaded": "MultiThreadedDLL", + }, + }, "build": [ cmd_rmdir("objs"), - # freetype setting is /MD for .dll and /MT for .lib, we need /MD - cmd_patch_replace( - r"builds\windows\vc2010\freetype.vcxproj", - "MultiThreaded<", - "MultiThreadedDLL<" - ), cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Clean"), # TODO failing on GHA # noqa: E501 cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Build"), cmd_xcopy("include", "{inc_dir}"), @@ -141,15 +140,21 @@ deps = { }, "lcms2-2.9": { "name": "lcms2", - "url": SF_MIRROR + "/project/lcms/lcms/2.8/lcms2-2.9.tar.gz", + "url": SF_MIRROR + "/project/lcms/lcms/2.9/lcms2-2.9.tar.gz", "filename": "lcms2-2.9.tar.gz", + "patch": { + r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { + # lcms2-2.8\VC2015 setting is /MD for x86 and /MT for x64, we need /MD always + "MultiThreaded": "MultiThreadedDLL", + # retarget to default msvc + "v141": "$(DefaultPlatformToolset)", + # retarget to latest SDK 10.0 + "8.1": "10.0", + }, + }, "build": [ cmd_rmdir("Lib"), cmd_rmdir(r"Projects\VC{vs_ver}\Release"), - # lcms2-2.8\VC2015 setting is /MD for x86 and /MT for x64, we need /MD always - cmd_patch_replace( - r"Projects\VC2017\lcms2.sln", "MultiThreaded<", "MultiThreadedDLL<" - ), cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "Clean"), cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "lcms2_static"), cmd_xcopy("include", "{inc_dir}"), @@ -337,8 +342,17 @@ def build_dep(name): extract_dep(dep["url"], dep["filename"]) + for patch_file, patch_list in dep.get("patch", {}).items(): + patch_file = os.path.join(build_dir, name, patch_file) + with open(patch_file, "r") as f: + text = f.read() + for patch_from, patch_to in patch_list.items(): + text = text.replace(patch_from, patch_to) + with open(patch_file, "w") as f: + f.write(text) + lines = [ - "echo Building {name} ({dir})...".format(name=dep["name"], dir=name), + "@echo Building {name} ({dir})...".format(name=dep["name"], dir=name), "cd /D %s" % os.path.join(build_dir, name), *prefs["header"], *dep.get("build", []), @@ -444,7 +458,7 @@ if __name__ == "__main__": dicts = [vs2017, arch_prefs, python_prefs, config] for x in dicts: prefs.update(x) - prefs["header"] = sum((x.get("header", []) for x in dicts), header) + prefs["header"] = sum((x.get("header", []) for x in dicts), header) + ["@echo on"] del prefs["name"] build_dep_all() diff --git a/winbuild/commands.py b/winbuild/commands.py index 2fbbc9b1a..f78a596f1 100644 --- a/winbuild/commands.py +++ b/winbuild/commands.py @@ -94,18 +94,3 @@ def cmd_msbuild( "/m", ] ).format(**locals()) - - -# patch tools - - -def cmd_patch_replace(file, src, dst): - return " ".join( - [ - "echo", - "(Get-Content '{file}')", - '-replace "{src}", "{dst}"', - "^| Out-File -encoding ASCII '{file}'", - "> temp.ps1\r\npowershell .\\temp.ps1", - ] - ).format(**locals()) From 9784874dc6605787889ff888df9b8444b82b933c Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 18:13:16 +0100 Subject: [PATCH 08/27] extract remaining libs --- .github/workflows/test-windows.yml | 100 ++--------- winbuild/build_prepare.py | 257 ++++++++++++++++++----------- 2 files changed, 177 insertions(+), 180 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index af30b98e9..fc341e446 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -61,11 +61,9 @@ jobs: - name: Fetch dependencies run: | 7z x ..\pillow-depends\nasm-2.14.02-win64.zip "-o$env:RUNNER_WORKSPACE\" - Write-Host "`#`#[add-path]$env:RUNNER_WORKSPACE\nasm-2.14.02" Write-Host "::add-path::$env:RUNNER_WORKSPACE\nasm-2.14.02" ..\pillow-depends\gs950w32.exe /S - Write-Host "`#`#[add-path]C:\Program Files (x86)\gs\gs9.50\bin" Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" $env:PYTHON=$env:pythonLocation @@ -76,7 +74,7 @@ jobs: cd $env:GITHUB_WORKSPACE/winbuild/ # reg query "HKLM\Software\Microsoft\Microsoft SDKs\Windows" /s ls 'C:\Program Files (x86)\Microsoft SDKs\Windows\' - python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py + & python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py shell: pwsh - name: Build dependencies / libjpeg @@ -97,84 +95,15 @@ jobs: # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant if: "github.event_name != 'push' || contains(matrix.python-version, 'pypy')" - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - rem e5d454b: Merge tag '2.12.6' into msvc - cd /D %BUILD%\libimagequant-e5d454bc7f5eb63ee50c84a83a7fa5ac94f68ec4 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - echo (gc CMakeLists.txt) -replace 'add_library', "add_compile_options(-openmp-)`r`nadd_library" ^| Out-File -encoding ASCII CMakeLists.txt > patch.ps1 - echo (gc CMakeLists.txt) -replace ' SHARED', ' STATIC' ^| Out-File -encoding ASCII CMakeLists.txt >> patch.ps1 - powershell .\patch.ps1 - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile - copy /Y /B *.h %INCLIB% - copy /Y /B *.lib %INCLIB% - shell: cmd + run: "& .\\winbuild\\build_dep_libimagequant.cmd" - # for Raqm + # Raqm dependencies - name: Build dependencies / HarfBuzz - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - set INCLUDE=%INCLUDE%;%INCLIB% - set LIB=%LIB%;%INCLIB% - cd /D %BUILD%\harfbuzz-2.6.4 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DHB_HAVE_FREETYPE:BOOL=ON -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile harfbuzz - copy /Y /B src\*.h %INCLIB% - copy /Y /B *.lib %INCLIB% - shell: cmd - - # for Raqm + run: "& .\\winbuild\\build_dep_harfbuzz.cmd" - name: Build dependencies / FriBidi - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\fribidi-1.0.9 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - copy /Y /B %GITHUB_WORKSPACE%\winbuild\fribidi.cmake CMakeLists.txt - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile fribidi - copy /Y /B lib\*.h %INCLIB% - copy /Y /B *.lib %INCLIB% - shell: cmd - + run: "& .\\winbuild\\build_dep_fribidi.cmd" - name: Build dependencies / Raqm - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - set INCLUDE=%INCLUDE%;%INCLIB% - set LIB=%LIB%;%INCLIB% - cd /D %BUILD%\libraqm-0.7.0 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - copy /Y /B %GITHUB_WORKSPACE%\winbuild\raqm.cmake CMakeLists.txt - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile libraqm - copy /Y /B src\*.h %INCLIB% - copy /Y /B libraqm.dll %INCLIB% - shell: cmd + run: "& .\\winbuild\\build_dep_libraqm.cmd" - name: Build Pillow run: | @@ -188,9 +117,7 @@ jobs: # failing with PyPy3 - name: Enable heap verification if: "!contains(matrix.python-version, 'pypy')" - run: | - c:\"Program Files (x86)"\"Windows Kits"\10\Debuggers\x86\gflags.exe /p /enable %PYTHON%\python.exe - shell: cmd + run: "& 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x86\\gflags.exe' /p /enable $env:pythonLocation\\python.exe" - name: Test Pillow run: | @@ -219,19 +146,22 @@ jobs: - name: Upload coverage uses: codecov/codecov-action@v1 with: - file: ./coverage.xml - flags: GHA_Windows - name: ${{ runner.os }} Python ${{ matrix.python-version }} + file: ./coverage.xml + flags: GHA_Windows + name: ${{ runner.os }} Python ${{ matrix.python-version }} - name: Build wheel id: wheel if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" - run: "%GITHUB_WORKSPACE%\\winbuild\\build_pillow.cmd bdist_wheel" + run: | + for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a + %GITHUB_WORKSPACE%\\winbuild\\build_pillow.cmd bdist_wheel" + shell: cmd - uses: actions/upload-artifact@v1 if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" with: - name: dist + name: ${{ steps.wheel.outputs.dist }} path: dist # - uses: actions/upload-artifact@v1 diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 454d80f41..1354730ca 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -15,36 +15,15 @@ architectures = { "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, } -# use PYTHON + architecture to select config -pythons = { - "pypy3.6": {"config-x86": "3.5"}, - "3.5": {"config-x86": "3.5", "config-x64": "3.5"}, - "3.6": {"config-x86": "3.6", "config-x64": "3.6"}, - "3.7": {"config-x86": "3.6", "config-x64": "3.6"}, -} - # select preferred compiler -configs = { - "3.5": { - "vcvars_ver": "14.0", - "vs_ver": "2015", - }, - "3.6": { - "vs_ver": "2017", - }, +pythons = { + "pypy3.6": {"vs_ver": "2015", "vcvars_ver": "14.0"}, + "3.5": {"vs_ver": "2015", "vcvars_ver": "14.0"}, + "3.6": {"vs_ver": "2017"}, + "3.7": {"vs_ver": "2017"}, + "3.8": {"vs_ver": "2017"}, # TODO check } -# selected dependencies -deps_list = [ - "libjpeg-turbo-2.0.3", - "zlib-1.2.11", - "tiff-4.0.10", - "libwebp-1.0.3", - "freetype-2.10.1", - "lcms2-2.9", - "openjpeg-2.3.1", -] - header = [ cmd_set("BUILD", "{build_dir}"), cmd_set("INCLUDE", "{inc_dir}"), @@ -55,16 +34,18 @@ header = [ # dependencies deps = { - "libjpeg-turbo-2.0.3": { - "name": "libjpeg", + "libjpeg": { "url": SF_MIRROR + "/project/libjpeg-turbo/2.0.3/libjpeg-turbo-2.0.3.tar.gz", "filename": "libjpeg-turbo-2.0.3.tar.gz", + "dir": "libjpeg-turbo-2.0.3", "build": [ - cmd_cmake([ - "-DENABLE_SHARED:BOOL=FALSE", - "-DWITH_JPEG8:BOOL=TRUE", - "-DWITH_CRT_DLL:BOOL=TRUE", - ]), + cmd_cmake( + [ + "-DENABLE_SHARED:BOOL=FALSE", + "-DWITH_JPEG8:BOOL=TRUE", + "-DWITH_CRT_DLL:BOOL=TRUE", + ] + ), cmd_nmake(target="clean"), cmd_nmake(target="jpeg-static"), cmd_copy("jpeg-static.lib", "libjpeg.lib"), @@ -77,10 +58,10 @@ deps = { "libs": ["libjpeg.lib"], "bins": ["cjpeg.exe", "djpeg.exe"], }, - "zlib-1.2.11": { - "name": "zlib", + "zlib": { "url": "http://zlib.net/zlib1211.zip", "filename": "zlib1211.zip", + "dir": "zlib-1.2.11", "build": [ cmd_nmake(r"win32\Makefile.msc", "clean"), cmd_nmake(r"win32\Makefile.msc", "zlib.lib"), @@ -89,82 +70,90 @@ deps = { "headers": [r"z*.h"], "libs": [r"*.lib"], }, - "tiff-4.0.10": { - "name": "libtiff", + "libtiff": { # FIXME FTP timeout - "url": "ftp://download.osgeo.org/libtiff/tiff-4.0.10.tar.gz", - "filename": "tiff-4.0.10.tar.gz", + "url": "ftp://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz", + "filename": "tiff-4.1.0.tar.gz", + "dir": "tiff-4.1.0", "build": [ cmd_copy(r"{script_dir}\tiff.opt", "nmake.opt"), cmd_nmake("makefile.vc", "clean"), - cmd_nmake("makefile.vc", "lib", "RuntimeLibrary=-MT"), + cmd_nmake("makefile.vc", "lib"), ], "headers": [r"libtiff\tiff*.h"], "libs": [r"libtiff\*.lib"], # "bins": [r"libtiff\*.dll"], }, - "libwebp-1.0.3": { - "name": "libwebp", + "libwebp": { "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.0.3.tar.gz", # noqa: E501 "filename": "libwebp-1.0.3.tar.gz", + "dir": "libwebp-1.0.3", "build": [ cmd_rmdir(r"output\release-static"), # clean cmd_nmake( "Makefile.vc", "all", - ["CFG=release-static", "OBJDIR=output", "ARCH={architecture}"] + ["CFG=release-static", "OBJDIR=output", "ARCH={architecture}"], ), cmd_mkdir(r"{inc_dir}\webp"), cmd_copy(r"src\webp\*.h", r"{inc_dir}\webp"), ], "libs": [r"output\release-static\{architecture}\lib\*.lib"], }, - "freetype-2.10.1": { - "name": "freetype", + "freetype": { "url": "https://download.savannah.gnu.org/releases/freetype/freetype-2.10.1.tar.gz", # noqa: E501 "filename": "freetype-2.10.1.tar.gz", + "dir": "freetype-2.10.1", "patch": { r"builds\windows\vc2010\freetype.vcxproj": { # freetype setting is /MD for .dll and /MT for .lib, we need /MD - "MultiThreaded": "MultiThreadedDLL", - }, + "MultiThreaded": + "MultiThreadedDLL", + } }, "build": [ cmd_rmdir("objs"), - cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Clean"), # TODO failing on GHA # noqa: E501 - cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Build"), + cmd_msbuild( + r"builds\windows\vc2010\freetype.sln", "Release Static", "Clean" + ), + cmd_msbuild( + r"builds\windows\vc2010\freetype.sln", "Release Static", "Build" + ), cmd_xcopy("include", "{inc_dir}"), ], "libs": [r"objs\{msbuild_arch}\Release Static\freetype.lib"], # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], }, - "lcms2-2.9": { - "name": "lcms2", + "lcms2": { "url": SF_MIRROR + "/project/lcms/lcms/2.9/lcms2-2.9.tar.gz", "filename": "lcms2-2.9.tar.gz", + "dir": "lcms2-2.9", "patch": { r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { - # lcms2-2.8\VC2015 setting is /MD for x86 and /MT for x64, we need /MD always - "MultiThreaded": "MultiThreadedDLL", - # retarget to default msvc - "v141": "$(DefaultPlatformToolset)", - # retarget to latest SDK 10.0 - "8.1": "10.0", - }, + # default is /MD for x86 and /MT for x64, we need /MD always + "MultiThreaded": + "MultiThreadedDLL", + # retarget to default toolset (selected by vcvarsall.bat) + "v141": + "$(DefaultPlatformToolset)", + # retarget to latest (selected by vcvarsall.bat) + "8.1": + "$(WindowsSDKVersion)", # noqa E501 + } }, "build": [ cmd_rmdir("Lib"), - cmd_rmdir(r"Projects\VC{vs_ver}\Release"), - cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "Clean"), - cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "lcms2_static"), + cmd_rmdir(r"Projects\VC2017\Release"), + cmd_msbuild(r"Projects\VC2017\lcms2.sln", "Release", "Clean"), + cmd_msbuild(r"Projects\VC2017\lcms2.sln", "Release", "lcms2_static"), cmd_xcopy("include", "{inc_dir}"), ], "libs": [r"Lib\MS\*.lib"], }, - "openjpeg-2.3.1": { - "name": "openjpeg", + "openjpeg": { "url": "https://github.com/uclouvain/openjpeg/archive/v2.3.1.tar.gz", "filename": "openjpeg-2.3.1.tar.gz", + "dir": "openjpeg-2.3.1", "build": [ cmd_cmake(("-DBUILD_THIRDPARTY:BOOL=OFF", "-DBUILD_SHARED_LIBS:BOOL=OFF")), cmd_nmake(target="clean"), @@ -174,6 +163,64 @@ deps = { ], "libs": [r"bin\*.lib"], }, + "libimagequant": { + # ba653c8: Merge tag '2.12.5' into msvc + "url": "https://github.com/ImageOptim/libimagequant/archive/ba653c8ccb34dde4e21c6076d85a72d21ed9d971.zip", # noqa: E501 + "filename": "libimagequant-ba653c8ccb34dde4e21c6076d85a72d21ed9d971.zip", + "dir": "libimagequant-ba653c8ccb34dde4e21c6076d85a72d21ed9d971", + "patch": { + "CMakeLists.txt": { + "add_library": "add_compile_options(-openmp-)\r\nadd_library", + " SHARED": " STATIC", + } + }, + "build": [ + # lint: do not inline + cmd_cmake(), + cmd_nmake(target="clean"), + cmd_nmake(), + ], + "headers": [r"*.h"], + "libs": [r"*.lib"], + }, + "harfbuzz": { + "url": "https://github.com/harfbuzz/harfbuzz/archive/2.6.1.zip", + "filename": "harfbuzz-2.6.1.zip", + "dir": "harfbuzz-2.6.1", + "build": [ + cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), + cmd_nmake(target="clean"), + cmd_nmake(target="harfbuzz"), + ], + "headers": [r"src\*.h"], + "libs": [r"*.lib"], + }, + "fribidi": { + "url": "https://github.com/fribidi/fribidi/archive/v1.0.7.zip", + "filename": "fribidi-1.0.7.zip", + "dir": "fribidi-1.0.7", + "build": [ + cmd_copy(r"{script_dir}\fribidi.cmake", r"CMakeLists.txt"), + cmd_cmake(), + cmd_nmake(target="clean"), + cmd_nmake(target="fribidi"), + ], + "headers": [r"lib\*.h"], + "libs": [r"*.lib"], + }, + "libraqm": { + "url": "https://github.com/HOST-Oman/libraqm/archive/v0.7.0.zip", + "filename": "libraqm-0.7.0.zip", + "dir": "libraqm-0.7.0", + "build": [ + cmd_copy(r"{script_dir}\raqm.cmake", r"CMakeLists.txt"), + cmd_cmake(), + cmd_nmake(target="clean"), + cmd_nmake(target="libraqm"), + ], + "headers": [r"src\*.h"], + "bins": [r"libraqm.dll"], + }, } @@ -185,14 +232,25 @@ def find_vs2017(config): return None try: - vspath = subprocess.check_output([ - os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), - "-latest", - "-prerelease", - "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", - "-property", "installationPath", - "-products", "*", - ]).decode(encoding="mbcs").strip() + vspath = ( + subprocess.check_output( + [ + os.path.join( + root, "Microsoft Visual Studio", "Installer", "vswhere.exe" + ), + "-latest", + "-prerelease", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "installationPath", + "-products", + "*", + ] + ) + .decode(encoding="mbcs") + .strip() + ) except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): print("vswhere not found") return None @@ -211,13 +269,13 @@ def find_vs2017(config): # vs2017 msbuild = os.path.join(vspath, "MSBuild", "15.0", "Bin", "MSBuild.exe") if os.path.isfile(msbuild): - default_platform_toolset = "v140" + # default_platform_toolset = "v140" vs["msbuild"] = '"{}"'.format(msbuild) else: # vs2019 msbuild = os.path.join(vspath, "MSBuild", "Current", "Bin", "MSBuild.exe") if os.path.isfile(msbuild): - default_platform_toolset = "v142" + # default_platform_toolset = "v142" vs["msbuild"] = '"{}"'.format(msbuild) else: print("Visual Studio MSBuild not found") @@ -228,7 +286,9 @@ def find_vs2017(config): if not os.path.isfile(vcvarsall): print("Visual Studio vcvarsall not found") return None - vcvars_ver = "-vcvars_ver={}".format(config["vcvars_ver"]) if "vcvars_ver" in config else "" + vcvars_ver = ( + "-vcvars_ver={}".format(config["vcvars_ver"]) if "vcvars_ver" in config else "" + ) vs["header"].append('call "{}" {{vcvars_arch}} {}'.format(vcvarsall, vcvars_ver)) return vs @@ -338,22 +398,23 @@ def get_footer(dep): def build_dep(name): dep = deps[name] - file = "build_dep_{name}.cmd".format(name=dep["name"]) + dir = dep["dir"] + file = "build_dep_{name}.cmd".format(**locals()) extract_dep(dep["url"], dep["filename"]) for patch_file, patch_list in dep.get("patch", {}).items(): - patch_file = os.path.join(build_dir, name, patch_file) + patch_file = os.path.join(build_dir, dir, patch_file.format(**prefs)) with open(patch_file, "r") as f: text = f.read() for patch_from, patch_to in patch_list.items(): - text = text.replace(patch_from, patch_to) + text = text.replace(patch_from.format(**prefs), patch_to.format(**prefs)) with open(patch_file, "w") as f: f.write(text) lines = [ - "@echo Building {name} ({dir})...".format(name=dep["name"], dir=name), - "cd /D %s" % os.path.join(build_dir, name), + "@echo ---- Building {name} ({dir}) ----".format(**locals()), + "cd /D %s" % os.path.join(build_dir, dir), *prefs["header"], *dep.get("build", []), *get_footer(dep), @@ -368,7 +429,7 @@ def build_dep_all(): "$ErrorActionPreference = 'stop'", "cd {script_dir}", ] - for dep_name in deps_list: + for dep_name in deps: lines.append('cmd.exe /c "%s"' % build_dep(dep_name)) write_script("build_dep_all.ps1", lines) @@ -380,10 +441,13 @@ def build_pillow(wheel=False): lines.extend(prefs["header"]) lines.extend( [ + "@echo ---- Building Pillow (%s) ----", cmd_cd("{pillow_dir}"), cmd_append("LIB", r"{python_dir}\tcl"), - r'"{{python_dir}}\python.exe" setup.py build_ext %*', - # r"""%PYTHON%\python.exe selftest.py --installed""", + cmd_set("MSSdk", "1"), + cmd_set("DISTUTILS_USE_SDK", "1"), + cmd_set("py_vcruntime_redist", "true"), + r'"{python_dir}\python.exe" setup.py build_ext %*', ] ) @@ -393,7 +457,9 @@ def build_pillow(wheel=False): if __name__ == "__main__": script_dir = os.path.dirname(os.path.realpath(__file__)) depends_dir = os.path.join(script_dir, "depends") - python_dir = os.environ.get("PYTHON", os.path.dirname(os.path.realpath(sys.executable))) + python_dir = os.environ.get( + "PYTHON", os.path.dirname(os.path.realpath(sys.executable)) + ) # copy binaries to this directory path_dir = os.environ.get("PILLOW_BIN") @@ -410,15 +476,17 @@ if __name__ == "__main__": # use PYTHON to select python version python_prefs = match(pythons, python_dir) if python_prefs is None: - raise KeyError("Failed to determine Python version from PYTHON: " + python_dir) + raise KeyError("Failed to determine Python version for {}".format(python_dir)) - print("Target: Python {python_version} {architecture} at: {python_dir}".format(python_version=python_prefs["name"], architecture=architecture, python_dir=python_dir)) + print( + "Target: Python {python_version} {architecture} at: {python_dir}".format( + python_version=python_prefs["name"], + architecture=architecture, + python_dir=python_dir, + ) + ) - # use python version + architecture to select build config - config_name = python_prefs["config-" + architecture] - config = configs[config_name] - - vs2017 = find_vs2017(config) + vs2017 = find_vs2017(python_prefs) if vs2017 is None: raise RuntimeError("Visual Studio 2017 not found") @@ -430,13 +498,12 @@ if __name__ == "__main__": # # print("Found Windows SDK 7.1A at: {}".format(sdk71a["sdk_dir"])) - build_dir = os.path.join(script_dir, "build", config_name, architecture) + build_dir = os.path.join(script_dir, "build", python_prefs["name"], architecture) lib_dir = os.path.join(build_dir, "lib") inc_dir = os.path.join(build_dir, "inc") bin_dir = os.path.join(build_dir, "bin") - # for path in [lib_dir, inc_dir, bin_dir]: - # shutil.rmtree(path) + # shutil.rmtree(build_dir) for path in [depends_dir, build_dir, lib_dir, inc_dir, bin_dir]: os.makedirs(path, exist_ok=True) @@ -455,7 +522,7 @@ if __name__ == "__main__": "cmake": "cmake.exe", } - dicts = [vs2017, arch_prefs, python_prefs, config] + dicts = [vs2017, arch_prefs, python_prefs] for x in dicts: prefs.update(x) prefs["header"] = sum((x.get("header", []) for x in dicts), header) + ["@echo on"] From aa4592c95184bc847898a446f20275f5fd03da39 Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 31 Dec 2019 20:57:17 +0100 Subject: [PATCH 09/27] clean msvs selection --- .github/workflows/test-windows.yml | 9 +-- winbuild/build_prepare.py | 109 ++++------------------------- 2 files changed, 18 insertions(+), 100 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index fc341e446..910a5b927 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -58,7 +58,7 @@ jobs: "%pythonLocation%\python.exe" -m pip install wheel pytest pytest-cov shell: cmd - - name: Fetch dependencies + - name: Prepare dependencies run: | 7z x ..\pillow-depends\nasm-2.14.02-win64.zip "-o$env:RUNNER_WORKSPACE\" Write-Host "::add-path::$env:RUNNER_WORKSPACE\nasm-2.14.02" @@ -66,14 +66,13 @@ jobs: ..\pillow-depends\gs950w32.exe /S Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" - $env:PYTHON=$env:pythonLocation mkdir $env:GITHUB_WORKSPACE\winbuild\depends\ xcopy ..\pillow-depends\*.zip $env:GITHUB_WORKSPACE\winbuild\depends\ xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\depends\ xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\ + + $env:PYTHON=$env:pythonLocation cd $env:GITHUB_WORKSPACE/winbuild/ - # reg query "HKLM\Software\Microsoft\Microsoft SDKs\Windows" /s - ls 'C:\Program Files (x86)\Microsoft SDKs\Windows\' & python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py shell: pwsh @@ -108,8 +107,6 @@ jobs: - name: Build Pillow run: | cd $env:GITHUB_WORKSPACE - # & winbuild\build_dep_tcl.cmd - # & winbuild\build_dep_tk.cmd & winbuild\build_pillow.cmd install & $env:pythonLocation\python.exe selftest.py --installed shell: pwsh diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 1354730ca..77f7716a5 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -9,21 +9,11 @@ from commands import * SF_MIRROR = "http://iweb.dl.sourceforge.net" -# use PYTHON to select architecture architectures = { "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, } -# select preferred compiler -pythons = { - "pypy3.6": {"vs_ver": "2015", "vcvars_ver": "14.0"}, - "3.5": {"vs_ver": "2015", "vcvars_ver": "14.0"}, - "3.6": {"vs_ver": "2017"}, - "3.7": {"vs_ver": "2017"}, - "3.8": {"vs_ver": "2017"}, # TODO check -} - header = [ cmd_set("BUILD", "{build_dir}"), cmd_set("INCLUDE", "{inc_dir}"), @@ -225,7 +215,7 @@ deps = { # based on distutils._msvccompiler from CPython 3.7.4 -def find_vs2017(config): +def find_msvs(): root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") if not root: print("Program Files not found") @@ -269,77 +259,25 @@ def find_vs2017(config): # vs2017 msbuild = os.path.join(vspath, "MSBuild", "15.0", "Bin", "MSBuild.exe") if os.path.isfile(msbuild): - # default_platform_toolset = "v140" vs["msbuild"] = '"{}"'.format(msbuild) else: # vs2019 msbuild = os.path.join(vspath, "MSBuild", "Current", "Bin", "MSBuild.exe") if os.path.isfile(msbuild): - # default_platform_toolset = "v142" vs["msbuild"] = '"{}"'.format(msbuild) else: print("Visual Studio MSBuild not found") return None - # vs["header"].append(cmd_set("DefaultPlatformToolset", default_platform_toolset)) vcvarsall = os.path.join(vspath, "VC", "Auxiliary", "Build", "vcvarsall.bat") if not os.path.isfile(vcvarsall): print("Visual Studio vcvarsall not found") return None - vcvars_ver = ( - "-vcvars_ver={}".format(config["vcvars_ver"]) if "vcvars_ver" in config else "" - ) - vs["header"].append('call "{}" {{vcvars_arch}} {}'.format(vcvarsall, vcvars_ver)) + vs["header"].append('call "{}" {{vcvars_arch}}'.format(vcvarsall)) return vs -def find_sdk71a(): - try: - print("trace: opening sdk key") - key = winreg.OpenKeyEx( - winreg.HKEY_LOCAL_MACHINE, - r"SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A", - access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY, - ) - print("trace: opened sdk key") - except OSError: - return None - print("trace: enumerating key") - with key: - for i in count(): - try: - v_name, v_value, v_type = winreg.EnumValue(key, i) - print("trace: entry: {} {} = {}".format(v_type, v_name, v_value)) - except OSError: - return None - if v_name == "InstallationFolder" and v_type == winreg.REG_SZ: - sdk_dir = v_value - print("trace: found dir: {}".format(sdk_dir)) - break - else: - return None - - print("trace: closed key") - - if not os.path.isdir(sdk_dir): - return None - - print("trace: creating dict") - - sdk = { - "header": [ - # for win32.mak - cmd_append("INCLUDE", os.path.join(sdk_dir, "Include")), - # for ghostscript - cmd_set("RCOMP", '"{}"'.format(os.path.join(sdk_dir, "Bin", "RC.EXE"))), - ], - "sdk_dir": sdk_dir, - } - - return sdk - - def match(values, target): for key, value in values.items(): if key in target: @@ -461,6 +399,8 @@ if __name__ == "__main__": "PYTHON", os.path.dirname(os.path.realpath(sys.executable)) ) + print("Target Python: {}".format(python_dir)) + # copy binaries to this directory path_dir = os.environ.get("PILLOW_BIN") @@ -473,42 +413,26 @@ if __name__ == "__main__": else: architecture = arch_prefs["name"] - # use PYTHON to select python version - python_prefs = match(pythons, python_dir) - if python_prefs is None: - raise KeyError("Failed to determine Python version for {}".format(python_dir)) + print("Target Architecture: {}".format(architecture)) - print( - "Target: Python {python_version} {architecture} at: {python_dir}".format( - python_version=python_prefs["name"], - architecture=architecture, - python_dir=python_dir, + msvs = find_msvs() + if msvs is None: + raise RuntimeError( + "Visual Studio not found. Please install Visual Studio 2017 or newer." ) - ) - vs2017 = find_vs2017(python_prefs) - if vs2017 is None: - raise RuntimeError("Visual Studio 2017 not found") + print("Found Visual Studio at: {}".format(msvs["vs_dir"])) - print("Found Visual Studio at: {}".format(vs2017["vs_dir"])) - - # sdk71a = find_sdk71a() - # if sdk71a is None: - # raise RuntimeError("Windows SDK v7.1A not found") - # - # print("Found Windows SDK 7.1A at: {}".format(sdk71a["sdk_dir"])) - - build_dir = os.path.join(script_dir, "build", python_prefs["name"], architecture) + build_dir = os.path.join(script_dir, "build", architecture) lib_dir = os.path.join(build_dir, "lib") inc_dir = os.path.join(build_dir, "inc") bin_dir = os.path.join(build_dir, "bin") - # shutil.rmtree(build_dir) + shutil.rmtree(build_dir, ignore_errors=True) for path in [depends_dir, build_dir, lib_dir, inc_dir, bin_dir]: os.makedirs(path, exist_ok=True) prefs = { - "python_version": python_prefs["name"], "architecture": architecture, "script_dir": script_dir, "depends_dir": depends_dir, @@ -521,12 +445,9 @@ if __name__ == "__main__": # TODO auto find: "cmake": "cmake.exe", } - - dicts = [vs2017, arch_prefs, python_prefs] - for x in dicts: - prefs.update(x) - prefs["header"] = sum((x.get("header", []) for x in dicts), header) + ["@echo on"] - del prefs["name"] + prefs.update(msvs) + prefs.update(arch_prefs) + prefs["header"] = sum([header, msvs["header"], ["@echo on"]], []) build_dep_all() build_pillow() From ae0c0f3d0ad558e9d7cd74518c25003dff77d65a Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 1 Jan 2020 01:48:13 +0100 Subject: [PATCH 10/27] cleanup --- .github/workflows/test-windows.yml | 34 +++---- winbuild/build_prepare.py | 140 ++++++++++++++++++++++++----- winbuild/commands.py | 93 ------------------- 3 files changed, 131 insertions(+), 136 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 910a5b927..7120d7c32 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -77,37 +77,37 @@ jobs: shell: pwsh - name: Build dependencies / libjpeg - run: "& .\\winbuild\\build_dep_libjpeg.cmd" + run: "& .\\winbuild\\build\\build_dep_libjpeg.cmd" - name: Build dependencies / zlib - run: "& .\\winbuild\\build_dep_zlib.cmd" + run: "& .\\winbuild\\build\\build_dep_zlib.cmd" - name: Build dependencies / LibTiff - run: "& .\\winbuild\\build_dep_libtiff.cmd" + run: "& .\\winbuild\\build\\build_dep_libtiff.cmd" - name: Build dependencies / WebP - run: "& .\\winbuild\\build_dep_libwebp.cmd" + run: "& .\\winbuild\\build\\build_dep_libwebp.cmd" - name: Build dependencies / FreeType - run: "& .\\winbuild\\build_dep_freetype.cmd" + run: "& .\\winbuild\\build\\build_dep_freetype.cmd" - name: Build dependencies / LCMS2 - run: "& .\\winbuild\\build_dep_lcms2.cmd" + run: "& .\\winbuild\\build\\build_dep_lcms2.cmd" - name: Build dependencies / OpenJPEG - run: "& .\\winbuild\\build_dep_openjpeg.cmd" + run: "& .\\winbuild\\build\\build_dep_openjpeg.cmd" # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant if: "github.event_name != 'push' || contains(matrix.python-version, 'pypy')" - run: "& .\\winbuild\\build_dep_libimagequant.cmd" + run: "& .\\winbuild\\build\\build_dep_libimagequant.cmd" # Raqm dependencies - name: Build dependencies / HarfBuzz - run: "& .\\winbuild\\build_dep_harfbuzz.cmd" + run: "& .\\winbuild\\build\\build_dep_harfbuzz.cmd" - name: Build dependencies / FriBidi - run: "& .\\winbuild\\build_dep_fribidi.cmd" + run: "& .\\winbuild\\build\\build_dep_fribidi.cmd" - name: Build dependencies / Raqm - run: "& .\\winbuild\\build_dep_libraqm.cmd" + run: "& .\\winbuild\\build\\build_dep_libraqm.cmd" - name: Build Pillow run: | cd $env:GITHUB_WORKSPACE - & winbuild\build_pillow.cmd install + & winbuild\build\build_pillow.cmd install & $env:pythonLocation\python.exe selftest.py --installed shell: pwsh @@ -145,14 +145,14 @@ jobs: with: file: ./coverage.xml flags: GHA_Windows - name: ${{ runner.os }} Python ${{ matrix.python-version }} + name: ${{ runner.os }} Python ${{ matrix.python-version }} ${{ matrix.architecture }} - name: Build wheel id: wheel if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" run: | for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a - %GITHUB_WORKSPACE%\\winbuild\\build_pillow.cmd bdist_wheel" + %GITHUB_WORKSPACE%\\winbuild\\build\\build_pillow.cmd bdist_wheel" shell: cmd - uses: actions/upload-artifact@v1 @@ -160,9 +160,3 @@ jobs: with: name: ${{ steps.wheel.outputs.dist }} path: dist - -# - uses: actions/upload-artifact@v1 -# if: matrix.architecture == 'x86' -# with: -# name: lib -# path: "winbuild\\build\\3.x\\x86\\lib" diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 77f7716a5..c772a580f 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -2,10 +2,90 @@ import os import shutil import subprocess import sys -import winreg -from itertools import count -from commands import * + +def cmd_cd(path): + return "cd /D {path}".format(**locals()) + + +def cmd_set(name, value): + return "set {name}={value}".format(**locals()) + + +def cmd_append(name, value): + op = "path " if name == "PATH" else "set {name}=" + return (op + "%{name}%;{value}").format(**locals()) + + +def cmd_copy(src, tgt): + return 'copy /Y /B "{src}" "{tgt}"'.format(**locals()) + + +def cmd_xcopy(src, tgt): + return 'xcopy /Y /E "{src}" "{tgt}"'.format(**locals()) + + +def cmd_mkdir(path): + return 'mkdir "{path}"'.format(**locals()) + + +def cmd_rmdir(path): + return 'rmdir /S /Q "{path}"'.format(**locals()) + + +def cmd_nmake(makefile=None, target="", params=None): + if params is None: + params = "" + elif isinstance(params, list) or isinstance(params, tuple): + params = " ".join(params) + else: + params = str(params) + + return " ".join( + [ + "{{nmake}}", + "-nologo", + '-f "{makefile}"' if makefile is not None else "", + "{params}", + '"{target}"', + ] + ).format(**locals()) + + +def cmd_cmake(params=None, file="."): + if params is None: + params = "" + elif isinstance(params, list) or isinstance(params, tuple): + params = " ".join(params) + else: + params = str(params) + return " ".join( + [ + "{{cmake}}", + "-DCMAKE_VERBOSE_MAKEFILE=ON", + "-DCMAKE_RULE_MESSAGES:BOOL=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "{params}", + '-G "NMake Makefiles"', + '"{file}"', + ] + ).format(**locals()) + + +def cmd_msbuild( + file, configuration="Release", target="Build", platform="{msbuild_arch}" +): + return " ".join( + [ + "{{msbuild}}", + "{file}", + '/t:"{target}"', + '/p:Configuration="{configuration}"', + "/p:Platform={platform}", + "/m", + ] + ).format(**locals()) + SF_MIRROR = "http://iweb.dl.sourceforge.net" @@ -61,8 +141,7 @@ deps = { "libs": [r"*.lib"], }, "libtiff": { - # FIXME FTP timeout - "url": "ftp://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz", + "url": "https://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz", "filename": "tiff-4.1.0.tar.gz", "dir": "tiff-4.1.0", "build": [ @@ -147,7 +226,7 @@ deps = { "build": [ cmd_cmake(("-DBUILD_THIRDPARTY:BOOL=OFF", "-DBUILD_SHARED_LIBS:BOOL=OFF")), cmd_nmake(target="clean"), - cmd_nmake(), + cmd_nmake(target="openjp2"), cmd_mkdir(r"{inc_dir}\openjpeg-2.3.1"), cmd_copy(r"src\lib\openjp2\*.h", r"{inc_dir}\openjpeg-2.3.1"), ], @@ -303,6 +382,8 @@ def extract_dep(url, filename): ex = e else: raise RuntimeError(ex) + + print("Extracting " + filename) if filename.endswith(".zip"): with zipfile.ZipFile(file) as zf: zf.extractall(build_dir) @@ -314,7 +395,7 @@ def extract_dep(url, filename): def write_script(name, lines): - name = os.path.join(script_dir, name) + name = os.path.join(build_dir, name) lines = [line.format(**prefs) for line in lines] print("Writing " + name) with open(name, "w") as f: @@ -350,8 +431,11 @@ def build_dep(name): with open(patch_file, "w") as f: f.write(text) + banner = "Building {name} ({dir})".format(**locals()) lines = [ - "@echo ---- Building {name} ({dir}) ----".format(**locals()), + "@echo " + ("=" * 70), + "@echo ==== {:<60} ====".format(banner), + "@echo " + ("=" * 70), "cd /D %s" % os.path.join(build_dir, dir), *prefs["header"], *dep.get("build", []), @@ -363,13 +447,12 @@ def build_dep(name): def build_dep_all(): - lines = [ - "$ErrorActionPreference = 'stop'", - "cd {script_dir}", - ] + lines = ["@echo on"] for dep_name in deps: - lines.append('cmd.exe /c "%s"' % build_dep(dep_name)) - write_script("build_dep_all.ps1", lines) + lines.append(r'cmd.exe /c "{{build_dir}}\{}"'.format(build_dep(dep_name))) + lines.append("if errorlevel 1 echo Build failed! && exit /B 1") + lines.append("@echo All Pillow dependencies built successfully!") + write_script("build_dep_all.cmd", lines) def build_pillow(wheel=False): @@ -379,7 +462,7 @@ def build_pillow(wheel=False): lines.extend(prefs["header"]) lines.extend( [ - "@echo ---- Building Pillow (%s) ----", + "@echo ---- Building Pillow (build_ext %*) ----", cmd_cd("{pillow_dir}"), cmd_append("LIB", r"{python_dir}\tcl"), cmd_set("MSSdk", "1"), @@ -393,39 +476,50 @@ def build_pillow(wheel=False): if __name__ == "__main__": + # winbuild directory script_dir = os.path.dirname(os.path.realpath(__file__)) + + # dependency cache directory depends_dir = os.path.join(script_dir, "depends") + print("Caching dependencies in:", depends_dir) + + # python bin directory python_dir = os.environ.get( "PYTHON", os.path.dirname(os.path.realpath(sys.executable)) ) - - print("Target Python: {}".format(python_dir)) + print("Target Python:", python_dir) # copy binaries to this directory path_dir = os.environ.get("PILLOW_BIN") + print("Copying binary files to:", path_dir) # use PYTHON to select architecture arch_prefs = match(architectures, python_dir) if arch_prefs is None: architecture = "x86" - print("WARN: Could not determine architecture, guessing " + architecture) arch_prefs = architectures[architecture] else: architecture = arch_prefs["name"] - - print("Target Architecture: {}".format(architecture)) + print("Target Architecture:", architecture) msvs = find_msvs() if msvs is None: raise RuntimeError( "Visual Studio not found. Please install Visual Studio 2017 or newer." ) + print("Found Visual Studio at:", msvs["vs_dir"]) - print("Found Visual Studio at: {}".format(msvs["vs_dir"])) + # build root directory + build_dir = os.environ.get("PILLOW_BUILD", os.path.join(script_dir, "build")) + print("Using output directory:", build_dir) - build_dir = os.path.join(script_dir, "build", architecture) - lib_dir = os.path.join(build_dir, "lib") + # build directory for *.h files inc_dir = os.path.join(build_dir, "inc") + + # build directory for *.lib files + lib_dir = os.path.join(build_dir, "lib") + + # build directory for *.bin files bin_dir = os.path.join(build_dir, "bin") shutil.rmtree(build_dir, ignore_errors=True) diff --git a/winbuild/commands.py b/winbuild/commands.py index f78a596f1..ea6e73b4e 100644 --- a/winbuild/commands.py +++ b/winbuild/commands.py @@ -1,96 +1,3 @@ # builtins -def cmd_cd(path): - return "cd /D {path}".format(**locals()) - - -def cmd_set(name, value): - return "set {name}={value}".format(**locals()) - - -def cmd_prepend(name, value): - op = "path " if name == "PATH" else "set {name}=" - return (op + "{value};%{name}%").format(**locals()) - - -def cmd_append(name, value): - op = "path " if name == "PATH" else "set {name}=" - return (op + "%{name}%;{value}").format(**locals()) - - -def cmd_copy(src, tgt): - return 'copy /Y /B "{src}" "{tgt}"'.format(**locals()) - - -def cmd_xcopy(src, tgt): - return 'xcopy /Y /E "{src}" "{tgt}"'.format(**locals()) - - -def cmd_mkdir(path): - return 'mkdir "{path}"'.format(**locals()) - - -def cmd_rmdir(path): - return 'rmdir /S /Q "{path}"'.format(**locals()) - - -def cmd_if_eq(a, b, cmd): - return 'if "{a}"=="{b}" {cmd}'.format(**locals()) - - -# tools - - -def cmd_nmake(makefile=None, target="", params=None): - if params is None: - params = "" - elif isinstance(params, list) or isinstance(params, tuple): - params = " ".join(params) - else: - params = str(params) - - return " ".join( - [ - "{{nmake}}", - "-nologo", - '-f "{makefile}"' if makefile is not None else "", - "{params}", - '"{target}"', - ] - ).format(**locals()) - - -def cmd_cmake(params=None, file="."): - if params is None: - params = "" - elif isinstance(params, list) or isinstance(params, tuple): - params = " ".join(params) - else: - params = str(params) - return " ".join( - [ - "{{cmake}}", - "-DCMAKE_VERBOSE_MAKEFILE=ON", - "-DCMAKE_RULE_MESSAGES:BOOL=OFF", - "-DCMAKE_BUILD_TYPE=Release", - "{params}", - '-G "NMake Makefiles"', - '"{file}"', - ] - ).format(**locals()) - - -def cmd_msbuild( - file, configuration="Release", target="Build", platform="{msbuild_arch}" -): - return " ".join( - [ - "{{msbuild}}", - "{file}", - '/t:"{target}"', - '/p:Configuration="{configuration}"', - "/p:Platform={platform}", - "/m", - ] - ).format(**locals()) From 510d010f56e9ccf3af5ec7b0a2935bcb73fa87e8 Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 1 Jan 2020 02:43:00 +0100 Subject: [PATCH 11/27] cleanup gha config --- .github/workflows/test-windows.yml | 41 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 7120d7c32..7c1887745 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -26,14 +26,16 @@ jobs: name: Python ${{ matrix.python-version }} ${{ matrix.architecture }} steps: - - uses: actions/checkout@v1 + - name: Checkout Pillow + uses: actions/checkout@v1 - - uses: actions/checkout@v1 + - name: Checkout cached dependencies + uses: actions/checkout@v1 with: repository: python-pillow/pillow-depends ref: master - - name: Cache + - name: Cache pip uses: actions/cache@v1 with: path: ~\AppData\Local\pip\Cache @@ -50,13 +52,11 @@ jobs: python-version: ${{ matrix.python-version }} architecture: ${{ matrix.architecture }} - - name: Build system information + - name: Print build system information run: python .github/workflows/system-info.py - name: pip install wheel pytest pytest-cov - run: | - "%pythonLocation%\python.exe" -m pip install wheel pytest pytest-cov - shell: cmd + run: python -m pip install wheel pytest pytest-cov - name: Prepare dependencies run: | @@ -76,37 +76,36 @@ jobs: & python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py shell: pwsh - - name: Build dependencies / libjpeg - run: "& .\\winbuild\\build\\build_dep_libjpeg.cmd" + - name: Build dependencies / libjpeg-turbo + run: "& winbuild\\build\\build_dep_libjpeg.cmd" - name: Build dependencies / zlib - run: "& .\\winbuild\\build\\build_dep_zlib.cmd" + run: "& winbuild\\build\\build_dep_zlib.cmd" - name: Build dependencies / LibTiff - run: "& .\\winbuild\\build\\build_dep_libtiff.cmd" + run: "& winbuild\\build\\build_dep_libtiff.cmd" - name: Build dependencies / WebP - run: "& .\\winbuild\\build\\build_dep_libwebp.cmd" + run: "& winbuild\\build\\build_dep_libwebp.cmd" - name: Build dependencies / FreeType - run: "& .\\winbuild\\build\\build_dep_freetype.cmd" + run: "& winbuild\\build\\build_dep_freetype.cmd" - name: Build dependencies / LCMS2 - run: "& .\\winbuild\\build\\build_dep_lcms2.cmd" + run: "& winbuild\\build\\build_dep_lcms2.cmd" - name: Build dependencies / OpenJPEG - run: "& .\\winbuild\\build\\build_dep_openjpeg.cmd" + run: "& winbuild\\build\\build_dep_openjpeg.cmd" # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant if: "github.event_name != 'push' || contains(matrix.python-version, 'pypy')" - run: "& .\\winbuild\\build\\build_dep_libimagequant.cmd" + run: "& winbuild\\build\\build_dep_libimagequant.cmd" # Raqm dependencies - name: Build dependencies / HarfBuzz - run: "& .\\winbuild\\build\\build_dep_harfbuzz.cmd" + run: "& winbuild\\build\\build_dep_harfbuzz.cmd" - name: Build dependencies / FriBidi - run: "& .\\winbuild\\build\\build_dep_fribidi.cmd" + run: "& winbuild\\build\\build_dep_fribidi.cmd" - name: Build dependencies / Raqm - run: "& .\\winbuild\\build\\build_dep_libraqm.cmd" + run: "& winbuild\\build\\build_dep_libraqm.cmd" - name: Build Pillow run: | - cd $env:GITHUB_WORKSPACE & winbuild\build\build_pillow.cmd install & $env:pythonLocation\python.exe selftest.py --installed shell: pwsh @@ -152,7 +151,7 @@ jobs: if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" run: | for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a - %GITHUB_WORKSPACE%\\winbuild\\build\\build_pillow.cmd bdist_wheel" + winbuild\\build\\build_pillow.cmd bdist_wheel" shell: cmd - uses: actions/upload-artifact@v1 From af322141ce885a183c2ff3c68a142efd7819c3ab Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 1 Jan 2020 02:44:11 +0100 Subject: [PATCH 12/27] update appveyor --- .appveyor.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index e395f2ae6..bb1e91d0c 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -31,8 +31,9 @@ install: - curl -fsSL -o pillow-depends.zip https://github.com/python-pillow/pillow-depends/archive/master.zip - 7z x pillow-depends.zip -oc:\ - mv c:\pillow-depends-master c:\pillow-depends -- xcopy c:\pillow-depends\*.zip c:\pillow\winbuild\ -- xcopy c:\pillow-depends\*.tar.gz c:\pillow\winbuild\ +- mkdir c:\pillow\winbuild\depends +- xcopy c:\pillow-depends\*.zip c:\pillow\winbuild\depends\ +- xcopy c:\pillow-depends\*.tar.gz c:\pillow\winbuild\depends\ - xcopy /s c:\pillow-depends\test_images\* c:\pillow\tests\images - cd c:\pillow\winbuild\ - ps: | @@ -47,8 +48,9 @@ install: } else { - c:\python37\python.exe c:\pillow\winbuild\build_dep.py - c:\pillow\winbuild\build_deps.cmd + $env:PILLOW_BIN = "c:\pillow\" + c:\python37\python.exe c:\pillow\winbuild\build_prepare.py + c:\pillow\winbuild\build\build_dep_all.cmd $host.SetShouldExit(0) } - curl -fsSL -o gs952.exe https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs952/gs952w32.exe @@ -65,7 +67,7 @@ build_script: } else { - & $env:PYTHON/$env:EXECUTABLE c:\pillow\winbuild\build.py + & $env:PYTHON/$env:EXECUTABLE c:\pillow\winbuild\build\build_pillow.cmd install $host.SetShouldExit(0) } - cd c:\pillow @@ -98,7 +100,7 @@ before_deploy: - cd c:\pillow - '%PYTHON%\%PIP_DIR%\pip.exe install wheel' - cd c:\pillow\winbuild\ - - '%PYTHON%\%EXECUTABLE% c:\pillow\winbuild\build.py --wheel' + - '%PYTHON%\%EXECUTABLE% c:\pillow\winbuild\build\build_pillow.cmd bdist_wheel' - cd c:\pillow - ps: Get-ChildItem .\dist\*.* | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } From 15ce881a2b93b0ae035bf757704bd58382c11e07 Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 1 Jan 2020 02:47:23 +0100 Subject: [PATCH 13/27] fix appveyor --- .appveyor.yml | 7 +++++-- Tests/test_imageshow.py | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index bb1e91d0c..645c2256f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,4 +1,5 @@ version: '{build}' +image: Visual Studio 2017 clone_folder: c:\pillow init: - ECHO %PYTHON% @@ -35,6 +36,8 @@ install: - xcopy c:\pillow-depends\*.zip c:\pillow\winbuild\depends\ - xcopy c:\pillow-depends\*.tar.gz c:\pillow\winbuild\depends\ - xcopy /s c:\pillow-depends\test_images\* c:\pillow\tests\images +- 7z x ..\pillow-depends\nasm-2.14.02-win64.zip -oc:\ +- path c:\nasm-2.14.02;%PATH% - cd c:\pillow\winbuild\ - ps: | if ($env:PYTHON -eq "c:/vp/pypy3") @@ -67,7 +70,7 @@ build_script: } else { - & $env:PYTHON/$env:EXECUTABLE c:\pillow\winbuild\build\build_pillow.cmd install + c:\pillow\winbuild\build\build_pillow.cmd install $host.SetShouldExit(0) } - cd c:\pillow @@ -100,7 +103,7 @@ before_deploy: - cd c:\pillow - '%PYTHON%\%PIP_DIR%\pip.exe install wheel' - cd c:\pillow\winbuild\ - - '%PYTHON%\%EXECUTABLE% c:\pillow\winbuild\build\build_pillow.cmd bdist_wheel' + - c:\pillow\winbuild\build\build_pillow.cmd bdist_wheel - cd c:\pillow - ps: Get-ChildItem .\dist\*.* | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index 0d513a47d..c8c247dc2 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -38,8 +38,7 @@ def test_viewer_show(): @pytest.mark.skipif( - not on_ci() or (is_win32() and on_github_actions()), - reason="Only run on CIs; hangs on Windows on GitHub Actions", + not on_ci() or is_win32(), reason="Only run on CIs; hangs on Windows 10", ) def test_show(): for mode in ("1", "I;16", "LA", "RGB", "RGBA"): From c6a1c551d9840a79b473fe255d810bca7263addb Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 1 Jan 2020 13:02:27 +0100 Subject: [PATCH 14/27] cleanup build configuration --- .appveyor.yml | 15 ++-- .github/workflows/test-windows.yml | 4 +- Tests/test_imageshow.py | 2 +- winbuild/README.md | 48 +++++++----- winbuild/build_prepare.py | 115 +++++++++++++---------------- winbuild/commands.py | 3 - 6 files changed, 87 insertions(+), 100 deletions(-) delete mode 100644 winbuild/commands.py diff --git a/.appveyor.yml b/.appveyor.yml index 645c2256f..4d2cb81d2 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,10 +7,8 @@ init: # Uncomment previous line to get RDP access during the build. environment: - X64_EXT: -x64 EXECUTABLE: python.exe PIP_DIR: Scripts - VENV: NO TEST_OPTIONS: DEPLOY: YES matrix: @@ -32,12 +30,11 @@ install: - curl -fsSL -o pillow-depends.zip https://github.com/python-pillow/pillow-depends/archive/master.zip - 7z x pillow-depends.zip -oc:\ - mv c:\pillow-depends-master c:\pillow-depends -- mkdir c:\pillow\winbuild\depends -- xcopy c:\pillow-depends\*.zip c:\pillow\winbuild\depends\ -- xcopy c:\pillow-depends\*.tar.gz c:\pillow\winbuild\depends\ - xcopy /s c:\pillow-depends\test_images\* c:\pillow\tests\images - 7z x ..\pillow-depends\nasm-2.14.02-win64.zip -oc:\ -- path c:\nasm-2.14.02;%PATH% +- curl -fsSL -o gs952.exe https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs952/gs952w32.exe +- gs952.exe /S +- path c:\nasm-2.14.02;C:\Program Files (x86)\gs\gs9.52\bin;%PATH% - cd c:\pillow\winbuild\ - ps: | if ($env:PYTHON -eq "c:/vp/pypy3") @@ -51,14 +48,12 @@ install: } else { - $env:PILLOW_BIN = "c:\pillow\" + $env:PILLOW_DEPS = "C:\pillow-depends\" c:\python37\python.exe c:\pillow\winbuild\build_prepare.py c:\pillow\winbuild\build\build_dep_all.cmd $host.SetShouldExit(0) } -- curl -fsSL -o gs952.exe https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs952/gs952w32.exe -- gs952.exe /S -- path %path%;C:\Program Files (x86)\gs\gs9.52\bin +- path C:\pillow\winbuild\build\bin;%PATH% build_script: - ps: | diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 7c1887745..fe958ad5a 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -66,9 +66,7 @@ jobs: ..\pillow-depends\gs950w32.exe /S Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" - mkdir $env:GITHUB_WORKSPACE\winbuild\depends\ - xcopy ..\pillow-depends\*.zip $env:GITHUB_WORKSPACE\winbuild\depends\ - xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\depends\ + $env:PILLOW_DEPS = "$env:RUNNER_WORKSPACE\pillow-depends\" xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\ $env:PYTHON=$env:pythonLocation diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index c8c247dc2..a9332bef4 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -1,7 +1,7 @@ import pytest from PIL import Image, ImageShow -from .helper import hopper, is_win32, on_ci, on_github_actions +from .helper import hopper, is_win32, on_ci def test_sanity(): diff --git a/winbuild/README.md b/winbuild/README.md index 471b61a57..dabbc359a 100644 --- a/winbuild/README.md +++ b/winbuild/README.md @@ -1,18 +1,30 @@ -Quick README ------------- - -For more extensive info, see the [Windows build instructions](build.rst). - -* See https://github.com/python-pillow/Pillow/issues/553#issuecomment-37877416 and https://github.com/matplotlib/matplotlib/issues/1717#issuecomment-13343859 - -* Works best with Python 3.4, due to virtualenv and pip batteries included. Python3+ required for fetch command. -* Check config.py for virtual env paths, suffix for 64-bit releases. Defaults to `x64`, set `X64_EXT` to change. -* When running in CI with one Python per invocation, set the `PYTHON` env variable to the Python folder. (e.g. `PYTHON`=`c:\Python27\`) This overrides the matrix in config.py and will just build and test for the specific Python. -* `python get_pythons.py` downloads all the Python releases, and their signatures. (Manually) Install in `c:\PythonXX[x64]\`. -* `python build_dep.py` downloads and creates a build script for all the dependencies, in 32 and 64-bit versions, and with both compiler versions. -* (in powershell) `build_deps.cmd` invokes the dependency build. -* `python build.py --clean` makes Pillow for the matrix of Pythons. -* `python test.py` runs the tests on Pillow in all the virtual envs. -* Currently working with zlib, libjpeg, freetype, and libtiff on Python 2.7, and 3.4, both 32 and 64-bit, on a local win7 pro machine and appveyor.com -* WebP is built, not detected. -* LCMS, OpenJPEG and libimagequant are not building. +Quick README +------------ + +For more extensive info, see the [Windows build instructions](build.rst). + +* See [Current Windows Build/Testing process (Pillow#553)](https://github.com/python-pillow/Pillow/issues/553#issuecomment-37877416), + [Definitive docs for how to compile on Windows (matplotlib#1717)](https://github.com/matplotlib/matplotlib/issues/1717#issuecomment-13343859), + [Test Windows with GitHub Actions (Pillow#4084)](https://github.com/python-pillow/Pillow/pull/4084). + + +* Requires Microsoft Visual Studio 2017 or newer with C++ component. +* Requires NASM for libjpeg-turbo, a required dependency when using this script. +* Requires CMake 3.13 or newer. +* Python 3.6+ is required to generate valid scripts, but builds targeting Python 3.5+ are supported. +* Tested on Windows Server 2016 with Visual Studio 2017 Community (AppVeyor). +* Tested on Windows Server 2019 with Visual Studio 2019 Enterprise (GitHub Actions). + +The following is a simplified version of the script used on AppVeyor: +``` +set PYTHON=C:\Python35\bin +cd /D C:\Pillow\winbuild +C:\Python37\bin\python.exe build_prepare.py +build\build_dep_all.cmd +build\build_pillow.cmd install +cd .. +path C:\Pillow\winbuild\build\bin;%PATH% +%PYTHON%\python.exe selftest.py +%PYTHON%\python.exe -m pytest -vx --cov PIL --cov Tests --cov-report term --cov-report xml Tests +build\build_pillow.cmd bdist_wheel +``` diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index c772a580f..e56b71eb4 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -95,7 +95,6 @@ architectures = { } header = [ - cmd_set("BUILD", "{build_dir}"), cmd_set("INCLUDE", "{inc_dir}"), cmd_set("INCLIB", "{lib_dir}"), cmd_set("LIB", "{lib_dir}"), @@ -145,7 +144,7 @@ deps = { "filename": "tiff-4.1.0.tar.gz", "dir": "tiff-4.1.0", "build": [ - cmd_copy(r"{script_dir}\tiff.opt", "nmake.opt"), + cmd_copy(r"{winbuild_dir}\tiff.opt", "nmake.opt"), cmd_nmake("makefile.vc", "clean"), cmd_nmake("makefile.vc", "lib"), ], @@ -176,8 +175,7 @@ deps = { "patch": { r"builds\windows\vc2010\freetype.vcxproj": { # freetype setting is /MD for .dll and /MT for .lib, we need /MD - "MultiThreaded": - "MultiThreadedDLL", + "MultiThreaded": "MultiThreadedDLL", # noqa E501 } }, "build": [ @@ -200,14 +198,11 @@ deps = { "patch": { r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always - "MultiThreaded": - "MultiThreadedDLL", + "MultiThreaded": "MultiThreadedDLL", # noqa E501 # retarget to default toolset (selected by vcvarsall.bat) - "v141": - "$(DefaultPlatformToolset)", + "v141": "$(DefaultPlatformToolset)", # noqa E501 # retarget to latest (selected by vcvarsall.bat) - "8.1": - "$(WindowsSDKVersion)", # noqa E501 + "8.1": "$(WindowsSDKVersion)", # noqa E501 } }, "build": [ @@ -269,7 +264,7 @@ deps = { "filename": "fribidi-1.0.7.zip", "dir": "fribidi-1.0.7", "build": [ - cmd_copy(r"{script_dir}\fribidi.cmake", r"CMakeLists.txt"), + cmd_copy(r"{winbuild_dir}\fribidi.cmake", r"CMakeLists.txt"), cmd_cmake(), cmd_nmake(target="clean"), cmd_nmake(target="fribidi"), @@ -282,7 +277,7 @@ deps = { "filename": "libraqm-0.7.0.zip", "dir": "libraqm-0.7.0", "build": [ - cmd_copy(r"{script_dir}\raqm.cmake", r"CMakeLists.txt"), + cmd_copy(r"{winbuild_dir}\raqm.cmake", r"CMakeLists.txt"), cmd_cmake(), cmd_nmake(target="clean"), cmd_nmake(target="libraqm"), @@ -357,12 +352,6 @@ def find_msvs(): return vs -def match(values, target): - for key, value in values.items(): - if key in target: - return {"name": key, **value} - - def extract_dep(url, filename): import urllib.request import tarfile @@ -455,51 +444,42 @@ def build_dep_all(): write_script("build_dep_all.cmd", lines) -def build_pillow(wheel=False): - lines = [] - if path_dir is not None and not wheel: - lines.append(cmd_xcopy("{bin_dir}", path_dir)) - lines.extend(prefs["header"]) - lines.extend( - [ - "@echo ---- Building Pillow (build_ext %*) ----", - cmd_cd("{pillow_dir}"), - cmd_append("LIB", r"{python_dir}\tcl"), - cmd_set("MSSdk", "1"), - cmd_set("DISTUTILS_USE_SDK", "1"), - cmd_set("py_vcruntime_redist", "true"), - r'"{python_dir}\python.exe" setup.py build_ext %*', - ] - ) +def build_pillow(): + lines = [ + "@echo ---- Building Pillow (build_ext %*) ----", + cmd_cd("{pillow_dir}"), + *prefs["header"], + cmd_set("DISTUTILS_USE_SDK", "1"), # use same compiler to build Pillow + cmd_set("MSSdk", "1"), # for Python 3.5 and PyPy3.6 + cmd_set("py_vcruntime_redist", "true"), # use /MD, not /MT + r'"{python_dir}\{python_exe}" setup.py build_ext %*', + ] write_script("build_pillow.cmd", lines) if __name__ == "__main__": # winbuild directory - script_dir = os.path.dirname(os.path.realpath(__file__)) + winbuild_dir = os.path.dirname(os.path.realpath(__file__)) # dependency cache directory - depends_dir = os.path.join(script_dir, "depends") + depends_dir = os.environ.get("PILLOW_DEPS", os.path.join(winbuild_dir, "depends")) + os.makedirs(depends_dir, exist_ok=True) print("Caching dependencies in:", depends_dir) - # python bin directory - python_dir = os.environ.get( - "PYTHON", os.path.dirname(os.path.realpath(sys.executable)) + # Python bin directory + python_dir = os.environ.get("PYTHON") + python_exe = os.environ.get("EXECUTABLE", "python.exe") + if python_dir is None: + python_dir = os.path.dirname(os.path.realpath(sys.executable)) + python_exe = os.path.basename(sys.executable) + print("Target Python:", os.path.join(python_dir, python_exe)) + + # use ARCHITECTURE or PYTHON to select architecture + architecture = os.environ.get( + "ARCHITECTURE", "x64" if "x64" in python_dir else "x86" ) - print("Target Python:", python_dir) - - # copy binaries to this directory - path_dir = os.environ.get("PILLOW_BIN") - print("Copying binary files to:", path_dir) - - # use PYTHON to select architecture - arch_prefs = match(architectures, python_dir) - if arch_prefs is None: - architecture = "x86" - arch_prefs = architectures[architecture] - else: - architecture = arch_prefs["name"] + arch_prefs = architectures[architecture] print("Target Architecture:", architecture) msvs = find_msvs() @@ -510,7 +490,7 @@ if __name__ == "__main__": print("Found Visual Studio at:", msvs["vs_dir"]) # build root directory - build_dir = os.environ.get("PILLOW_BUILD", os.path.join(script_dir, "build")) + build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")) print("Using output directory:", build_dir) # build directory for *.h files @@ -523,25 +503,30 @@ if __name__ == "__main__": bin_dir = os.path.join(build_dir, "bin") shutil.rmtree(build_dir, ignore_errors=True) - for path in [depends_dir, build_dir, lib_dir, inc_dir, bin_dir]: - os.makedirs(path, exist_ok=True) + for path in [build_dir, inc_dir, lib_dir, bin_dir]: + os.makedirs(path) prefs = { - "architecture": architecture, - "script_dir": script_dir, - "depends_dir": depends_dir, + # Python paths / preferences "python_dir": python_dir, + "python_exe": python_exe, + "architecture": architecture, + **arch_prefs, + # Pillow paths + "pillow_dir": os.path.realpath(os.path.join(winbuild_dir, "..")), + "winbuild_dir": winbuild_dir, + # Build paths "build_dir": build_dir, - "lib_dir": lib_dir, "inc_dir": inc_dir, + "lib_dir": lib_dir, "bin_dir": bin_dir, - "pillow_dir": os.path.realpath(os.path.join(script_dir, "..")), - # TODO auto find: - "cmake": "cmake.exe", + # Compilers / Tools + **msvs, + "cmake": "cmake.exe", # TODO find CMAKE automatically + # TODO find NASM automatically + # script header + "header": sum([header, msvs["header"], ["@echo on"]], []), } - prefs.update(msvs) - prefs.update(arch_prefs) - prefs["header"] = sum([header, msvs["header"], ["@echo on"]], []) build_dep_all() build_pillow() diff --git a/winbuild/commands.py b/winbuild/commands.py deleted file mode 100644 index ea6e73b4e..000000000 --- a/winbuild/commands.py +++ /dev/null @@ -1,3 +0,0 @@ -# builtins - - From d51380cccb779af1713f87968835a43d0d2825be Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 1 Jan 2020 21:08:51 +0100 Subject: [PATCH 15/27] docs, remove old files --- winbuild/build.py | 205 ------------------------ winbuild/build.rst | 126 ++++++++------- winbuild/build_dep.py | 328 --------------------------------------- winbuild/config.py | 199 ------------------------ winbuild/fetch.py | 44 ------ winbuild/get_pythons.py | 15 -- winbuild/lcms2_patch.ps1 | 9 -- winbuild/test.py | 45 ------ winbuild/untar.py | 11 -- winbuild/unzip.py | 11 -- 10 files changed, 68 insertions(+), 925 deletions(-) delete mode 100755 winbuild/build.py delete mode 100644 winbuild/build_dep.py delete mode 100644 winbuild/config.py delete mode 100644 winbuild/fetch.py delete mode 100644 winbuild/get_pythons.py delete mode 100644 winbuild/lcms2_patch.ps1 delete mode 100755 winbuild/test.py delete mode 100644 winbuild/untar.py delete mode 100644 winbuild/unzip.py diff --git a/winbuild/build.py b/winbuild/build.py deleted file mode 100755 index e565226bd..000000000 --- a/winbuild/build.py +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env python3 - -import getopt -import os -import shutil -import subprocess -import sys - -from config import ( - VIRT_BASE, - X64_EXT, - bit_from_env, - compiler_from_env, - compilers, - pythons, - pyversion_from_env, -) - - -def setup_vms(): - ret = [] - for py in pythons: - for arch in ("", X64_EXT): - ret.append( - "virtualenv -p c:/Python%s%s/python.exe --clear %s%s%s" - % (py, arch, VIRT_BASE, py, arch) - ) - ret.append( - r"%s%s%s\Scripts\pip.exe install pytest pytest-cov" - % (VIRT_BASE, py, arch) - ) - return "\n".join(ret) - - -def run_script(params): - (version, script) = params - try: - print("Running %s" % version) - filename = "build_pillow_%s.cmd" % version - with open(filename, "w") as f: - f.write(script) - - command = ["powershell", "./%s" % filename] - proc = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - (trace, stderr) = proc.communicate() - status = proc.returncode - print("-- stderr --") - print(stderr.decode()) - print("-- stdout --") - print(trace.decode()) - print("Done with {}: {}".format(version, status)) - return (version, status, trace, stderr) - except Exception as msg: - print("Error with {}: {}".format(version, str(msg))) - return (version, -1, "", str(msg)) - - -def header(op): - return r""" -setlocal -set MPLSRC=%%~dp0\.. -set INCLIB=%%~dp0\depends -set BLDOPT=%s -cd /D %%MPLSRC%% -""" % ( - op - ) - - -def footer(): - return """endlocal -exit -""" - - -def vc_setup(compiler, bit): - script = "" - if compiler["vc_version"] == "2015": - arch = "x86" if bit == 32 else "x86_amd64" - script = ( - r""" -call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %s -echo on""" - % arch - ) - return script - - -def build_one(py_ver, compiler, bit): - # UNDONE virtual envs if we're not running on AppVeyor - args = {} - args.update(compiler) - if "PYTHON" in os.environ: - args["python_path"] = "%PYTHON%" - else: - args["python_path"] = "{}{}\\Scripts".format(VIRT_BASE, py_ver) - - args["executable"] = "python.exe" - if "EXECUTABLE" in os.environ: - args["executable"] = "%EXECUTABLE%" - - args["py_ver"] = py_ver - args["tcl_ver"] = "86" - - if compiler["vc_version"] == "2015": - args["imaging_libs"] = " build_ext --add-imaging-libs=msvcrt" - else: - args["imaging_libs"] = "" - - args["vc_setup"] = vc_setup(compiler, bit) - - script = r""" -setlocal EnableDelayedExpansion -call "%%ProgramFiles%%\Microsoft SDKs\Windows\%(env_version)s\Bin\SetEnv.Cmd" /Release %(env_flags)s -set DISTUTILS_USE_SDK=1 -set LIB=%%LIB%%;%%INCLIB%%\%(inc_dir)s -set INCLUDE=%%INCLUDE%%;%%INCLIB%%\%(inc_dir)s;%%INCLIB%%\tcl%(tcl_ver)s\include - -setlocal -set LIB=%%LIB%%;C:\Python%(py_ver)s\tcl%(vc_setup)s -call %(python_path)s\%(executable)s setup.py %(imaging_libs)s %%BLDOPT%% -call %(python_path)s\%(executable)s -c "from PIL import _webp;import os, shutil;shutil.copy(r'%%INCLIB%%\freetype.dll', os.path.dirname(_webp.__file__));" -endlocal - -endlocal -""" # noqa: E501 - return script % args - - -def clean(): - try: - shutil.rmtree("../build") - except Exception: - # could already be removed - pass - run_script(("virtualenvs", setup_vms())) - - -def main(op): - scripts = [] - - for py_version, py_info in pythons.items(): - py_compilers = compilers[py_info["compiler"]][py_info["vc"]] - scripts.append( - ( - py_version, - "\n".join( - [header(op), build_one(py_version, py_compilers[32], 32), footer()] - ), - ) - ) - - scripts.append( - ( - "{}{}".format(py_version, X64_EXT), - "\n".join( - [ - header(op), - build_one("%sx64" % py_version, py_compilers[64], 64), - footer(), - ] - ), - ) - ) - - results = map(run_script, scripts) - - for (version, status, trace, err) in results: - print("Compiled {}: {}".format(version, status and "ERR" or "OK")) - - -def run_one(op): - - compiler = compiler_from_env() - py_version = pyversion_from_env() - bit = bit_from_env() - - run_script( - ( - py_version, - "\n".join([header(op), build_one(py_version, compiler, bit), footer()]), - ) - ) - - -if __name__ == "__main__": - opts, args = getopt.getopt(sys.argv[1:], "", ["clean", "wheel"]) - opts = dict(opts) - - if "--clean" in opts: - clean() - - op = "install" - if "--wheel" in opts: - op = "bdist_wheel" - - if "PYTHON" in os.environ: - run_one(op) - else: - main(op) diff --git a/winbuild/build.rst b/winbuild/build.rst index 1d2084044..f47efb363 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -5,89 +5,99 @@ Building Pillow on Windows <../docs/installation.rst#windows-installation>`_ should be sufficient. -This page will describe a build setup to build Pillow against the -supported Python versions in 32 and 64-bit modes, using freely -available Microsoft compilers. This has been developed and tested -against 64-bit Windows 7 Professional and Windows Server 2012 -64-bit version on Amazon EC2. +This page describes the steps necessary to build Pillow using the same +scripts used on GitHub Actions and AppVeyor CIs. Prerequisites ------------- -Extra Build Helpers -^^^^^^^^^^^^^^^^^^^ -* Powershell (available by default on Windows Server) -* GitHub client (provides git+bash shell) - -Optional: -* GPG (for checking signatures) (UNDONE -- Python signature checking) - - -Pythons -^^^^^^^ - -The build routines expect Python to be installed at C:\PythonXX for -32-bit versions or C:\PythonXXx64 for the 64-bit versions. - -Download Python 3.4, install it, and add it to the path. This is the -Python that we will use to bootstrap the build process. (The download -routines are using 3 features, and installing 3.4 gives us pip and -virtualenv as well, reducing the number of packages that we need to -install.) - -Download the rest of the Pythons by opening a command window, changing -to the ``winbuild`` directory, and running ``python -get_pythons.py``. - -UNDONE -- gpg verify the signatures (note that we can download from -https) - -Run each installer and set the proper path to the installation. Don't -set any of them as the default Python, or add them to the path. +Python +^^^^^^ +While the scripts can target any version of Python supported by Pillow, +Python 3.6+ is required to generate valid build scripts. Compilers ^^^^^^^^^ Download and install: -* `Microsoft Windows SDK for Windows 7 and .NET Framework - 4 `_ +* `Microsoft Visual Studio 2017 or newer (with C++ component) + `_ -* `CMake-2.8.10.2-win32-x86.exe - `_ +* `CMake 3.13 or newer + `_ -The samples and the .NET SDK portions aren't required, just the -compilers and other tools. UNDONE -- check exact wording. +* `NASM `_ + +Any version of Visual Studio 2017 or newer should be supported, +including Visual Studio 2017 Community. + +Paths to CMake and NASM must be added to the ``PATH`` environment variable. + +Build configuration +------------------- + +The following environment variables, if set, will override the default +behaviour of ``build_prepare.py``: + +* ``PYTHON`` + ``EXECUTABLE`` point to the target version of Python. + If ``PYTHON`` is unset, the version of Python used to run + ``build_prepare.py`` will be used. If only ``PYTHON`` is set, + ``EXECUTABLE`` defaults to ``python.exe``. +* ``ARCHITECTURE`` is used to select a ``x86`` or ``x64`` build. By default, + ``x86`` is used, unless ``PYTHON`` contains ``x64``, in which case ``x64`` + is used. +* ``PILLOW_BUILD`` can be used to override the ``winbuild\build`` directory + path, used to store generated build scripts and compiled libraries. + **Warning:** This directory is wiped when ``build_prepare.py`` is run. +* ``PILLOW_DEPS`` points to the directory used to store downloaded + dependencies. By default ``winbuild\depends`` is used. Dependencies ------------ -The script 'build_dep.py' downloads and builds the dependencies. Open -a command window, change directory into ``winbuild`` and run ``python -build_dep.py``. +Dependencies will be automatically downloaded by ``build_prepare.py``. +By default, downloaded dependencies are stored in ``winbuild\depends``; +set the ``PILLOW_DEPS`` environment variable to override this location. -This will download libjpeg, libtiff, libz, and freetype. It will then -compile 32 and 64-bit versions of the libraries, with both versions of -the compilers. - -UNDONE -- lcms fails. -UNDONE -- webp, jpeg2k not recognized +To build all dependencies, run ``winbuild\build\build_dep_all.cmd``, +or run the individual scripts to build each dependency separately. Building Pillow --------------- -Once the dependencies are built, run ``python build.py --clean`` to -build and install Pillow in virtualenvs for each Python -build. ``build.py --wheel`` will build wheels instead of -installing into virtualenvs. - -UNDONE -- suppressed output, what about failures. +Once the dependencies are built, run +``winbuild\build\build_pillow.cmd install`` to build and install +Pillow for the selected version of Python. +``winbuild\build\build_pillow.cmd bdist_wheel`` will build wheels +instead of installing Pillow. Testing Pillow -------------- -Build and install Pillow, then run ``python test.py`` from the -``winbuild`` directory. +Some binary dependencies (e.g. ``libraqm.dll``) will be stored in the +``winbuild\build\bin`` directory; this directory should be added to ``PATH`` +before running tests. +Build and install Pillow, then run ``python -m pytest Tests`` +from the root Pillow directory. + +Example +------- + +The following is a simplified version of the script used on AppVeyor: + +.. code-block:: + + set PYTHON=C:\Python35\bin + cd /D C:\Pillow\winbuild + C:\Python37\bin\python.exe build_prepare.py + build\build_dep_all.cmd + build\build_pillow.cmd install + cd .. + path C:\Pillow\winbuild\build\bin;%PATH% + %PYTHON%\python.exe selftest.py + %PYTHON%\python.exe -m pytest -vx --cov PIL --cov Tests --cov-report term --cov-report xml Tests + build\build_pillow.cmd bdist_wheel diff --git a/winbuild/build_dep.py b/winbuild/build_dep.py deleted file mode 100644 index 778570139..000000000 --- a/winbuild/build_dep.py +++ /dev/null @@ -1,328 +0,0 @@ -import os - -from build import vc_setup -from config import all_compilers, bit_from_env, compiler_from_env, compilers, libs -from fetch import fetch -from untar import untar -from unzip import unzip - - -def _relpath(*args): - return os.path.join(os.getcwd(), *args) - - -build_dir = _relpath("build") -inc_dir = _relpath("depends") - - -def check_sig(filename, signame): - # UNDONE -- need gpg - return filename - - -def mkdirs(): - try: - os.mkdir(build_dir) - except OSError: - pass - try: - os.mkdir(inc_dir) - except OSError: - pass - for compiler in all_compilers(): - try: - os.mkdir(os.path.join(inc_dir, compiler["inc_dir"])) - except OSError: - pass - - -def extract(src, dest): - if ".zip" in src: - return unzip(src, dest) - if ".tar.gz" in src or ".tgz" in src: - return untar(src, dest) - - -def extract_libs(): - for name, lib in libs.items(): - filename = fetch(lib["url"]) - if name == "openjpeg": - for compiler in all_compilers(): - if not os.path.exists( - os.path.join(build_dir, lib["dir"] + compiler["inc_dir"]) - ): - extract(filename, build_dir) - os.rename( - os.path.join(build_dir, lib["dir"]), - os.path.join(build_dir, lib["dir"] + compiler["inc_dir"]), - ) - else: - extract(filename, build_dir) - - -def extract_openjpeg(compiler): - return ( - r""" -rem build openjpeg -setlocal -cd %%BUILD%% -mkdir %%INCLIB%%\openjpeg-2.0 -copy /Y /B openjpeg-2.0.0-win32-x86\include\openjpeg-2.0 %%INCLIB%%\openjpeg-2.0 -copy /Y /B openjpeg-2.0.0-win32-x86\bin\ %%INCLIB%% -copy /Y /B openjpeg-2.0.0-win32-x86\lib\ %%INCLIB%% -endlocal -""" - % compiler - ) - - -def cp_tk(ver_85, ver_86): - versions = {"ver_85": ver_85, "ver_86": ver_86} - return ( - r""" -mkdir %%INCLIB%%\tcl85\include\X11 -copy /Y /B %%BUILD%%\tcl%(ver_85)s\generic\*.h %%INCLIB%%\tcl85\include\ -copy /Y /B %%BUILD%%\tk%(ver_85)s\generic\*.h %%INCLIB%%\tcl85\include\ -copy /Y /B %%BUILD%%\tk%(ver_85)s\xlib\X11\* %%INCLIB%%\tcl85\include\X11\ - -mkdir %%INCLIB%%\tcl86\include\X11 -copy /Y /B %%BUILD%%\tcl%(ver_86)s\generic\*.h %%INCLIB%%\tcl86\include\ -copy /Y /B %%BUILD%%\tk%(ver_86)s\generic\*.h %%INCLIB%%\tcl86\include\ -copy /Y /B %%BUILD%%\tk%(ver_86)s\xlib\X11\* %%INCLIB%%\tcl86\include\X11\ -""" - % versions - ) - - -def header(): - return r"""setlocal -set MSBUILD=C:\Windows\Microsoft.NET\Framework64\v4.0.30319\MSBuild.exe -set CMAKE="cmake.exe" -set INCLIB=%~dp0\depends -set BUILD=%~dp0\build -""" + "\n".join( - r"set {}=%BUILD%\{}".format(k.upper(), v["dir"]) - for (k, v) in libs.items() - if v["dir"] - ) - - -def setup_compiler(compiler): - return ( - r"""setlocal EnableDelayedExpansion -call "%%ProgramFiles%%\Microsoft SDKs\Windows\%(env_version)s\Bin\SetEnv.Cmd" /Release %(env_flags)s -echo on -set INCLIB=%%INCLIB%%\%(inc_dir)s -""" # noqa: E501 - % compiler - ) - - -def end_compiler(): - return """ -endlocal -""" - - -def nmake_openjpeg(compiler, bit): - if compiler["env_version"] == "v7.0": - return "" - - atts = {"op_ver": "2.3.1"} - atts.update(compiler) - return ( - r""" -rem build openjpeg -setlocal -""" - + vc_setup(compiler, bit) - + r""" -cd /D %%OPENJPEG%%%(inc_dir)s - -%%CMAKE%% -DBUILD_THIRDPARTY:BOOL=OFF -DBUILD_SHARED_LIBS:BOOL=OFF -DCMAKE_BUILD_TYPE=Release -G "NMake Makefiles" . -nmake -nologo -f Makefile clean -nmake -nologo -f Makefile -copy /Y /B bin\* %%INCLIB%% -mkdir %%INCLIB%%\openjpeg-%(op_ver)s -copy /Y /B src\lib\openjp2\*.h %%INCLIB%%\openjpeg-%(op_ver)s -endlocal -""" # noqa: E501 - % atts - ) - - -def nmake_libs(compiler, bit): - # undone -- pre, makes, headers, libs - script = ( - r""" -rem Build libjpeg -setlocal -""" - + vc_setup(compiler, bit) - + r""" -cd /D %%JPEG%% -nmake -nologo -f makefile.vc setup-vc6 -nmake -nologo -f makefile.vc clean -nmake -nologo -f makefile.vc nodebug=1 libjpeg.lib -copy /Y /B *.dll %%INCLIB%% -copy /Y /B *.lib %%INCLIB%% -copy /Y /B j*.h %%INCLIB%% -endlocal - -rem Build zlib -setlocal -cd /D %%ZLIB%% -nmake -nologo -f win32\Makefile.msc clean -nmake -nologo -f win32\Makefile.msc zlib.lib -copy /Y /B *.dll %%INCLIB%% -copy /Y /B *.lib %%INCLIB%% -copy /Y /B zlib.lib %%INCLIB%%\z.lib -copy /Y /B zlib.h %%INCLIB%% -copy /Y /B zconf.h %%INCLIB%% -endlocal - -rem Build webp -setlocal -""" - + vc_setup(compiler, bit) - + r""" -cd /D %%WEBP%% -rd /S /Q %%WEBP%%\output\release-static -nmake -nologo -f Makefile.vc CFG=release-static RTLIBCFG=static OBJDIR=output all -copy /Y /B output\release-static\%(webp_platform)s\lib\* %%INCLIB%% -mkdir %%INCLIB%%\webp -copy /Y /B src\webp\*.h %%INCLIB%%\\webp -endlocal - -rem Build libtiff -setlocal -""" - + vc_setup(compiler, bit) - + r""" -rem do after building jpeg and zlib -copy %%~dp0\tiff.opt %%TIFF%%\nmake.opt - -cd /D %%TIFF%% -nmake -nologo -f makefile.vc clean -nmake -nologo -f makefile.vc lib -copy /Y /B libtiff\*.dll %%INCLIB%% -copy /Y /B libtiff\*.lib %%INCLIB%% -copy /Y /B libtiff\tiff*.h %%INCLIB%% -endlocal -""" - ) - return script % compiler - - -def msbuild_freetype(compiler, bit): - script = r""" -rem Build freetype -setlocal -rd /S /Q %%FREETYPE%%\objs -set DefaultPlatformToolset=v100 -""" - properties = r"""/p:Configuration="Release" /p:Platform=%(platform)s""" - if bit == 64: - script += ( - r"copy /Y /B " - r'"C:\Program Files (x86)\Microsoft SDKs\Windows\v7.1A\Lib\x64\*.Lib" ' - r"%%FREETYPE%%\builds\windows\vc2010" - ) - properties += r" /p:_IsNativeEnvironment=false" - script += ( - r""" -%%MSBUILD%% %%FREETYPE%%\builds\windows\vc2010\freetype.sln /t:Clean;Build """ - + properties - + r""" /m -xcopy /Y /E /Q %%FREETYPE%%\include %%INCLIB%% -""" - ) - freetypeReleaseDir = r"%%FREETYPE%%\objs\%(platform)s\Release" - script += ( - r""" -copy /Y /B """ - + freetypeReleaseDir - + r"""\freetype.lib %%INCLIB%%\freetype.lib -copy /Y /B """ - + freetypeReleaseDir - + r"""\freetype.dll %%INCLIB%%\..\freetype.dll -endlocal -""" - ) - return script % compiler - - -def build_lcms2(compiler): - if compiler["env_version"] == "v7.1": - return build_lcms_71(compiler) - return build_lcms_70(compiler) - - -def build_lcms_70(compiler): - """Link error here on x64""" - if compiler["platform"] == "x64": - return "" - - """Build LCMS on VC2008. This version is only 32bit/Win32""" - return ( - r""" -rem Build lcms2 -setlocal -set LCMS=%%LCMS-2.7%% -rd /S /Q %%LCMS%%\Lib -rd /S /Q %%LCMS%%\Projects\VC%(vc_version)s\Release -%%MSBUILD%% %%LCMS%%\Projects\VC%(vc_version)s\lcms2.sln /t:Clean /p:Configuration="Release" /p:Platform=Win32 /m -%%MSBUILD%% %%LCMS%%\Projects\VC%(vc_version)s\lcms2.sln /t:lcms2_static /p:Configuration="Release" /p:Platform=Win32 /p:PlatformToolset=v90 /m -xcopy /Y /E /Q %%LCMS%%\include %%INCLIB%% -copy /Y /B %%LCMS%%\Lib\MS\*.lib %%INCLIB%% -endlocal -""" # noqa: E501 - % compiler - ) - - -def build_lcms_71(compiler): - return ( - r""" -rem Build lcms2 -setlocal -set LCMS=%%LCMS-2.8%% -rd /S /Q %%LCMS%%\Lib -rd /S /Q %%LCMS%%\Projects\VC%(vc_version)s\Release -powershell -Command "(gc Projects\VC2015\lcms2_static\lcms2_static.vcxproj) -replace 'MultiThreadedDLL', 'MultiThreaded' | Out-File -encoding ASCII Projects\VC2015\lcms2_static\lcms2_static.vcxproj" -%%MSBUILD%% %%LCMS%%\Projects\VC%(vc_version)s\lcms2.sln /t:Clean /p:Configuration="Release" /p:Platform=%(platform)s /m -%%MSBUILD%% %%LCMS%%\Projects\VC%(vc_version)s\lcms2.sln /t:lcms2_static /p:Configuration="Release" /p:Platform=%(platform)s /m -xcopy /Y /E /Q %%LCMS%%\include %%INCLIB%% -copy /Y /B %%LCMS%%\Lib\MS\*.lib %%INCLIB%% -endlocal -""" # noqa: E501 - % compiler - ) - - -def add_compiler(compiler, bit): - script.append(setup_compiler(compiler)) - script.append(nmake_libs(compiler, bit)) - - # script.append(extract_openjpeg(compiler)) - - script.append(msbuild_freetype(compiler, bit)) - script.append(build_lcms2(compiler)) - script.append(nmake_openjpeg(compiler, bit)) - script.append(end_compiler()) - - -mkdirs() -extract_libs() -script = [header(), cp_tk(libs["tk-8.5"]["version"], libs["tk-8.6"]["version"])] - - -if "PYTHON" in os.environ: - add_compiler(compiler_from_env(), bit_from_env()) -else: - # for compiler in all_compilers(): - # add_compiler(compiler) - add_compiler(compilers[7.0][2010][32], 32) - -with open("build_deps.cmd", "w") as f: - f.write("\n".join(script)) diff --git a/winbuild/config.py b/winbuild/config.py deleted file mode 100644 index 93413d1e5..000000000 --- a/winbuild/config.py +++ /dev/null @@ -1,199 +0,0 @@ -import os - -SF_MIRROR = "https://iweb.dl.sourceforge.net" - -pythons = { - "pypy3": {"compiler": 7.1, "vc": 2015}, - # for AppVeyor - "35": {"compiler": 7.1, "vc": 2015}, - "36": {"compiler": 7.1, "vc": 2015}, - "37": {"compiler": 7.1, "vc": 2015}, - "38": {"compiler": 7.1, "vc": 2015}, - # for GitHub Actions - "3.5": {"compiler": 7.1, "vc": 2015}, - "3.6": {"compiler": 7.1, "vc": 2015}, - "3.7": {"compiler": 7.1, "vc": 2015}, - "3.8": {"compiler": 7.1, "vc": 2015}, -} - -VIRT_BASE = "c:/vp/" -X64_EXT = os.environ.get("X64_EXT", "x64") - -libs = { - # 'openjpeg': { - # 'filename': 'openjpeg-2.0.0-win32-x86.zip', - # 'version': '2.0' - # }, - "zlib": { - "url": "http://zlib.net/zlib1211.zip", - "filename": "zlib1211.zip", - "dir": "zlib-1.2.11", - }, - "jpeg": { - "url": "http://www.ijg.org/files/jpegsr9d.zip", - "filename": "jpegsr9d.zip", - "dir": "jpeg-9d", - }, - "tiff": { - "url": "ftp://download.osgeo.org/libtiff/tiff-4.1.0.tar.gz", - "filename": "tiff-4.1.0.tar.gz", - "dir": "tiff-4.1.0", - }, - "freetype": { - "url": "https://download.savannah.gnu.org/releases/freetype/freetype-2.10.1.tar.gz", # noqa: E501 - "filename": "freetype-2.10.1.tar.gz", - "dir": "freetype-2.10.1", - }, - "lcms-2.7": { - "url": SF_MIRROR + "/project/lcms/lcms/2.7/lcms2-2.7.zip", - "filename": "lcms2-2.7.zip", - "dir": "lcms2-2.7", - }, - "lcms-2.8": { - "url": SF_MIRROR + "/project/lcms/lcms/2.8/lcms2-2.8.zip", - "filename": "lcms2-2.8.zip", - "dir": "lcms2-2.8", - }, - "tcl-8.5": { - "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tcl8519-src.zip", - "filename": "tcl8519-src.zip", - "dir": "", - }, - "tk-8.5": { - "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tk8519-src.zip", - "filename": "tk8519-src.zip", - "dir": "", - "version": "8.5.19", - }, - "tcl-8.6": { - "url": SF_MIRROR + "/project/tcl/Tcl/8.6.10/tcl8610-src.zip", - "filename": "tcl8610-src.zip", - "dir": "", - }, - "tk-8.6": { - "url": SF_MIRROR + "/project/tcl/Tcl/8.6.10/tk8610-src.zip", - "filename": "tk8610-src.zip", - "dir": "", - "version": "8.6.10", - }, - "webp": { - "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.1.0.tar.gz", - "filename": "libwebp-1.1.0.tar.gz", - "dir": "libwebp-1.1.0", - }, - "openjpeg": { - "url": "https://github.com/uclouvain/openjpeg/archive/v2.3.1.tar.gz", - "filename": "openjpeg-2.3.1.tar.gz", - "dir": "openjpeg-2.3.1", - }, - "jpeg-turbo": { - "url": SF_MIRROR + "/project/libjpeg-turbo/2.0.3/libjpeg-turbo-2.0.3.tar.gz", - "filename": "libjpeg-turbo-2.0.3.tar.gz", - "dir": "libjpeg-turbo-2.0.3", - }, - # e5d454b: Merge tag '2.12.6' into msvc - "imagequant": { - "url": "https://github.com/ImageOptim/libimagequant/archive/e5d454bc7f5eb63ee50c84a83a7fa5ac94f68ec4.zip", # noqa: E501 - "filename": "libimagequant-e5d454bc7f5eb63ee50c84a83a7fa5ac94f68ec4.zip", - "dir": "libimagequant-e5d454bc7f5eb63ee50c84a83a7fa5ac94f68ec4", - }, - "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/2.6.4.zip", - "filename": "harfbuzz-2.6.4.zip", - "dir": "harfbuzz-2.6.4", - }, - "fribidi": { - "url": "https://github.com/fribidi/fribidi/archive/v1.0.9.zip", - "filename": "fribidi-1.0.9.zip", - "dir": "fribidi-1.0.9", - }, - "libraqm": { - "url": "https://github.com/HOST-Oman/libraqm/archive/v0.7.0.zip", - "filename": "libraqm-0.7.0.zip", - "dir": "libraqm-0.7.0", - }, -} - -compilers = { - 7: { - 2010: { - 64: { - "env_version": "v7.0", - "vc_version": "2010", - "env_flags": "/x64 /xp", - "inc_dir": "msvcr90-x64", - "platform": "x64", - "webp_platform": "x64", - }, - 32: { - "env_version": "v7.0", - "vc_version": "2010", - "env_flags": "/x86 /xp", - "inc_dir": "msvcr90-x32", - "platform": "Win32", - "webp_platform": "x86", - }, - } - }, - 7.1: { - 2015: { - 64: { - "env_version": "v7.1", - "vc_version": "2015", - "env_flags": "/x64 /vista", - "inc_dir": "msvcr10-x64", - "platform": "x64", - "webp_platform": "x64", - }, - 32: { - "env_version": "v7.1", - "vc_version": "2015", - "env_flags": "/x86 /vista", - "inc_dir": "msvcr10-x32", - "platform": "Win32", - "webp_platform": "x86", - }, - } - }, -} - - -def pyversion_from_env(): - py = os.environ["PYTHON"] - - py_version = "35" - for k in pythons: - if k in py: - py_version = k - break - - if "64" in py: - py_version = "{}{}".format(py_version, X64_EXT) - - return py_version - - -def compiler_from_env(): - py = os.environ["PYTHON"] - - for k, v in pythons.items(): - if k in py: - py_info = v - break - - bit = bit_from_env() - return compilers[py_info["compiler"]][py_info["vc"]][bit] - - -def bit_from_env(): - py = os.environ["PYTHON"] - - return 64 if "64" in py else 32 - - -def all_compilers(): - all = [] - for vc_compilers in compilers.values(): - for bit_compilers in vc_compilers.values(): - all += bit_compilers.values() - return all diff --git a/winbuild/fetch.py b/winbuild/fetch.py deleted file mode 100644 index adc45429a..000000000 --- a/winbuild/fetch.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import sys -import urllib.parse -import urllib.request - -from config import libs - - -def fetch(url): - depends_filename = None - for lib in libs.values(): - if lib["url"] == url: - depends_filename = lib["filename"] - break - if depends_filename and os.path.exists(depends_filename): - return depends_filename - name = urllib.parse.urlsplit(url)[2].split("/")[-1] - - if not os.path.exists(name): - - def retrieve(request_url): - print("Fetching", request_url) - try: - return urllib.request.urlopen(request_url) - except urllib.error.URLError: - return urllib.request.urlopen(request_url) - - try: - r = retrieve(url) - except urllib.error.HTTPError: - if depends_filename: - r = retrieve( - "https://github.com/python-pillow/pillow-depends/raw/master/" - + depends_filename - ) - name = depends_filename - content = r.read() - with open(name, "wb") as fd: - fd.write(content) - return name - - -if __name__ == "__main__": - fetch(sys.argv[1]) diff --git a/winbuild/get_pythons.py b/winbuild/get_pythons.py deleted file mode 100644 index a853fc6f7..000000000 --- a/winbuild/get_pythons.py +++ /dev/null @@ -1,15 +0,0 @@ -import os - -from fetch import fetch - -if __name__ == "__main__": - for version in ["3.4.4"]: - for platform in ["", ".amd64"]: - for extension in ["", ".asc"]: - fetch( - "https://www.python.org/ftp/python/%s/python-%s%s.msi%s" - % (version, version, platform, extension) - ) - - # find pip, if it's not in the path! - os.system("pip install virtualenv") diff --git a/winbuild/lcms2_patch.ps1 b/winbuild/lcms2_patch.ps1 deleted file mode 100644 index 7fc48c034..000000000 --- a/winbuild/lcms2_patch.ps1 +++ /dev/null @@ -1,9 +0,0 @@ - -Get-ChildItem .\Projects\VC2015\ *.vcxproj -recurse | - Foreach-Object { - $c = ($_ | Get-Content) - $c = $c -replace 'MultiThreaded<','MultiThreadedDLL<' - $c = $c -replace '8.1','10' - $c = $c -replace 'v140','v142' - [IO.File]::WriteAllText($_.FullName, ($c -join "`r`n")) - } diff --git a/winbuild/test.py b/winbuild/test.py deleted file mode 100755 index a05a20b18..000000000 --- a/winbuild/test.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 - -import glob -import os -import subprocess -import sys - -from config import VIRT_BASE, X64_EXT, pythons - - -def test_one(params): - python, architecture = params - try: - print("Running: %s, %s" % params) - command = [ - r"{}\{}{}\Scripts\python.exe".format(VIRT_BASE, python, architecture), - "test-installed.py", - "--processes=-0", - "--process-timeout=30", - ] - command.extend(glob.glob("Tests/test*.py")) - proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE) - (trace, stderr) = proc.communicate() - status = proc.returncode - print("Done with {}, {} -- {}".format(python, architecture, status)) - return (python, architecture, status, trace) - except Exception as msg: - print("Error with {}, {}: {}".format(python, architecture, msg)) - return (python, architecture, -1, str(msg)) - - -if __name__ == "__main__": - - os.chdir("..") - matrix = [ - (python, architecture) for python in pythons for architecture in ("", X64_EXT) - ] - - results = map(test_one, matrix) - - for (python, architecture, status, trace) in results: - print("{}{}: {}".format(python, architecture, status and "ERR" or "PASS")) - - res = all(status for (python, architecture, status, trace) in results) - sys.exit(res) diff --git a/winbuild/untar.py b/winbuild/untar.py deleted file mode 100644 index f2713b2f2..000000000 --- a/winbuild/untar.py +++ /dev/null @@ -1,11 +0,0 @@ -import sys -import tarfile - - -def untar(src, dest): - with tarfile.open(src, "r:gz") as tgz: - tgz.extractall(dest) - - -if __name__ == "__main__": - untar(sys.argv[1], sys.argv[2]) diff --git a/winbuild/unzip.py b/winbuild/unzip.py deleted file mode 100644 index eb17a2e63..000000000 --- a/winbuild/unzip.py +++ /dev/null @@ -1,11 +0,0 @@ -import sys -import zipfile - - -def unzip(src, dest): - with zipfile.ZipFile(src) as zf: - zf.extractall(dest) - - -if __name__ == "__main__": - unzip(sys.argv[1], sys.argv[2]) From 98dd1b7e6ef2b0f967e7c90abc3db6b77a113c61 Mon Sep 17 00:00:00 2001 From: nulano Date: Mon, 24 Feb 2020 23:00:57 +0000 Subject: [PATCH 16/27] add parameters to build script --- .appveyor.yml | 3 +-- .github/workflows/test-windows.yml | 5 +---- setup.py | 2 +- winbuild/README.md | 2 +- winbuild/build.rst | 29 +++++++++++++++++------- winbuild/build_prepare.py | 36 +++++++++++++++++++++++++++--- 6 files changed, 58 insertions(+), 19 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 4d2cb81d2..8602293f7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -48,8 +48,7 @@ install: } else { - $env:PILLOW_DEPS = "C:\pillow-depends\" - c:\python37\python.exe c:\pillow\winbuild\build_prepare.py + c:\python37\python.exe c:\pillow\winbuild\build_prepare.py -v --depends=C:\pillow-depends\ c:\pillow\winbuild\build\build_dep_all.cmd $host.SetShouldExit(0) } diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index fe958ad5a..fe0fdc04c 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -66,12 +66,9 @@ jobs: ..\pillow-depends\gs950w32.exe /S Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" - $env:PILLOW_DEPS = "$env:RUNNER_WORKSPACE\pillow-depends\" xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\ - $env:PYTHON=$env:pythonLocation - cd $env:GITHUB_WORKSPACE/winbuild/ - & python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py + & python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py -v --depends=$env:RUNNER_WORKSPACE\pillow-depends\ --python=$env:pythonLocation shell: pwsh - name: Build dependencies / libjpeg-turbo diff --git a/setup.py b/setup.py index b55b1de62..1236b5df5 100755 --- a/setup.py +++ b/setup.py @@ -120,7 +120,7 @@ _LIB_IMAGING = ( "codec_fd", ) -DEBUG = True +DEBUG = False class DependencyException(Exception): diff --git a/winbuild/README.md b/winbuild/README.md index dabbc359a..45c37e4cb 100644 --- a/winbuild/README.md +++ b/winbuild/README.md @@ -19,7 +19,7 @@ The following is a simplified version of the script used on AppVeyor: ``` set PYTHON=C:\Python35\bin cd /D C:\Pillow\winbuild -C:\Python37\bin\python.exe build_prepare.py +C:\Python37\bin\python.exe build_prepare.py -v --depends=C:\pillow-depends build\build_dep_all.cmd build\build_pillow.cmd install cd .. diff --git a/winbuild/build.rst b/winbuild/build.rst index f47efb363..465bc2591 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -23,18 +23,20 @@ Compilers Download and install: -* `Microsoft Visual Studio 2017 or newer (with C++ component) - `_ +* `Microsoft Visual Studio 2017 or newer or Build Tools for Visual Studio 2017 or newer + `_ + (MSVC C++ build tools, and any Windows SDK version required) -* `CMake 3.13 or newer - `_ +* `CMake 3.13 or newer `_ + (also available as Visual Studio component C++ CMake tools for Windows) * `NASM `_ Any version of Visual Studio 2017 or newer should be supported, -including Visual Studio 2017 Community. +including Visual Studio 2017 Community, or Build Tools for Visual Studio 2019. -Paths to CMake and NASM must be added to the ``PATH`` environment variable. +Paths to CMake (if standalone) and NASM must be added to the ``PATH`` environment variable. +Visual Studio is found automatically with ``vswhere.exe``. Build configuration ------------------- @@ -47,7 +49,7 @@ behaviour of ``build_prepare.py``: ``build_prepare.py`` will be used. If only ``PYTHON`` is set, ``EXECUTABLE`` defaults to ``python.exe``. * ``ARCHITECTURE`` is used to select a ``x86`` or ``x64`` build. By default, - ``x86`` is used, unless ``PYTHON`` contains ``x64``, in which case ``x64`` + uses same architecture as the version of Python used to run ``build_prepare.py``. is used. * ``PILLOW_BUILD`` can be used to override the ``winbuild\build`` directory path, used to store generated build scripts and compiled libraries. @@ -55,6 +57,17 @@ behaviour of ``build_prepare.py``: * ``PILLOW_DEPS`` points to the directory used to store downloaded dependencies. By default ``winbuild\depends`` is used. +``build_prepare.py`` also supports the following command line parameters: + +* ``-v`` will print generated scripts. +* ``--no-imagequant`` will skip GPL-licensed ``libimagequant`` optional dependency +* ``--no-raqm`` will skip optional dependency Raqm (which itself depends on + LGPL-licensed ``fribidi``). +* ``--python=`` and ``--executable=`` override ``PYTHON`` and ``EXECUTABLE``. +* ``--architecture=`` overrides ``ARCHITECTURE``. +* ``--dir=`` and ``--depends=`` override ``PILLOW_BUILD`` + and ``PILLOW_DEPS``. + Dependencies ------------ @@ -93,7 +106,7 @@ The following is a simplified version of the script used on AppVeyor: set PYTHON=C:\Python35\bin cd /D C:\Pillow\winbuild - C:\Python37\bin\python.exe build_prepare.py + C:\Python37\bin\python.exe build_prepare.py -v --depends=C:\pillow-depends build\build_dep_all.cmd build\build_pillow.cmd install cd .. diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index e56b71eb4..475982da2 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -1,5 +1,6 @@ import os import shutil +import struct import subprocess import sys @@ -176,6 +177,8 @@ deps = { r"builds\windows\vc2010\freetype.vcxproj": { # freetype setting is /MD for .dll and /MT for .lib, we need /MD "MultiThreaded": "MultiThreadedDLL", # noqa E501 + # freetype doesn't specify SDK version, MSBuild may guess incorrectly + '': '\n $(WindowsSDKVersion)', # noqa E501 } }, "build": [ @@ -389,8 +392,9 @@ def write_script(name, lines): print("Writing " + name) with open(name, "w") as f: f.write("\n\r".join(lines)) - for line in lines: - print(" " + line) + if verbose: + for line in lines: + print(" " + line) def get_footer(dep): @@ -438,6 +442,8 @@ def build_dep(name): def build_dep_all(): lines = ["@echo on"] for dep_name in deps: + if dep_name in disabled: + continue lines.append(r'cmd.exe /c "{{build_dir}}\{}"'.format(build_dep(dep_name))) lines.append("if errorlevel 1 echo Build failed! && exit /B 1") lines.append("@echo All Pillow dependencies built successfully!") @@ -459,6 +465,28 @@ def build_pillow(): if __name__ == "__main__": + verbose = False + disabled = [] + for arg in sys.argv[1:]: + if arg == "-v": + verbose = True + elif arg == "--no-imagequant": + disabled += ["libimagequant"] + elif arg == "--no-raqm": + disabled += ["harfbuzz", "fribidi", "libraqm"] + elif arg.startswith("--depends="): + os.environ["PILLOW_DEPS"] = arg[10:] + elif arg.startswith("--python="): + os.environ["PYTHON"] = arg[9:] + elif arg.startswith("--executable="): + os.environ["EXECUTABLE"] = arg[13:] + elif arg.startswith("--architecture="): + os.environ["ARCHITECTURE"] = arg[15:] + elif arg.startswith("--dir="): + os.environ["PILLOW_BUILD"] = arg[6:] + else: + raise ValueError("Unknown parameter: " + arg) + # winbuild directory winbuild_dir = os.path.dirname(os.path.realpath(__file__)) @@ -477,7 +505,7 @@ if __name__ == "__main__": # use ARCHITECTURE or PYTHON to select architecture architecture = os.environ.get( - "ARCHITECTURE", "x64" if "x64" in python_dir else "x86" + "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64" ) arch_prefs = architectures[architecture] print("Target Architecture:", architecture) @@ -528,5 +556,7 @@ if __name__ == "__main__": "header": sum([header, msvs["header"], ["@echo on"]], []), } + print() + build_dep_all() build_pillow() From 0e5a6e6eb1a61392f3bcc29340c717b495081cda Mon Sep 17 00:00:00 2001 From: nulano Date: Fri, 27 Mar 2020 22:07:59 +0100 Subject: [PATCH 17/27] simplify default arguments in build_prepare --- winbuild/build_prepare.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 475982da2..36730ca0b 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -465,8 +465,18 @@ def build_pillow(): if __name__ == "__main__": + # winbuild directory + winbuild_dir = os.path.dirname(os.path.realpath(__file__)) + verbose = False disabled = [] + depends_dir = os.environ.get("PILLOW_DEPS", os.path.join(winbuild_dir, "depends")) + python_dir = os.environ.get("PYTHON") + python_exe = os.environ.get("EXECUTABLE", "python.exe") + architecture = os.environ.get( + "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64" + ) + build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")) for arg in sys.argv[1:]: if arg == "-v": verbose = True @@ -475,38 +485,27 @@ if __name__ == "__main__": elif arg == "--no-raqm": disabled += ["harfbuzz", "fribidi", "libraqm"] elif arg.startswith("--depends="): - os.environ["PILLOW_DEPS"] = arg[10:] + depends_dir = arg[10:] elif arg.startswith("--python="): - os.environ["PYTHON"] = arg[9:] + python_dir = arg[9:] elif arg.startswith("--executable="): - os.environ["EXECUTABLE"] = arg[13:] + python_exe = arg[13:] elif arg.startswith("--architecture="): - os.environ["ARCHITECTURE"] = arg[15:] + architecture = arg[15:] elif arg.startswith("--dir="): - os.environ["PILLOW_BUILD"] = arg[6:] + build_dir = arg[6:] else: raise ValueError("Unknown parameter: " + arg) - # winbuild directory - winbuild_dir = os.path.dirname(os.path.realpath(__file__)) - # dependency cache directory - depends_dir = os.environ.get("PILLOW_DEPS", os.path.join(winbuild_dir, "depends")) os.makedirs(depends_dir, exist_ok=True) print("Caching dependencies in:", depends_dir) - # Python bin directory - python_dir = os.environ.get("PYTHON") - python_exe = os.environ.get("EXECUTABLE", "python.exe") if python_dir is None: python_dir = os.path.dirname(os.path.realpath(sys.executable)) python_exe = os.path.basename(sys.executable) print("Target Python:", os.path.join(python_dir, python_exe)) - # use ARCHITECTURE or PYTHON to select architecture - architecture = os.environ.get( - "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64" - ) arch_prefs = architectures[architecture] print("Target Architecture:", architecture) @@ -517,16 +516,12 @@ if __name__ == "__main__": ) print("Found Visual Studio at:", msvs["vs_dir"]) - # build root directory - build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")) print("Using output directory:", build_dir) # build directory for *.h files inc_dir = os.path.join(build_dir, "inc") - # build directory for *.lib files lib_dir = os.path.join(build_dir, "lib") - # build directory for *.bin files bin_dir = os.path.join(build_dir, "bin") From 9aa42c3fd6a12a4069e38bd4744e1f4a3fc46b37 Mon Sep 17 00:00:00 2001 From: nulano Date: Fri, 27 Mar 2020 22:22:51 +0100 Subject: [PATCH 18/27] fix AppVeyor --- .appveyor.yml | 5 +++++ Tests/test_imageshow.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index 8602293f7..a06ba9902 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,11 +13,16 @@ environment: DEPLOY: YES matrix: - PYTHON: C:/Python38 + ARCHITECTURE: x86 - PYTHON: C:/Python38-x64 + ARCHITECTURE: x64 - PYTHON: C:/Python35 + ARCHITECTURE: x86 - PYTHON: C:/Python35-x64 + ARCHITECTURE: x64 - PYTHON: C:/msys64/mingw32 EXECUTABLE: bin/python3 + ARCHITECTURE: x86 PIP_DIR: bin TEST_OPTIONS: --processes=0 DEPLOY: NO diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index a9332bef4..64f15326b 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -38,7 +38,7 @@ def test_viewer_show(): @pytest.mark.skipif( - not on_ci() or is_win32(), reason="Only run on CIs; hangs on Windows 10", + not on_ci() or is_win32(), reason="Only run on CIs; hangs on Windows CIs", ) def test_show(): for mode in ("1", "I;16", "LA", "RGB", "RGBA"): From 3dedaec32278715db477bd0e2a540c693d57c7fb Mon Sep 17 00:00:00 2001 From: nulano Date: Sun, 29 Mar 2020 16:52:02 +0200 Subject: [PATCH 19/27] allow older cmake to support Visual Studio 2017 --- winbuild/README.md | 2 +- winbuild/build.rst | 2 +- winbuild/fribidi.cmake | 2 +- winbuild/raqm.cmake | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/winbuild/README.md b/winbuild/README.md index 45c37e4cb..d46361c9e 100644 --- a/winbuild/README.md +++ b/winbuild/README.md @@ -10,7 +10,7 @@ For more extensive info, see the [Windows build instructions](build.rst). * Requires Microsoft Visual Studio 2017 or newer with C++ component. * Requires NASM for libjpeg-turbo, a required dependency when using this script. -* Requires CMake 3.13 or newer. +* Requires CMake 3.12 or newer (available as Visual Studio component). * Python 3.6+ is required to generate valid scripts, but builds targeting Python 3.5+ are supported. * Tested on Windows Server 2016 with Visual Studio 2017 Community (AppVeyor). * Tested on Windows Server 2019 with Visual Studio 2019 Enterprise (GitHub Actions). diff --git a/winbuild/build.rst b/winbuild/build.rst index 465bc2591..517843a66 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -27,7 +27,7 @@ Download and install: `_ (MSVC C++ build tools, and any Windows SDK version required) -* `CMake 3.13 or newer `_ +* `CMake 3.12 or newer `_ (also available as Visual Studio component C++ CMake tools for Windows) * `NASM `_ diff --git a/winbuild/fribidi.cmake b/winbuild/fribidi.cmake index 247e79e4c..112174731 100644 --- a/winbuild/fribidi.cmake +++ b/winbuild/fribidi.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.12) project(fribidi) diff --git a/winbuild/raqm.cmake b/winbuild/raqm.cmake index 88eb7f284..e8e71800e 100644 --- a/winbuild/raqm.cmake +++ b/winbuild/raqm.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13) +cmake_minimum_required(VERSION 3.12) project(libraqm) From 398b8a64d67b35f026587ea85ff6a967d945f2d6 Mon Sep 17 00:00:00 2001 From: nulano Date: Sun, 29 Mar 2020 20:18:39 +0200 Subject: [PATCH 20/27] use v2 actions --- .github/workflows/test-windows.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index fe0fdc04c..a5761a0e0 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -27,13 +27,13 @@ jobs: steps: - name: Checkout Pillow - uses: actions/checkout@v1 + uses: actions/checkout@v2 - name: Checkout cached dependencies - uses: actions/checkout@v1 + uses: actions/checkout@v2 with: repository: python-pillow/pillow-depends - ref: master + path: winbuild\depends - name: Cache pip uses: actions/cache@v1 @@ -60,15 +60,15 @@ jobs: - name: Prepare dependencies run: | - 7z x ..\pillow-depends\nasm-2.14.02-win64.zip "-o$env:RUNNER_WORKSPACE\" + 7z x winbuild\depends\nasm-2.14.02-win64.zip "-o$env:RUNNER_WORKSPACE\" Write-Host "::add-path::$env:RUNNER_WORKSPACE\nasm-2.14.02" - ..\pillow-depends\gs950w32.exe /S + winbuild\depends\gs950w32.exe /S Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" - xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\ + xcopy /s winbuild\depends\test_images\* Tests\images\ - & python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py -v --depends=$env:RUNNER_WORKSPACE\pillow-depends\ --python=$env:pythonLocation + & python.exe winbuild\build_prepare.py -v --python=$env:pythonLocation shell: pwsh - name: Build dependencies / libjpeg-turbo @@ -149,8 +149,8 @@ jobs: winbuild\\build\\build_pillow.cmd bdist_wheel" shell: cmd - - uses: actions/upload-artifact@v1 + - uses: actions/upload-artifact@v2-preview if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" with: name: ${{ steps.wheel.outputs.dist }} - path: dist + path: dist\*.whl From 999823130f29daefb3ee081a2c73b474af0abafd Mon Sep 17 00:00:00 2001 From: nulano Date: Sun, 29 Mar 2020 22:27:24 +0200 Subject: [PATCH 21/27] use v2 checkout in all workflows --- .github/workflows/lint.yml | 2 +- .github/workflows/test-docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3cad8d417..6c5d81ac4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,7 +13,7 @@ jobs: name: Python ${{ matrix.python-version }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: pip cache uses: actions/cache@v1 diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index b6e703fd9..a2cb70b20 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -29,7 +29,7 @@ jobs: name: ${{ matrix.docker }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Build system information run: python .github/workflows/system-info.py From 79181c20a544c31e2e778e9cbacfae2f9ac3ba15 Mon Sep 17 00:00:00 2001 From: nulano Date: Mon, 30 Mar 2020 11:02:44 +0200 Subject: [PATCH 22/27] remove more AppVeyor jobs --- .appveyor.yml | 12 ------------ docs/installation.rst | 14 ++++++++------ winbuild/appveyor_install_pypy3.cmd | 3 --- 3 files changed, 8 insertions(+), 21 deletions(-) delete mode 100644 winbuild/appveyor_install_pypy3.cmd diff --git a/.appveyor.yml b/.appveyor.yml index a06ba9902..c0a8a0237 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -14,10 +14,6 @@ environment: matrix: - PYTHON: C:/Python38 ARCHITECTURE: x86 - - PYTHON: C:/Python38-x64 - ARCHITECTURE: x64 - - PYTHON: C:/Python35 - ARCHITECTURE: x86 - PYTHON: C:/Python35-x64 ARCHITECTURE: x64 - PYTHON: C:/msys64/mingw32 @@ -26,9 +22,6 @@ environment: PIP_DIR: bin TEST_OPTIONS: --processes=0 DEPLOY: NO - - PYTHON: C:/vp/pypy3 - EXECUTABLE: bin/pypy.exe - VENV: YES install: @@ -41,11 +34,6 @@ install: - gs952.exe /S - path c:\nasm-2.14.02;C:\Program Files (x86)\gs\gs9.52\bin;%PATH% - cd c:\pillow\winbuild\ -- ps: | - if ($env:PYTHON -eq "c:/vp/pypy3") - { - c:\pillow\winbuild\appveyor_install_pypy3.cmd - } - ps: | if ($env:PYTHON -eq "c:/msys64/mingw32") { diff --git a/docs/installation.rst b/docs/installation.rst index a83bf5748..307a05ad3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -292,9 +292,7 @@ or from within the uncompressed source directory:: Building on Windows ^^^^^^^^^^^^^^^^^^^ -We don't recommend trying to build on Windows. It is a maze of twisty -passages, mostly dead ends. There are build scripts and notes for the -Windows build in the ``winbuild`` directory. +There are build scripts and notes for the Windows build in the ``winbuild`` directory. Building on FreeBSD ^^^^^^^^^^^^^^^^^^^ @@ -404,9 +402,11 @@ These platforms are built and tested for every change. +----------------------------------+--------------------------+-----------------------+ | Ubuntu Linux 16.04 LTS | 3.5, 3.6, 3.7, 3.8, PyPy3|x86-64 | +----------------------------------+--------------------------+-----------------------+ -| Windows Server 2012 R2 | 3.5, 3.8 |x86, x86-64 | +| Windows Server 2016 | 3.8 |x86 | | +--------------------------+-----------------------+ -| | PyPy3, 3.7/MinGW |x86 | +| | 3.5 |x86-64 | +| +--------------------------+-----------------------+ +| | 3.7/MinGW |x86 | +----------------------------------+--------------------------+-----------------------+ | Windows Server 2019 | 3.5, 3.6, 3.7, 3.8 |x86, x86-64 | | +--------------------------+-----------------------+ @@ -474,11 +474,13 @@ These platforms have been reported to work at the versions mentioned. +----------------------------------+------------------------------+--------------------------------+-----------------------+ | FreeBSD 10.2 | 2.7, 3.4 | 3.1.0 |x86-64 | +----------------------------------+------------------------------+--------------------------------+-----------------------+ +| Windows 10 | 3.7 | 7.1.0 |x86-64 | ++----------------------------------+------------------------------+--------------------------------+-----------------------+ | Windows 8.1 Pro | 2.6, 2.7, 3.2, 3.3, 3.4 | 2.4.0 |x86,x86-64 | +----------------------------------+------------------------------+--------------------------------+-----------------------+ | Windows 8 Pro | 2.6, 2.7, 3.2, 3.3, 3.4a3 | 2.2.0 |x86,x86-64 | +----------------------------------+------------------------------+--------------------------------+-----------------------+ -| Windows 7 Pro | 2.7, 3.2, 3.3 | 3.4.1 |x86-64 | +| Windows 7 Professional | 3.7 | 7.0.0 |x86,x86-64 | +----------------------------------+------------------------------+--------------------------------+-----------------------+ | Windows Server 2008 R2 Enterprise| 3.3 | |x86-64 | +----------------------------------+------------------------------+--------------------------------+-----------------------+ diff --git a/winbuild/appveyor_install_pypy3.cmd b/winbuild/appveyor_install_pypy3.cmd deleted file mode 100644 index 4ec5384be..000000000 --- a/winbuild/appveyor_install_pypy3.cmd +++ /dev/null @@ -1,3 +0,0 @@ -curl -fsSL -o pypy3.zip https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.1-win32.zip -7z x pypy3.zip -oc:\ -c:\Python37\Scripts\virtualenv.exe -p c:\pypy3.6-v7.3.1-win32\pypy3.exe c:\vp\pypy3 From 2df26d8ea64e83d60b17b72bca2c73007ea925dd Mon Sep 17 00:00:00 2001 From: nulano Date: Sun, 12 Apr 2020 10:18:49 +0200 Subject: [PATCH 23/27] restore library updates in master --- winbuild/build_prepare.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 36730ca0b..ad344ffa7 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -102,7 +102,7 @@ header = [ cmd_append("PATH", "{bin_dir}"), ] -# dependencies +# dependencies, listed in order of compilation deps = { "libjpeg": { "url": SF_MIRROR + "/project/libjpeg-turbo/2.0.3/libjpeg-turbo-2.0.3.tar.gz", @@ -154,9 +154,9 @@ deps = { # "bins": [r"libtiff\*.dll"], }, "libwebp": { - "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.0.3.tar.gz", # noqa: E501 - "filename": "libwebp-1.0.3.tar.gz", - "dir": "libwebp-1.0.3", + "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.1.0.tar.gz", # noqa: E501 + "filename": "libwebp-1.1.0.tar.gz", + "dir": "libwebp-1.1.0", "build": [ cmd_rmdir(r"output\release-static"), # clean cmd_nmake( @@ -176,9 +176,9 @@ deps = { "patch": { r"builds\windows\vc2010\freetype.vcxproj": { # freetype setting is /MD for .dll and /MT for .lib, we need /MD - "MultiThreaded": "MultiThreadedDLL", # noqa E501 + "MultiThreaded": "MultiThreadedDLL", # noqa: E501 # freetype doesn't specify SDK version, MSBuild may guess incorrectly - '': '\n $(WindowsSDKVersion)', # noqa E501 + '': '\n $(WindowsSDKVersion)', # noqa: E501 } }, "build": [ @@ -201,11 +201,11 @@ deps = { "patch": { r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always - "MultiThreaded": "MultiThreadedDLL", # noqa E501 + "MultiThreaded": "MultiThreadedDLL", # noqa: E501 # retarget to default toolset (selected by vcvarsall.bat) - "v141": "$(DefaultPlatformToolset)", # noqa E501 + "v141": "$(DefaultPlatformToolset)", # noqa: E501 # retarget to latest (selected by vcvarsall.bat) - "8.1": "$(WindowsSDKVersion)", # noqa E501 + "8.1": "$(WindowsSDKVersion)", # noqa: E501 } }, "build": [ @@ -231,10 +231,10 @@ deps = { "libs": [r"bin\*.lib"], }, "libimagequant": { - # ba653c8: Merge tag '2.12.5' into msvc - "url": "https://github.com/ImageOptim/libimagequant/archive/ba653c8ccb34dde4e21c6076d85a72d21ed9d971.zip", # noqa: E501 - "filename": "libimagequant-ba653c8ccb34dde4e21c6076d85a72d21ed9d971.zip", - "dir": "libimagequant-ba653c8ccb34dde4e21c6076d85a72d21ed9d971", + # e5d454b: Merge tag '2.12.6' into msvc + "url": "https://github.com/ImageOptim/libimagequant/archive/e5d454bc7f5eb63ee50c84a83a7fa5ac94f68ec4.zip", # noqa: E501 + "filename": "libimagequant-e5d454bc7f5eb63ee50c84a83a7fa5ac94f68ec4.zip", + "dir": "libimagequant-e5d454bc7f5eb63ee50c84a83a7fa5ac94f68ec4", "patch": { "CMakeLists.txt": { "add_library": "add_compile_options(-openmp-)\r\nadd_library", @@ -251,9 +251,9 @@ deps = { "libs": [r"*.lib"], }, "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/2.6.1.zip", - "filename": "harfbuzz-2.6.1.zip", - "dir": "harfbuzz-2.6.1", + "url": "https://github.com/harfbuzz/harfbuzz/archive/2.6.4.zip", + "filename": "harfbuzz-2.6.4.zip", + "dir": "harfbuzz-2.6.4", "build": [ cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), cmd_nmake(target="clean"), @@ -263,9 +263,9 @@ deps = { "libs": [r"*.lib"], }, "fribidi": { - "url": "https://github.com/fribidi/fribidi/archive/v1.0.7.zip", - "filename": "fribidi-1.0.7.zip", - "dir": "fribidi-1.0.7", + "url": "https://github.com/fribidi/fribidi/archive/v1.0.9.zip", + "filename": "fribidi-1.0.9.zip", + "dir": "fribidi-1.0.9", "build": [ cmd_copy(r"{winbuild_dir}\fribidi.cmake", r"CMakeLists.txt"), cmd_cmake(), From 13dcab0fb74f6a6d8e6f69471a4ec51dc89bf890 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 13 Apr 2020 07:16:46 +1000 Subject: [PATCH 24/27] Replaced DEBUG with logging --- src/PIL/TiffImagePlugin.py | 133 +++++++++++++++---------------------- 1 file changed, 54 insertions(+), 79 deletions(-) diff --git a/src/PIL/TiffImagePlugin.py b/src/PIL/TiffImagePlugin.py index 4b04752ed..dbb165071 100644 --- a/src/PIL/TiffImagePlugin.py +++ b/src/PIL/TiffImagePlugin.py @@ -40,6 +40,7 @@ # import io import itertools +import logging import os import struct import warnings @@ -51,7 +52,7 @@ from . import Image, ImageFile, ImagePalette, TiffTags from ._binary import i8, o8 from .TiffTags import TYPES -DEBUG = False # Needs to be merged with the new logging approach. +logger = logging.getLogger(__name__) # Set these to true to force use of libtiff for reading or writing. READ_LIBTIFF = False @@ -734,29 +735,21 @@ class ImageFileDirectory_v2(MutableMapping): try: for i in range(self._unpack("H", self._ensure_read(fp, 2))[0]): tag, typ, count, data = self._unpack("HHL4s", self._ensure_read(fp, 12)) - if DEBUG: - tagname = TiffTags.lookup(tag).name - typname = TYPES.get(typ, "unknown") - print( - "tag: %s (%d) - type: %s (%d)" % (tagname, tag, typname, typ), - end=" ", - ) + + tagname = TiffTags.lookup(tag).name + typname = TYPES.get(typ, "unknown") + msg = "tag: %s (%d) - type: %s (%d)" % (tagname, tag, typname, typ) try: unit_size, handler = self._load_dispatch[typ] except KeyError: - if DEBUG: - print("- unsupported type", typ) + logger.debug(msg + " - unsupported type {}".format(typ)) continue # ignore unsupported type size = count * unit_size if size > 4: here = fp.tell() (offset,) = self._unpack("L", data) - if DEBUG: - print( - "Tag Location: {} - Data Location: {}".format(here, offset), - end=" ", - ) + msg += " Tag Location: {} - Data Location: {}".format(here, offset) fp.seek(offset) data = ImageFile._safe_read(fp, size) fp.seek(here) @@ -769,19 +762,20 @@ class ImageFileDirectory_v2(MutableMapping): "Expecting to read %d bytes but only got %d." " Skipping tag %s" % (size, len(data), tag) ) + logger.debug(msg) continue if not data: + logger.debug(msg) continue self._tagdata[tag] = data self.tagtype[tag] = typ - if DEBUG: - if size > 32: - print("- value: " % size) - else: - print("- value:", self[tag]) + msg += " - value: " + ( + "" % size if size > 32 else str(data) + ) + logger.debug(msg) (self.next,) = self._unpack("L", self._ensure_read(fp, 4)) except OSError as msg: @@ -802,21 +796,17 @@ class ImageFileDirectory_v2(MutableMapping): if tag == STRIPOFFSETS: stripoffsets = len(entries) typ = self.tagtype.get(tag) - if DEBUG: - print("Tag {}, Type: {}, Value: {}".format(tag, typ, value)) + logger.debug("Tag {}, Type: {}, Value: {}".format(tag, typ, value)) values = value if isinstance(value, tuple) else (value,) data = self._write_dispatch[typ](self, *values) - if DEBUG: - tagname = TiffTags.lookup(tag).name - typname = TYPES.get(typ, "unknown") - print( - "save: %s (%d) - type: %s (%d)" % (tagname, tag, typname, typ), - end=" ", - ) - if len(data) >= 16: - print("- value: " % len(data)) - else: - print("- value:", values) + + tagname = TiffTags.lookup(tag).name + typname = TYPES.get(typ, "unknown") + msg = "save: %s (%d) - type: %s (%d)" % (tagname, tag, typname, typ) + msg += " - value: " + ( + "" % len(data) if len(data) >= 16 else str(values) + ) + logger.debug(msg) # count is sum of lengths for string and arbitrary data if typ in [TiffTags.BYTE, TiffTags.ASCII, TiffTags.UNDEFINED]: @@ -840,8 +830,9 @@ class ImageFileDirectory_v2(MutableMapping): # pass 2: write entries to file for tag, typ, count, value, data in entries: - if DEBUG: - print(tag, typ, count, repr(value), repr(data)) + logger.debug( + "{} {} {} {} {}".format(tag, typ, count, repr(value), repr(data)) + ) result += self._pack("HHL4s", tag, typ, count, value) # -- overwrite here for multi-page -- @@ -997,10 +988,9 @@ class TiffImageFile(ImageFile.ImageFile): self._frame_pos = [] self._n_frames = None - if DEBUG: - print("*** TiffImageFile._open ***") - print("- __first:", self.__first) - print("- ifh: ", ifh) + logger.debug("*** TiffImageFile._open ***") + logger.debug("- __first: {}".format(self.__first)) + logger.debug("- ifh: {}".format(ifh)) # and load the first frame self._seek(0) @@ -1035,18 +1025,16 @@ class TiffImageFile(ImageFile.ImageFile): while len(self._frame_pos) <= frame: if not self.__next: raise EOFError("no more images in TIFF file") - if DEBUG: - print( - "Seeking to frame %s, on frame %s, __next %s, location: %s" - % (frame, self.__frame, self.__next, self.fp.tell()) - ) + logger.debug( + "Seeking to frame %s, on frame %s, __next %s, location: %s" + % (frame, self.__frame, self.__next, self.fp.tell()) + ) # reset buffered io handle in case fp # was passed to libtiff, invalidating the buffer self.fp.tell() self.fp.seek(self.__next) self._frame_pos.append(self.__next) - if DEBUG: - print("Loading tags, location: %s" % self.fp.tell()) + logger.debug("Loading tags, location: %s" % self.fp.tell()) self.tag_v2.load(self.fp) self.__next = self.tag_v2.next if self.__next == 0: @@ -1149,21 +1137,18 @@ class TiffImageFile(ImageFile.ImageFile): # Rearranging for supporting byteio items, since they have a fileno # that returns an OSError if there's no underlying fp. Easier to # deal with here by reordering. - if DEBUG: - print("have getvalue. just sending in a string from getvalue") + logger.debug("have getvalue. just sending in a string from getvalue") n, err = decoder.decode(self.fp.getvalue()) elif fp: # we've got a actual file on disk, pass in the fp. - if DEBUG: - print("have fileno, calling fileno version of the decoder.") + logger.debug("have fileno, calling fileno version of the decoder.") if not close_self_fp: self.fp.seek(0) # 4 bytes, otherwise the trace might error out n, err = decoder.decode(b"fpfp") else: # we have something else. - if DEBUG: - print("don't have fileno or getvalue. just reading") + logger.debug("don't have fileno or getvalue. just reading") self.fp.seek(0) # UNDONE -- so much for that buffer size thing. n, err = decoder.decode(self.fp.read()) @@ -1203,21 +1188,19 @@ class TiffImageFile(ImageFile.ImageFile): fillorder = self.tag_v2.get(FILLORDER, 1) - if DEBUG: - print("*** Summary ***") - print("- compression:", self._compression) - print("- photometric_interpretation:", photo) - print("- planar_configuration:", self._planar_configuration) - print("- fill_order:", fillorder) - print("- YCbCr subsampling:", self.tag.get(530)) + logger.debug("*** Summary ***") + logger.debug("- compression: {}".format(self._compression)) + logger.debug("- photometric_interpretation: {}".format(photo)) + logger.debug("- planar_configuration: {}".format(self._planar_configuration)) + logger.debug("- fill_order: {}".format(fillorder)) + logger.debug("- YCbCr subsampling: {}".format(self.tag.get(530))) # size xsize = int(self.tag_v2.get(IMAGEWIDTH)) ysize = int(self.tag_v2.get(IMAGELENGTH)) self._size = xsize, ysize - if DEBUG: - print("- size:", self.size) + logger.debug("- size: {}".format(self.size)) sampleFormat = self.tag_v2.get(SAMPLEFORMAT, (1,)) if len(sampleFormat) > 1 and max(sampleFormat) == min(sampleFormat) == 1: @@ -1251,18 +1234,15 @@ class TiffImageFile(ImageFile.ImageFile): bps_tuple, extra_tuple, ) - if DEBUG: - print("format key:", key) + logger.debug("format key: {}".format(key)) try: self.mode, rawmode = OPEN_INFO[key] except KeyError: - if DEBUG: - print("- unsupported format") + logger.debug("- unsupported format") raise SyntaxError("unknown pixel mode") - if DEBUG: - print("- raw mode:", rawmode) - print("- pil mode:", self.mode) + logger.debug("- raw mode: {}".format(rawmode)) + logger.debug("- pil mode: {}".format(self.mode)) self.info["compression"] = self._compression @@ -1303,8 +1283,7 @@ class TiffImageFile(ImageFile.ImageFile): if fillorder == 2: # Replace fillorder with fillorder=1 key = key[:3] + (1,) + key[4:] - if DEBUG: - print("format key:", key) + logger.debug("format key: {}".format(key)) # this should always work, since all the # fillorder==2 modes have a corresponding # fillorder=1 mode @@ -1366,8 +1345,7 @@ class TiffImageFile(ImageFile.ImageFile): x = y = 0 layer += 1 else: - if DEBUG: - print("- unsupported data organization") + logger.debug("- unsupported data organization") raise SyntaxError("unknown data organization") # Fix up info. @@ -1447,8 +1425,7 @@ def _save(im, fp, filename): # write any arbitrary tags passed in as an ImageFileDirectory info = im.encoderinfo.get("tiffinfo", {}) - if DEBUG: - print("Tiffinfo Keys: %s" % list(info)) + logger.debug("Tiffinfo Keys: %s" % list(info)) if isinstance(info, ImageFileDirectory_v1): info = info.to_v2() for key in info: @@ -1533,9 +1510,8 @@ def _save(im, fp, filename): ) ifd[JPEGQUALITY] = quality - if DEBUG: - print("Saving using libtiff encoder") - print("Items: %s" % sorted(ifd.items())) + logger.debug("Saving using libtiff encoder") + logger.debug("Items: %s" % sorted(ifd.items())) _fp = 0 if hasattr(fp, "fileno"): try: @@ -1597,8 +1573,7 @@ def _save(im, fp, filename): else: atts[tag] = value - if DEBUG: - print("Converted items: %s" % sorted(atts.items())) + logger.debug("Converted items: %s" % sorted(atts.items())) # libtiff always expects the bytes in native order. # we're storing image byte order. So, if the rawmode From 713ed87e9dd756593c5482b9464deca34dbae1e1 Mon Sep 17 00:00:00 2001 From: nulano Date: Sat, 9 May 2020 03:16:04 +0200 Subject: [PATCH 25/27] build PyPy wheels on GHA; update installation docs --- .github/workflows/test-windows.yml | 8 ++++---- docs/installation.rst | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index a5761a0e0..24f8406bc 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -88,7 +88,7 @@ jobs: # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant - if: "github.event_name != 'push' || contains(matrix.python-version, 'pypy')" + if: "github.event_name != 'push'" run: "& winbuild\\build\\build_dep_libimagequant.cmd" # Raqm dependencies @@ -143,14 +143,14 @@ jobs: - name: Build wheel id: wheel - if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" + if: "github.event_name == 'push'" run: | for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a winbuild\\build\\build_pillow.cmd bdist_wheel" shell: cmd - - uses: actions/upload-artifact@v2-preview - if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" + - uses: actions/upload-artifact@v2 + if: "github.event_name == 'push'" with: name: ${{ steps.wheel.outputs.dist }} path: dist\*.whl diff --git a/docs/installation.rst b/docs/installation.rst index 7d6307b69..58d985691 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -292,7 +292,10 @@ or from within the uncompressed source directory:: Building on Windows ^^^^^^^^^^^^^^^^^^^ -There are build scripts and notes for the Windows build in the ``winbuild`` directory. +We recommend you use prebuilt wheels from PyPI. +If you wish to compile Pillow manually, you can use the build scripts +in the ``winbuild`` directory used for CI testing and development. +These scripts require Visual Studio 2017 or newer and NASM. Building on FreeBSD ^^^^^^^^^^^^^^^^^^^ From 9d701a0026199ff7819d54b7a9bdf6b0a443324c Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 10 May 2020 19:55:33 +1000 Subject: [PATCH 26/27] Removed duplicate return statements --- src/libImaging/Except.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libImaging/Except.c b/src/libImaging/Except.c index 4850b4f87..cce93cdc2 100644 --- a/src/libImaging/Except.c +++ b/src/libImaging/Except.c @@ -43,14 +43,12 @@ void * ImagingError_ModeError(void) { return ImagingError_ValueError("bad image mode"); - return NULL; } void * ImagingError_Mismatch(void) { return ImagingError_ValueError("images don't match"); - return NULL; } void * From b8ec793898c9cebd9f3b109d4083be09b17f8abd Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Fri, 15 May 2020 18:29:52 +1000 Subject: [PATCH 27/27] Fixed ZeroDivisionError in thumbnail --- Tests/test_image_thumbnail.py | 6 ++++++ src/PIL/Image.py | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Tests/test_image_thumbnail.py b/Tests/test_image_thumbnail.py index f4ed8e746..da63efe55 100644 --- a/Tests/test_image_thumbnail.py +++ b/Tests/test_image_thumbnail.py @@ -63,6 +63,12 @@ def test_aspect(): assert im.size == (75, 23) # ratio is 3.260869565217 +def test_division_by_zero(): + im = Image.new("L", (200, 2)) + im.thumbnail((75, 75)) + assert im.size == (75, 1) + + def test_float(): im = Image.new("L", (128, 128)) im.thumbnail((99.9, 99.9)) diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 0c8b42a09..8c5fff8ed 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -2277,7 +2277,9 @@ class Image: if x / y >= aspect: x = round_aspect(y * aspect, key=lambda n: abs(aspect - n / y)) else: - y = round_aspect(x / aspect, key=lambda n: abs(aspect - x / n)) + y = round_aspect( + x / aspect, key=lambda n: 0 if n == 0 else abs(aspect - x / n) + ) size = (x, y) box = None