From 89e05cdf56fa3dd93df9c4301f8bd88399534080 Mon Sep 17 00:00:00 2001 From: nulano Date: Wed, 2 Oct 2019 19:48:55 +0200 Subject: [PATCH] extract scripts from test-windows.yml (cherry-picked from commit 1a6ee35d0e5fa1bf4081649a6ebf67208d76f322) --- .github/workflows/test-windows.yml | 204 ++---------- setup.py | 2 +- winbuild/build_prepare.py | 494 +++++++++++++++++++++++++++++ winbuild/commands.py | 111 +++++++ 4 files changed, 634 insertions(+), 177 deletions(-) create mode 100644 winbuild/build_prepare.py create mode 100644 winbuild/commands.py diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index adbe97968..6425f3b5c 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -4,7 +4,6 @@ on: [push, pull_request] jobs: build: - runs-on: windows-2019 strategy: fail-fast: false @@ -70,155 +69,28 @@ jobs: Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" $env:PYTHON=$env:pythonLocation - xcopy ..\pillow-depends\*.zip $env:GITHUB_WORKSPACE\winbuild\ - xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\ + mkdir $env:GITHUB_WORKSPACE\winbuild\depends\ + xcopy ..\pillow-depends\*.zip $env:GITHUB_WORKSPACE\winbuild\depends\ + xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\depends\ xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\ cd $env:GITHUB_WORKSPACE/winbuild/ - python.exe $env:GITHUB_WORKSPACE\winbuild\build_dep.py - env: - EXECUTABLE: bin\python.exe + python.exe $env:GITHUB_WORKSPACE\winbuild\build_prepare.py shell: pwsh - name: Build dependencies / libjpeg - if: false - run: | - REM FIXME uses /MT not /MD, see makefile.vc and win32.mak for more info - - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\jpeg-9d - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - nmake -nologo -f makefile.vc setup-vc6 - nmake -nologo -f makefile.vc clean - nmake -nologo -f makefile.vc nodebug=1 libjpeg.lib cjpeg.exe djpeg.exe - copy /Y /B j*.h %INCLIB% - copy /Y /B *.lib %INCLIB% - copy /Y /B *.exe %INCLIB% - shell: cmd - - - name: Build dependencies / libjpeg-turbo - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\libjpeg-turbo-2.0.3 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DENABLE_SHARED:BOOL=OFF -DWITH_JPEG8:BOOL=TRUE -DWITH_CRT_DLL:BOOL=TRUE -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile jpeg-static cjpeg-static djpeg-static - copy /Y /B j*.h %INCLIB% - copy /Y /B jpeg-static.lib %INCLIB%\libjpeg.lib - copy /Y /B cjpeg-static.exe %INCLIB%\cjpeg.exe - copy /Y /B djpeg-static.exe %INCLIB%\djpeg.exe - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libjpeg.cmd" - name: Build dependencies / zlib - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\zlib-1.2.11 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - nmake -nologo -f win32\Makefile.msc clean - nmake -nologo -f win32\Makefile.msc zlib.lib - copy /Y /B z*.h %INCLIB% - copy /Y /B *.lib %INCLIB% - copy /Y /B zlib.lib %INCLIB%\z.lib - shell: cmd - - - name: Build dependencies / LibTIFF - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\tiff-4.1.0 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - copy %GITHUB_WORKSPACE%\winbuild\tiff.opt nmake.opt - nmake -nologo -f makefile.vc clean - nmake -nologo -f makefile.vc lib - copy /Y /B libtiff\tiff*.h %INCLIB% - copy /Y /B libtiff\*.dll %INCLIB% - copy /Y /B libtiff\*.lib %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_zlib.cmd" + - name: Build dependencies / LibTiff + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libtiff.cmd" - name: Build dependencies / WebP - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\libwebp-1.1.0 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - rmdir /S /Q output\release-static - nmake -nologo -f Makefile.vc CFG=release-static OBJDIR=output ARCH=${{ matrix.architecture }} all - mkdir %INCLIB%\webp - copy /Y /B src\webp\*.h %INCLIB%\webp - copy /Y /B output\release-static\${{ matrix.architecture }}\lib\* %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_libwebp.cmd" - name: Build dependencies / FreeType - run: | - REM Toolkit v100 not available; missing VCTargetsPath; Clean fails - - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\freetype-2.10.1 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - rmdir /S /Q objs - set DefaultPlatformToolset=v142 - set VCTargetsPath=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\ - set MSBUILD="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe" - powershell -Command "(gc builds\windows\vc2010\freetype.vcxproj) -replace 'MultiThreaded<', 'MultiThreadedDLL<' | Out-File -encoding ASCII builds\windows\vc2010\freetype.vcxproj" - %MSBUILD% builds\windows\vc2010\freetype.sln /t:Build /p:Configuration="Release Static" /p:Platform=${{ matrix.platform-msbuild }} /m - xcopy /Y /E /Q include %INCLIB% - copy /Y /B "objs\${{ matrix.platform-msbuild }}\Release Static\freetype.lib" %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_freetype.cmd" - name: Build dependencies / LCMS2 - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\lcms2-2.8 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - rmdir /S /Q Lib - rmdir /S /Q Projects\VC2015\Release - set VCTargetsPath=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\ - set MSBUILD="C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\MSBuild.exe" - powershell %GITHUB_WORKSPACE%\winbuild\lcms2_patch.ps1 - %MSBUILD% Projects\VC2015\lcms2.sln /t:Clean;lcms2_static /p:Configuration="Release" /p:Platform=${{ matrix.platform-msbuild }} /m - xcopy /Y /E /Q include %INCLIB% - copy /Y /B Lib\MS\*.lib %INCLIB% - shell: cmd - + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_lcms2.cmd" - name: Build dependencies / OpenJPEG - run: | - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - set BUILD=%GITHUB_WORKSPACE%\winbuild\build - cd /D %BUILD%\openjpeg-2.3.1msvcr10-x32 - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - echo on - set CMAKE=cmake.exe -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES:BOOL=OFF - set CMAKE=%CMAKE% -DBUILD_THIRDPARTY:BOOL=OFF -DBUILD_SHARED_LIBS:BOOL=OFF - set CMAKE=%CMAKE% -DCMAKE_BUILD_TYPE=Release - %CMAKE% -G "NMake Makefiles" . - nmake -nologo -f Makefile clean - nmake -nologo -f Makefile - mkdir %INCLIB%\openjpeg-2.3.1 - copy /Y /B src\lib\openjp2\*.h %INCLIB%\openjpeg-2.3.1 - copy /Y /B bin\*.lib %INCLIB% - shell: cmd + run: "%GITHUB_WORKSPACE%\\winbuild\\build_dep_openjpeg.cmd" # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant @@ -304,22 +176,12 @@ jobs: - name: Build Pillow run: | - set PYTHON=%pythonLocation% - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set MPLSRC=%GITHUB_WORKSPACE% - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - cd /D %GITHUB_WORKSPACE% - set LIB=%INCLIB%;%PYTHON%\tcl - set INCLUDE=%INCLIB%;%GITHUB_WORKSPACE%\depends\tcl86\include;%INCLUDE% - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - set MSSdk=1 - set DISTUTILS_USE_SDK=1 - set py_vcruntime_redist=true - %PYTHON%\python.exe setup.py build_ext install - rem Add libraqm.dll (copied to INCLIB) to PATH. - path %INCLIB%;%PATH% - %PYTHON%\python.exe selftest.py --installed - shell: cmd + cd $env:GITHUB_WORKSPACE + # & winbuild\build_dep_tcl.cmd + # & winbuild\build_dep_tk.cmd + & winbuild\build_pillow.cmd + & $env:pythonLocation\python.exe selftest.py --installed + shell: pwsh # failing with PyPy3 - name: Enable heap verification @@ -330,12 +192,8 @@ jobs: - name: Test Pillow run: | - set PYTHON=%pythonLocation% - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - rem Add libraqm.dll (copied to INCLIB) to PATH. - path %INCLIB%;%PATH% - cd /D %GITHUB_WORKSPACE% - %PYTHON%\python.exe -m pytest -vx -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests + path %GITHUB_WORKSPACE%\\winbuild\\build\\bin;%PATH% + python.exe -m pytest -vx -W always --cov PIL --cov Tests --cov-report term --cov-report xml Tests shell: cmd - name: Prepare to upload errors @@ -366,22 +224,16 @@ jobs: - name: Build wheel id: wheel if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" - run: | - for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ##[set-output name=dist;]dist-%%a - for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a - set PYTHON=%pythonLocation% - set INCLUDE=C:\Program Files (x86)\Microsoft SDKs\Windows\V7.1A\Include - set MPLSRC=%GITHUB_WORKSPACE% - set INCLIB=%GITHUB_WORKSPACE%\winbuild\depends\msvcr10-x32 - cd /D %GITHUB_WORKSPACE% - set LIB=%INCLIB%;%PYTHON%\tcl - set INCLUDE=%INCLIB%;%GITHUB_WORKSPACE%\depends\tcl86\include;%INCLUDE% - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.platform-vcvars }} - %PYTHON%\python.exe setup.py bdist_wheel - shell: cmd + run: "%GITHUB_WORKSPACE%\\winbuild\\build_pillow_wheel.cmd" - uses: actions/upload-artifact@v1 if: "github.event_name == 'push' && !contains(matrix.python-version, 'pypy')" with: - name: ${{ steps.wheel.outputs.dist }} + name: dist path: dist + +# - uses: actions/upload-artifact@v1 +# if: matrix.architecture == 'x86' +# with: +# name: lib +# path: "winbuild\\build\\3.x\\x86\\lib" diff --git a/setup.py b/setup.py index 1236b5df5..b55b1de62 100755 --- a/setup.py +++ b/setup.py @@ -120,7 +120,7 @@ _LIB_IMAGING = ( "codec_fd", ) -DEBUG = False +DEBUG = True class DependencyException(Exception): diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py new file mode 100644 index 000000000..c17af7b95 --- /dev/null +++ b/winbuild/build_prepare.py @@ -0,0 +1,494 @@ +import os +import shutil +import subprocess +import winreg +from itertools import count + +from commands import * + +SF_MIRROR = "http://iweb.dl.sourceforge.net" + +# use PYTHON to select architecture +architectures = { + "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, + "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, +} + +# use PYTHON + architecture to select config +pythons = { + "pypy3.6": {"config-x86": "3.5"}, + "3.5": {"config-x86": "3.5", "config-x64": "3.5"}, + "3.6": {"config-x86": "3.6", "config-x64": "3.6"}, + "3.7": {"config-x86": "3.6", "config-x64": "3.6"}, +} + +# select deps and libs +configs = { + "3.5": { + "deps": [ + "libjpeg-turbo-2.0.3", + "zlib-1.2.11", + "tiff-4.0.10", + "libwebp-1.0.3", + "freetype-2.10.1", + "lcms2-2.9", + "openjpeg-2.3.1", + "ghostscript-9.27", + # "tcl-8.6", + # "tk-8.6", + ], + "vcvars_ver": "14.0", + "vs_ver": "2015", + }, + "3.6": { + "deps": [ + "libjpeg-turbo-2.0.3", + "zlib-1.2.11", + "tiff-4.0.10", + "libwebp-1.0.3", + "freetype-2.10.1", + "lcms2-2.9", + "openjpeg-2.3.1", + "ghostscript-9.27", + # "tcl-8.6", + # "tk-8.6", + ], + "vs_ver": "2017", + }, +} + +header = [ + cmd_set("BUILD", "{build_dir}"), + cmd_set("INCLUDE", "{inc_dir}"), + cmd_set("INCLIB", "{lib_dir}"), + cmd_set("LIB", "{lib_dir}"), + cmd_append("PATH", "{bin_dir}"), + "@echo on", +] + +# dependencies +deps = { + "jpeg-9c": { + "name": "libjpeg", + # FIXME HTTP 403 + "url": "http://www.ijg.org/files/jpegsr9c.zip", + "filename": "jpegsr9c.zip", + "build": [ + # FIXME builds with -MT, not -MD + cmd_nmake("makefile.vc", "setup-vc6"), + cmd_nmake("makefile.vc", "clean"), + cmd_nmake("makefile.vc", "libjpeg.lib", "nodebug=1"), + ], + "headers": [r"j*.h"], + "libs": [r"*.lib"], + }, + "libjpeg-turbo-2.0.3": { + "name": "libjpeg", + "url": SF_MIRROR + "/project/libjpeg-turbo/2.0.3/libjpeg-turbo-2.0.3.tar.gz", + "filename": "libjpeg-turbo-2.0.3.tar.gz", + "build": [ + cmd_cmake([ + "-DENABLE_SHARED:BOOL=FALSE", + "-DWITH_JPEG8:BOOL=TRUE", + "-DWITH_CRT_DLL:BOOL=TRUE", + ]), + cmd_nmake(target="clean"), + cmd_nmake(target="jpeg-static"), + cmd_copy("jpeg-static.lib", "libjpeg.lib"), + cmd_nmake(target="cjpeg-static"), + cmd_copy("cjpeg-static.exe", "cjpeg.exe"), + cmd_nmake(target="djpeg-static"), + cmd_copy("djpeg-static.exe", "djpeg.exe"), + ], + "headers": ["j*.h"], + "libs": ["libjpeg.lib"], + "bins": ["cjpeg.exe", "djpeg.exe"], + }, + "zlib-1.2.11": { + "name": "zlib", + "url": "http://zlib.net/zlib1211.zip", + "filename": "zlib1211.zip", + "build": [ + cmd_nmake(r"win32\Makefile.msc", "clean"), + cmd_nmake(r"win32\Makefile.msc", "zlib.lib"), + cmd_copy("zlib.lib", "z.lib"), + ], + "headers": [r"z*.h"], + "libs": [r"*.lib"], + }, + "tiff-4.0.10": { + "name": "libtiff", + # FIXME FTP timeout + "url": "ftp://download.osgeo.org/libtiff/tiff-4.0.10.tar.gz", + "filename": "tiff-4.0.10.tar.gz", + "build": [ + cmd_copy(r"{script_dir}\tiff.opt", "nmake.opt"), + cmd_nmake("makefile.vc", "clean"), + cmd_nmake("makefile.vc", "lib", "RuntimeLibrary=-MT"), + ], + "headers": [r"libtiff\tiff*.h"], + "libs": [r"libtiff\*.lib"], + # "bins": [r"libtiff\*.dll"], + }, + "libwebp-1.0.3": { + "name": "libwebp", + "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.0.3.tar.gz", # noqa: E501 + "filename": "libwebp-1.0.3.tar.gz", + "build": [ + cmd_rmdir(r"output\release-static"), # clean + cmd_nmake( + "Makefile.vc", + "all", + ["CFG=release-static", "OBJDIR=output", "ARCH={architecture}"] + ), + cmd_mkdir(r"{inc_dir}\webp"), + cmd_copy(r"src\webp\*.h", r"{inc_dir}\webp"), + ], + "libs": [r"output\release-static\{architecture}\lib\*.lib"], + }, + "freetype-2.10.1": { + "name": "freetype", + "url": "https://download.savannah.gnu.org/releases/freetype/freetype-2.10.1.tar.gz", # noqa: E501 + "filename": "freetype-2.10.1.tar.gz", + "build": [ + cmd_rmdir("objs"), + # freetype setting is /MD for .dll and /MT for .lib, we need /MD + cmd_patch_replace( + r"builds\windows\vc2010\freetype.vcxproj", + "MultiThreaded<", + "MultiThreadedDLL<" + ), + cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Clean"), # TODO failing on GHA # noqa: E501 + cmd_msbuild(r"builds\windows\vc2010\freetype.sln", "Release Static", "Build"), + cmd_xcopy("include", "{inc_dir}"), + ], + "libs": [r"objs\{msbuild_arch}\Release Static\freetype.lib"], + # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], + }, + "lcms2-2.9": { + "name": "lcms2", + "url": SF_MIRROR + "/project/lcms/lcms/2.8/lcms2-2.9.tar.gz", + "filename": "lcms2-2.9.tar.gz", + "build": [ + cmd_rmdir("Lib"), + cmd_rmdir(r"Projects\VC{vs_ver}\Release"), + # lcms2-2.8\VC2015 setting is /MD for x86 and /MT for x64, we need /MD always + cmd_patch_replace( + r"Projects\VC2017\lcms2.sln", "MultiThreaded<", "MultiThreadedDLL<" + ), + cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "Clean"), + cmd_msbuild(r"Projects\VC{vs_ver}\lcms2.sln", "Release", "lcms2_static"), + cmd_xcopy("include", "{inc_dir}"), + ], + "libs": [r"Lib\MS\*.lib"], + }, + "openjpeg-2.3.1": { + "name": "openjpeg", + "url": "https://github.com/uclouvain/openjpeg/archive/v2.3.1.tar.gz", + "filename": "openjpeg-2.3.1.tar.gz", + "build": [ + cmd_cmake(("-DBUILD_THIRDPARTY:BOOL=OFF", "-DBUILD_SHARED_LIBS:BOOL=OFF")), + cmd_nmake(target="clean"), + cmd_nmake(), + cmd_mkdir(r"{inc_dir}\openjpeg-2.3.1"), + cmd_copy(r"src\lib\openjp2\*.h", r"{inc_dir}\openjpeg-2.3.1"), + ], + "libs": [r"bin\*.lib"], + }, + "ghostscript-9.27": { + "name": "ghostscript", + "url": "https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs927/ghostscript-9.27.tar.gz", # noqa: E501 + "filename": "ghostscript-9.27.tar.gz", + "build": [ + cmd_set("MSVC_VERSION", 14), + cmd_if_eq("{architecture}", "x64", cmd_set("WIN64", '""')), + cmd_nmake(r"psi\msvc.mak"), + ], + "bins": [r"bin\*"], + }, + "tcl-8.5": { + "name": "tcl", + "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tcl8519-src.zip", + "filename": "tcl8519-src.zip", + }, + "tk-8.5": { + "name": "tk", + "url": SF_MIRROR + "/project/tcl/Tcl/8.5.19/tk8519-src.zip", + "filename": "tk8519-src.zip", + }, + "tcl-8.6": { + "name": "tcl", + "url": SF_MIRROR + "/project/tcl/Tcl/8.6.9/tcl869-src.zip", + "filename": "tcl869-src.zip", + "headers": [r"generic\*.h"], + }, + "tk-8.6": { + "name": "tk", + "url": SF_MIRROR + "/project/tcl/Tcl/8.6.9/tk869-src.zip", + "filename": "tk869-src.zip", + "build": [ + r"""mkdir {inc_dir}\X11""", + r"""copy /Y /B xlib\X11\* "{inc_dir}\X11\" """, + ], + "headers": [r"generic\*.h"], + }, +} + + +# based on distutils._msvccompiler from CPython 3.7.4 +def find_vs2017(config): + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if not root: + print("Program Files not found") + return None + + try: + vspath = subprocess.check_output([ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath", + "-products", "*", + ]).decode(encoding="mbcs").strip() + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + print("vswhere not found") + return None + + if not os.path.isdir(os.path.join(vspath, "VC", "Auxiliary", "Build")): + print("Visual Studio seems to be missing C compiler") + return None + + vs = { + "header": [], + # nmake selected by vcvarsall + "nmake": "nmake.exe", + } + + msbuild = os.path.join(vspath, "MSBuild", "15.0", "Bin", "MSBuild.exe") + if os.path.isfile(msbuild): + # default_platform_toolset = "v140" + vs["msbuild"] = '"{}"'.format(msbuild) + # vs["header"].append(cmd_set("DefaultPlatformToolset", default_platform_toolset)) + else: + print("Visual Studio MSBuild not found") + return None + + vcvarsall = os.path.join(vspath, "VC", "Auxiliary", "Build", "vcvarsall.bat") + if not os.path.isfile(vcvarsall): + print("Visual Studio vcvarsall not found") + return None + vcvars_ver = "-vcvars_ver={}".format(config["vcvars_ver"]) if "vcvars_ver" in config else "" + vs["header"].append('call "{}" {{vcvars_arch}} {}'.format(vcvarsall, vcvars_ver)) + + return vs + + +def find_sdk71a(): + try: + key = winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A", + access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY, + ) + except OSError: + return None + with key: + for i in count(): + try: + v_name, v_value, v_type = winreg.EnumValue(key, i) + except OSError: + return None + if v_name == "InstallationFolder" and v_type == winreg.REG_SZ: + sdk_dir = v_value + break + else: + return None + + if not os.path.isdir(sdk_dir): + return None + + sdk = { + "header": [ + # for win32.mak + cmd_append("INCLUDE", os.path.join(sdk_dir, "Include")), + # for ghostscript + cmd_set("RCOMP", '"{}"'.format(os.path.join(sdk_dir, "Bin", "RC.EXE"))), + ] + } + + return sdk + + +def match(values, target): + for key, value in values.items(): + if key in target: + return {"name": key, **value} + + +def extract_dep(url, filename): + import urllib.request + import tarfile + import zipfile + + file = os.path.join(depends_dir, filename) + if not os.path.exists(file): + ex = None + for i in range(3): + try: + print("Fetching %s (attempt %d)..." % (url, i + 1)) + content = urllib.request.urlopen(url).read() + with open(file, "wb") as f: + f.write(content) + break + except urllib.error.URLError as e: + ex = e + else: + raise RuntimeError(ex) + if filename.endswith(".zip"): + with zipfile.ZipFile(file) as zf: + zf.extractall(build_dir) + elif filename.endswith(".tar.gz") or filename.endswith(".tgz"): + with tarfile.open(file, "r:gz") as tgz: + tgz.extractall(build_dir) + else: + raise RuntimeError("Unknown archive type: " + filename) + + +def write_script(name, lines): + name = os.path.join(script_dir, name) + lines = [line.format(**prefs) for line in lines] + print("Writing " + name) + with open(name, "w") as f: + f.write("\n\r".join(lines)) + for line in lines: + print(" " + line) + + +def get_footer(dep): + lines = [] + for out in dep.get("headers", []): + lines.append(cmd_copy(out, "{inc_dir}")) + for out in dep.get("libs", []): + lines.append(cmd_copy(out, "{lib_dir}")) + for out in dep.get("bins", []): + lines.append(cmd_copy(out, "{bin_dir}")) + return lines + + +def build_dep(name): + dep = deps[name] + file = "build_dep_{name}.cmd".format(name=dep["name"]) + + extract_dep(dep["url"], dep["filename"]) + + lines = ["cd /D %s" % os.path.join(build_dir, name)] + lines.extend(prefs["header"]) + + lines.extend(dep.get("build", [])) + + lines.extend(get_footer(dep)) + + write_script(file, lines) + return file + + +def build_dep_all(): + lines = ["cd {script_dir}"] + for dep_name in prefs["deps"]: + lines.append('cmd.exe /c "%s"' % build_dep(dep_name)) + write_script("build_dep_all.ps1", lines) + + +def build_pillow(wheel=False): + if not wheel: + op, filename = "install", "build_pillow.cmd" + else: + op, filename = "bdist_wheel", "build_pillow_wheel.cmd" + + lines = [] + if path_dir is not None and not wheel: + lines.append(cmd_xcopy("{bin_dir}", path_dir)) + lines.extend(prefs["header"]) + lines.extend( + [ + cmd_cd("{pillow_dir}"), + cmd_append("LIB", r"{python_dir}\tcl"), + r'"{{python_dir}}\python.exe" setup.py build_ext {}'.format(op), + # r"""%PYTHON%\python.exe selftest.py --installed""", + ] + ) + + write_script(filename, lines) + + +if __name__ == "__main__": + script_dir = os.path.dirname(os.path.realpath(__file__)) + depends_dir = os.path.join(script_dir, "depends") + python_dir = os.environ["PYTHON"] + + # copy binaries to this directory + path_dir = os.environ.get("PILLOW_BIN") + + # use PYTHON to select architecture + arch_prefs = match(architectures, python_dir) + if arch_prefs is None: + architecture = "x86" + print("WARN: Could not determine architecture, guessing " + architecture) + arch_prefs = architectures[architecture] + else: + architecture = arch_prefs["name"] + + # use PYTHON to select python version + python_prefs = match(pythons, python_dir) + if python_prefs is None: + raise KeyError("Failed to determine Python version from PYTHON: " + python_dir) + + # use python version + architecture to select build config + config_name = python_prefs["config-" + architecture] + config = configs[config_name] + + vs2017 = find_vs2017(config) + if vs2017 is None: + raise RuntimeError("Visual Studio 2017 not found") + + sdk71a = find_sdk71a() + if sdk71a is None: + raise RuntimeError("Windows SDK v7.1A not found") + + build_dir = os.path.join(script_dir, "build", config_name, architecture) + lib_dir = os.path.join(build_dir, "lib") + inc_dir = os.path.join(build_dir, "inc") + bin_dir = os.path.join(build_dir, "bin") + + # for path in [lib_dir, inc_dir, bin_dir]: + # shutil.rmtree(path) + for path in [depends_dir, build_dir, lib_dir, inc_dir, bin_dir]: + os.makedirs(path, exist_ok=True) + + prefs = { + "python_version": python_prefs["name"], + "architecture": architecture, + "script_dir": script_dir, + "depends_dir": depends_dir, + "python_dir": python_dir, + "build_dir": build_dir, + "lib_dir": lib_dir, + "inc_dir": inc_dir, + "bin_dir": bin_dir, + "pillow_dir": os.path.realpath(os.path.join(script_dir, "..")), + # TODO auto find: + "cmake": "cmake.exe", + } + + dicts = [vs2017, sdk71a, arch_prefs, python_prefs, config] + for x in dicts: + prefs.update(x) + prefs["header"] = sum((x.get("header", []) for x in dicts), header) + del prefs["name"] + + print("Target: Python {python_version} {architecture}".format(**prefs)) + + build_dep_all() + build_pillow() + build_pillow(wheel=True) diff --git a/winbuild/commands.py b/winbuild/commands.py new file mode 100644 index 000000000..2fbbc9b1a --- /dev/null +++ b/winbuild/commands.py @@ -0,0 +1,111 @@ +# builtins + + +def cmd_cd(path): + return "cd /D {path}".format(**locals()) + + +def cmd_set(name, value): + return "set {name}={value}".format(**locals()) + + +def cmd_prepend(name, value): + op = "path " if name == "PATH" else "set {name}=" + return (op + "{value};%{name}%").format(**locals()) + + +def cmd_append(name, value): + op = "path " if name == "PATH" else "set {name}=" + return (op + "%{name}%;{value}").format(**locals()) + + +def cmd_copy(src, tgt): + return 'copy /Y /B "{src}" "{tgt}"'.format(**locals()) + + +def cmd_xcopy(src, tgt): + return 'xcopy /Y /E "{src}" "{tgt}"'.format(**locals()) + + +def cmd_mkdir(path): + return 'mkdir "{path}"'.format(**locals()) + + +def cmd_rmdir(path): + return 'rmdir /S /Q "{path}"'.format(**locals()) + + +def cmd_if_eq(a, b, cmd): + return 'if "{a}"=="{b}" {cmd}'.format(**locals()) + + +# tools + + +def cmd_nmake(makefile=None, target="", params=None): + if params is None: + params = "" + elif isinstance(params, list) or isinstance(params, tuple): + params = " ".join(params) + else: + params = str(params) + + return " ".join( + [ + "{{nmake}}", + "-nologo", + '-f "{makefile}"' if makefile is not None else "", + "{params}", + '"{target}"', + ] + ).format(**locals()) + + +def cmd_cmake(params=None, file="."): + if params is None: + params = "" + elif isinstance(params, list) or isinstance(params, tuple): + params = " ".join(params) + else: + params = str(params) + return " ".join( + [ + "{{cmake}}", + "-DCMAKE_VERBOSE_MAKEFILE=ON", + "-DCMAKE_RULE_MESSAGES:BOOL=OFF", + "-DCMAKE_BUILD_TYPE=Release", + "{params}", + '-G "NMake Makefiles"', + '"{file}"', + ] + ).format(**locals()) + + +def cmd_msbuild( + file, configuration="Release", target="Build", platform="{msbuild_arch}" +): + return " ".join( + [ + "{{msbuild}}", + "{file}", + '/t:"{target}"', + '/p:Configuration="{configuration}"', + "/p:Platform={platform}", + "/m", + ] + ).format(**locals()) + + +# patch tools + + +def cmd_patch_replace(file, src, dst): + return " ".join( + [ + "echo", + "(Get-Content '{file}')", + '-replace "{src}", "{dst}"', + "^| Out-File -encoding ASCII '{file}'", + "> temp.ps1\r\npowershell .\\temp.ps1", + ] + ).format(**locals())