cleanup build configuration

.appveyor.yml

@@ -7,10 +7,8 @@ init:
 # Uncomment previous line to get RDP access during the build.
 
 environment:
-  X64_EXT: -x64
   EXECUTABLE: python.exe
   PIP_DIR: Scripts
-  VENV: NO
   TEST_OPTIONS:
   DEPLOY: YES
   matrix:
@@ -32,12 +30,11 @@ install:
 - curl -fsSL -o pillow-depends.zip https://github.com/python-pillow/pillow-depends/archive/master.zip
 - 7z x pillow-depends.zip -oc:\
 - mv c:\pillow-depends-master c:\pillow-depends
-- mkdir c:\pillow\winbuild\depends
-- xcopy c:\pillow-depends\*.zip c:\pillow\winbuild\depends\
-- xcopy c:\pillow-depends\*.tar.gz c:\pillow\winbuild\depends\
 - xcopy /s c:\pillow-depends\test_images\* c:\pillow\tests\images
 - 7z x ..\pillow-depends\nasm-2.14.02-win64.zip -oc:\
-- path c:\nasm-2.14.02;%PATH%
+- curl -fsSL -o gs952.exe https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs952/gs952w32.exe
+- gs952.exe /S
+- path c:\nasm-2.14.02;C:\Program Files (x86)\gs\gs9.52\bin;%PATH%
 - cd c:\pillow\winbuild\
 - ps: |
       if ($env:PYTHON -eq "c:/vp/pypy3")
@@ -51,14 +48,12 @@ install:
       }
       else
       {
-        $env:PILLOW_BIN = "c:\pillow\"
+        $env:PILLOW_DEPS = "C:\pillow-depends\"
         c:\python37\python.exe c:\pillow\winbuild\build_prepare.py
         c:\pillow\winbuild\build\build_dep_all.cmd
         $host.SetShouldExit(0)
       }
-- curl -fsSL -o gs952.exe https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs952/gs952w32.exe
+- path C:\pillow\winbuild\build\bin;%PATH%
-- gs952.exe /S
-- path %path%;C:\Program Files (x86)\gs\gs9.52\bin
 
 build_script:
 - ps: |

.github/workflows/test-windows.yml

@@ -66,9 +66,7 @@ jobs:
         ..\pillow-depends\gs950w32.exe /S
         Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin"
 
-        mkdir $env:GITHUB_WORKSPACE\winbuild\depends\
+        $env:PILLOW_DEPS = "$env:RUNNER_WORKSPACE\pillow-depends\"
-        xcopy ..\pillow-depends\*.zip $env:GITHUB_WORKSPACE\winbuild\depends\
-        xcopy ..\pillow-depends\*.tar.gz $env:GITHUB_WORKSPACE\winbuild\depends\
         xcopy /s ..\pillow-depends\test_images\* $env:GITHUB_WORKSPACE\tests\images\
 
         $env:PYTHON=$env:pythonLocation

Tests/test_imageshow.py

@@ -1,7 +1,7 @@
 import pytest
 from PIL import Image, ImageShow
 
-from .helper import hopper, is_win32, on_ci, on_github_actions
+from .helper import hopper, is_win32, on_ci
 
 
 def test_sanity():

winbuild/README.md

@@ -3,16 +3,28 @@ Quick README
 
 For more extensive info, see the [Windows build instructions](build.rst).
 
-* See https://github.com/python-pillow/Pillow/issues/553#issuecomment-37877416 and https://github.com/matplotlib/matplotlib/issues/1717#issuecomment-13343859
+* See [Current Windows Build/Testing process (Pillow#553)](https://github.com/python-pillow/Pillow/issues/553#issuecomment-37877416),
+  [Definitive docs for how to compile on Windows (matplotlib#1717)](https://github.com/matplotlib/matplotlib/issues/1717#issuecomment-13343859),
+  [Test Windows with GitHub Actions (Pillow#4084)](https://github.com/python-pillow/Pillow/pull/4084).
   
-*  Works best with Python 3.4, due to virtualenv and pip batteries included. Python3+ required for fetch command.
+
-*  Check config.py for virtual env paths, suffix for 64-bit releases. Defaults to `x64`, set `X64_EXT` to change.
+* Requires Microsoft Visual Studio 2017 or newer with C++ component.
-*  When running in CI with one Python per invocation, set the `PYTHON` env variable to the Python folder. (e.g. `PYTHON`=`c:\Python27\`) This overrides the matrix in config.py and will just build and test for the specific Python.
+* Requires NASM for libjpeg-turbo, a required dependency when using this script.
-* `python get_pythons.py` downloads all the Python releases, and their signatures. (Manually) Install in `c:\PythonXX[x64]\`.
+* Requires CMake 3.13 or newer.
-* `python build_dep.py` downloads and creates a build script for all the dependencies, in 32 and 64-bit versions, and with both compiler versions.
+* Python 3.6+ is required to generate valid scripts, but builds targeting Python 3.5+ are supported.
-* (in powershell) `build_deps.cmd` invokes the dependency build.
+* Tested on Windows Server 2016 with Visual Studio 2017 Community (AppVeyor).
-* `python build.py --clean` makes Pillow for the matrix of Pythons.
+* Tested on Windows Server 2019 with Visual Studio 2019 Enterprise (GitHub Actions).
-* `python test.py` runs the tests on Pillow in all the virtual envs.
+
-*  Currently working with zlib, libjpeg, freetype, and libtiff on Python 2.7, and 3.4, both 32 and 64-bit, on a local win7 pro machine and appveyor.com
+The following is a simplified version of the script used on AppVeyor:
-* WebP is built, not detected.
+```
-* LCMS, OpenJPEG and libimagequant are not building.
+set PYTHON=C:\Python35\bin
+cd /D C:\Pillow\winbuild
+C:\Python37\bin\python.exe build_prepare.py
+build\build_dep_all.cmd
+build\build_pillow.cmd install
+cd ..
+path C:\Pillow\winbuild\build\bin;%PATH%
+%PYTHON%\python.exe selftest.py
+%PYTHON%\python.exe -m pytest -vx --cov PIL --cov Tests --cov-report term --cov-report xml Tests
+build\build_pillow.cmd bdist_wheel
+```

winbuild/build_prepare.py

@@ -95,7 +95,6 @@ architectures = {
 }
 
 header = [
-    cmd_set("BUILD", "{build_dir}"),
     cmd_set("INCLUDE", "{inc_dir}"),
     cmd_set("INCLIB", "{lib_dir}"),
     cmd_set("LIB", "{lib_dir}"),
@@ -145,7 +144,7 @@ deps = {
         "filename": "tiff-4.1.0.tar.gz",
         "dir": "tiff-4.1.0",
         "build": [
-            cmd_copy(r"{script_dir}\tiff.opt", "nmake.opt"),
+            cmd_copy(r"{winbuild_dir}\tiff.opt", "nmake.opt"),
             cmd_nmake("makefile.vc", "clean"),
             cmd_nmake("makefile.vc", "lib"),
         ],
@@ -176,8 +175,7 @@ deps = {
         "patch": {
             r"builds\windows\vc2010\freetype.vcxproj": {
                 # freetype setting is /MD for .dll and /MT for .lib, we need /MD
-                "<RuntimeLibrary>MultiThreaded</RuntimeLibrary>":
+                "<RuntimeLibrary>MultiThreaded</RuntimeLibrary>": "<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>",  # noqa E501
-                    "<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>",
             }
         },
         "build": [
@@ -200,14 +198,11 @@ deps = {
         "patch": {
             r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": {
                 # default is /MD for x86 and /MT for x64, we need /MD always
-                "<RuntimeLibrary>MultiThreaded</RuntimeLibrary>":
+                "<RuntimeLibrary>MultiThreaded</RuntimeLibrary>": "<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>",  # noqa E501
-                    "<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>",
                 # retarget to default toolset (selected by vcvarsall.bat)
-                "<PlatformToolset>v141</PlatformToolset>":
+                "<PlatformToolset>v141</PlatformToolset>": "<PlatformToolset>$(DefaultPlatformToolset)</PlatformToolset>",  # noqa E501
-                    "<PlatformToolset>$(DefaultPlatformToolset)</PlatformToolset>",
                 # retarget to latest (selected by vcvarsall.bat)
-                "<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>":
+                "<WindowsTargetPlatformVersion>8.1</WindowsTargetPlatformVersion>": "<WindowsTargetPlatformVersion>$(WindowsSDKVersion)</WindowsTargetPlatformVersion>",  # noqa E501
-                    "<WindowsTargetPlatformVersion>$(WindowsSDKVersion)</WindowsTargetPlatformVersion>",  # noqa E501
             }
         },
         "build": [
@@ -269,7 +264,7 @@ deps = {
         "filename": "fribidi-1.0.7.zip",
         "dir": "fribidi-1.0.7",
         "build": [
-            cmd_copy(r"{script_dir}\fribidi.cmake", r"CMakeLists.txt"),
+            cmd_copy(r"{winbuild_dir}\fribidi.cmake", r"CMakeLists.txt"),
             cmd_cmake(),
             cmd_nmake(target="clean"),
             cmd_nmake(target="fribidi"),
@@ -282,7 +277,7 @@ deps = {
         "filename": "libraqm-0.7.0.zip",
         "dir": "libraqm-0.7.0",
         "build": [
-            cmd_copy(r"{script_dir}\raqm.cmake", r"CMakeLists.txt"),
+            cmd_copy(r"{winbuild_dir}\raqm.cmake", r"CMakeLists.txt"),
             cmd_cmake(),
             cmd_nmake(target="clean"),
             cmd_nmake(target="libraqm"),
@@ -357,12 +352,6 @@ def find_msvs():
     return vs
 
 
-def match(values, target):
-    for key, value in values.items():
-        if key in target:
-            return {"name": key, **value}
-
-
 def extract_dep(url, filename):
     import urllib.request
     import tarfile
@@ -455,51 +444,42 @@ def build_dep_all():
     write_script("build_dep_all.cmd", lines)
 
 
-def build_pillow(wheel=False):
+def build_pillow():
-    lines = []
+    lines = [
-    if path_dir is not None and not wheel:
+        "@echo ---- Building Pillow (build_ext %*) ----",
-        lines.append(cmd_xcopy("{bin_dir}", path_dir))
+        cmd_cd("{pillow_dir}"),
-    lines.extend(prefs["header"])
+        *prefs["header"],
-    lines.extend(
+        cmd_set("DISTUTILS_USE_SDK", "1"),  # use same compiler to build Pillow
-        [
+        cmd_set("MSSdk", "1"),  # for Python 3.5 and PyPy3.6
-            "@echo ---- Building Pillow (build_ext %*) ----",
+        cmd_set("py_vcruntime_redist", "true"),  # use /MD, not /MT
-            cmd_cd("{pillow_dir}"),
+        r'"{python_dir}\{python_exe}" setup.py build_ext %*',
-            cmd_append("LIB", r"{python_dir}\tcl"),
+    ]
-            cmd_set("MSSdk", "1"),
-            cmd_set("DISTUTILS_USE_SDK", "1"),
-            cmd_set("py_vcruntime_redist", "true"),
-            r'"{python_dir}\python.exe" setup.py build_ext %*',
-        ]
-    )
 
     write_script("build_pillow.cmd", lines)
 
 
 if __name__ == "__main__":
     # winbuild directory
-    script_dir = os.path.dirname(os.path.realpath(__file__))
+    winbuild_dir = os.path.dirname(os.path.realpath(__file__))
 
     # dependency cache directory
-    depends_dir = os.path.join(script_dir, "depends")
+    depends_dir = os.environ.get("PILLOW_DEPS", os.path.join(winbuild_dir, "depends"))
+    os.makedirs(depends_dir, exist_ok=True)
     print("Caching dependencies in:", depends_dir)
 
-    # python bin directory
+    # Python bin directory
-    python_dir = os.environ.get(
+    python_dir = os.environ.get("PYTHON")
-        "PYTHON", os.path.dirname(os.path.realpath(sys.executable))
+    python_exe = os.environ.get("EXECUTABLE", "python.exe")
+    if python_dir is None:
+        python_dir = os.path.dirname(os.path.realpath(sys.executable))
+        python_exe = os.path.basename(sys.executable)
+    print("Target Python:", os.path.join(python_dir, python_exe))
+
+    # use ARCHITECTURE or PYTHON to select architecture
+    architecture = os.environ.get(
+        "ARCHITECTURE", "x64" if "x64" in python_dir else "x86"
     )
-    print("Target Python:", python_dir)
+    arch_prefs = architectures[architecture]
-
-    # copy binaries to this directory
-    path_dir = os.environ.get("PILLOW_BIN")
-    print("Copying binary files to:", path_dir)
-
-    # use PYTHON to select architecture
-    arch_prefs = match(architectures, python_dir)
-    if arch_prefs is None:
-        architecture = "x86"
-        arch_prefs = architectures[architecture]
-    else:
-        architecture = arch_prefs["name"]
     print("Target Architecture:", architecture)
 
     msvs = find_msvs()
@@ -510,7 +490,7 @@ if __name__ == "__main__":
     print("Found Visual Studio at:", msvs["vs_dir"])
 
     # build root directory
-    build_dir = os.environ.get("PILLOW_BUILD", os.path.join(script_dir, "build"))
+    build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build"))
     print("Using output directory:", build_dir)
 
     # build directory for *.h files
@@ -523,25 +503,30 @@ if __name__ == "__main__":
     bin_dir = os.path.join(build_dir, "bin")
 
     shutil.rmtree(build_dir, ignore_errors=True)
-    for path in [depends_dir, build_dir, lib_dir, inc_dir, bin_dir]:
+    for path in [build_dir, inc_dir, lib_dir, bin_dir]:
-        os.makedirs(path, exist_ok=True)
+        os.makedirs(path)
 
     prefs = {
-        "architecture": architecture,
+        # Python paths / preferences
-        "script_dir": script_dir,
-        "depends_dir": depends_dir,
         "python_dir": python_dir,
+        "python_exe": python_exe,
+        "architecture": architecture,
+        **arch_prefs,
+        # Pillow paths
+        "pillow_dir": os.path.realpath(os.path.join(winbuild_dir, "..")),
+        "winbuild_dir": winbuild_dir,
+        # Build paths
         "build_dir": build_dir,
-        "lib_dir": lib_dir,
         "inc_dir": inc_dir,
+        "lib_dir": lib_dir,
         "bin_dir": bin_dir,
-        "pillow_dir": os.path.realpath(os.path.join(script_dir, "..")),
+        # Compilers / Tools
-        # TODO auto find:
+        **msvs,
-        "cmake": "cmake.exe",
+        "cmake": "cmake.exe",  # TODO find CMAKE automatically
+        # TODO find NASM automatically
+        # script header
+        "header": sum([header, msvs["header"], ["@echo on"]], []),
     }
-    prefs.update(msvs)
-    prefs.update(arch_prefs)
-    prefs["header"] = sum([header, msvs["header"], ["@echo on"]], [])
 
     build_dep_all()
     build_pillow()

winbuild/commands.py

@@ -1,3 +0,0 @@
-# builtins
-
-