Merge pull request #4701 from nulano/winbuild-cache

.github/workflows/test-windows.yml

@@ -68,7 +68,7 @@ jobs:
       if: "contains(matrix.python-version, '3.9-dev')"
       run: python -m pip install -U "setuptools>=49.3.2"
 
-    - name: Prepare dependencies
+    - name: Install dependencies
       run: |
         7z x winbuild\depends\nasm-2.14.02-win64.zip "-o$env:RUNNER_WORKSPACE\"
         Write-Host "::add-path::$env:RUNNER_WORKSPACE\nasm-2.14.02"
@@ -77,41 +77,71 @@ jobs:
         Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin"
 
         xcopy /s winbuild\depends\test_images\* Tests\images\
+      shell: pwsh
 
-        & python.exe winbuild\build_prepare.py -v --python=$env:pythonLocation
+    - name: Cache build
+      id: build-cache
+      uses: actions/cache@v2
+      with:
+        path: winbuild\build
+        key:
+          ${{ hashFiles('winbuild\build_prepare.py') }}-${{ hashFiles('.github\workflows\test-windows.yml') }}-${{ env.pythonLocation }}
+
+    - name: Prepare build
+      if: steps.build-cache.outputs.cache-hit != 'true'
+      run: |
+        & python.exe winbuild\build_prepare.py -v --python=$env:pythonLocation --srcdir
       shell: pwsh
 
     - name: Build dependencies / libjpeg-turbo
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_libjpeg.cmd"
     - name: Build dependencies / zlib
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_zlib.cmd"
     - name: Build dependencies / LibTiff
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_libtiff.cmd"
     - name: Build dependencies / WebP
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_libwebp.cmd"
     - name: Build dependencies / FreeType
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_freetype.cmd"
     - name: Build dependencies / LCMS2
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_lcms2.cmd"
     - name: Build dependencies / OpenJPEG
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_openjpeg.cmd"
 
-    # GPL licensed; skip if building wheels
+    # GPL licensed
     - name: Build dependencies / libimagequant
-      if: "github.event_name != 'push'"
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_libimagequant.cmd"
 
     # Raqm dependencies
     - name: Build dependencies / HarfBuzz
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_harfbuzz.cmd"
     - name: Build dependencies / FriBidi
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_fribidi.cmd"
     - name: Build dependencies / Raqm
+      if: steps.build-cache.outputs.cache-hit != 'true'
       run: "& winbuild\\build\\build_dep_libraqm.cmd"
 
+    # trim ~150MB x 9
+    - name: Optimize build cache
+      if: steps.build-cache.outputs.cache-hit != 'true'
+      run: rmdir /S /Q winbuild\build\src
+      shell: cmd
+
     - name: Build Pillow
       run: |
-        & winbuild\build\build_pillow.cmd install
+        $FLAGS=""
+        if ('${{ github.event_name }}' -eq 'push') { $FLAGS="--disable-imagequant" }
+        & winbuild\build\build_pillow.cmd $FLAGS install
         & $env:pythonLocation\python.exe selftest.py --installed
       shell: pwsh
 
@@ -156,7 +186,7 @@ jobs:
       if: "github.event_name == 'push'"
       run: |
         for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a
-        winbuild\\build\\build_pillow.cmd bdist_wheel"
+        winbuild\\build\\build_pillow.cmd --disable-imagequant bdist_wheel
       shell: cmd
 
     - uses: actions/upload-artifact@v2

winbuild/build_prepare.py

@@ -378,10 +378,10 @@ def extract_dep(url, filename):
     print("Extracting " + filename)
     if filename.endswith(".zip"):
         with zipfile.ZipFile(file) as zf:
-            zf.extractall(build_dir)
+            zf.extractall(sources_dir)
     elif filename.endswith(".tar.gz") or filename.endswith(".tgz"):
         with tarfile.open(file, "r:gz") as tgz:
-            tgz.extractall(build_dir)
+            tgz.extractall(sources_dir)
     else:
         raise RuntimeError("Unknown archive type: " + filename)
 
@@ -416,7 +416,7 @@ def build_dep(name):
     extract_dep(dep["url"], dep["filename"])
 
     for patch_file, patch_list in dep.get("patch", {}).items():
-        patch_file = os.path.join(build_dir, dir, patch_file.format(**prefs))
+        patch_file = os.path.join(sources_dir, dir, patch_file.format(**prefs))
         with open(patch_file, "r") as f:
             text = f.read()
         for patch_from, patch_to in patch_list.items():
@@ -432,7 +432,7 @@ def build_dep(name):
         "@echo " + ("=" * 70),
         f"@echo ==== {banner:<60} ====",
         "@echo " + ("=" * 70),
-        "cd /D %s" % os.path.join(build_dir, dir),
+        "cd /D %s" % os.path.join(sources_dir, dir),
         *prefs["header"],
         *dep.get("build", []),
         *get_footer(dep),
@@ -481,6 +481,7 @@ if __name__ == "__main__":
         "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64"
     )
     build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build"))
+    sources_dir = ""
     for arg in sys.argv[1:]:
         if arg == "-v":
             verbose = True
@@ -498,6 +499,8 @@ if __name__ == "__main__":
             architecture = arg[15:]
         elif arg.startswith("--dir="):
             build_dir = arg[6:]
+        elif arg == "--srcdir":
+            sources_dir = os.path.sep + "src"
         else:
             raise ValueError("Unknown parameter: " + arg)
 
@@ -528,10 +531,13 @@ if __name__ == "__main__":
     lib_dir = os.path.join(build_dir, "lib")
     # build directory for *.bin files
     bin_dir = os.path.join(build_dir, "bin")
+    # directory for storing project files
+    sources_dir = build_dir + sources_dir
 
     shutil.rmtree(build_dir, ignore_errors=True)
-    for path in [build_dir, inc_dir, lib_dir, bin_dir]:
+    os.makedirs(build_dir, exist_ok=False)
-        os.makedirs(path)
+    for path in [inc_dir, lib_dir, bin_dir, sources_dir]:
+        os.makedirs(path, exist_ok=True)
 
     prefs = {
         # Python paths / preferences
@@ -547,6 +553,7 @@ if __name__ == "__main__":
         "inc_dir": inc_dir,
         "lib_dir": lib_dir,
         "bin_dir": bin_dir,
+        "src_dir": sources_dir,
         # Compilers / Tools
         **msvs,
         "cmake": "cmake.exe",  # TODO find CMAKE automatically