From e11612f589fca9f94e5c96f983ed775e9fd5a1b9 Mon Sep 17 00:00:00 2001 From: nulano Date: Tue, 16 Jun 2020 04:05:34 +0200 Subject: [PATCH 1/2] cache winbuild --- .github/workflows/test-windows.yml | 38 +++++++++++++++++++++++++----- winbuild/build_prepare.py | 19 ++++++++++----- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 7ae26b883..05937c17b 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -36,7 +36,7 @@ jobs: path: winbuild\depends - name: Cache pip - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: ~\AppData\Local\pip\Cache key: @@ -58,7 +58,7 @@ jobs: - name: pip install wheel pytest pytest-cov run: python -m pip install wheel pytest pytest-cov - - name: Prepare dependencies + - name: Install dependencies run: | 7z x winbuild\depends\nasm-2.14.02-win64.zip "-o$env:RUNNER_WORKSPACE\" Write-Host "::add-path::$env:RUNNER_WORKSPACE\nasm-2.14.02" @@ -67,38 +67,66 @@ jobs: Write-Host "::add-path::C:\Program Files (x86)\gs\gs9.50\bin" xcopy /s winbuild\depends\test_images\* Tests\images\ + shell: pwsh - & python.exe winbuild\build_prepare.py -v --python=$env:pythonLocation + - name: Cache build + id: build-cache + uses: actions/cache@v2 + with: + path: winbuild\build + key: + ${{ hashFiles('winbuild\build_prepare.py') }}-${{ hashFiles('.github\workflows\test-windows.yml') }}-${{ env.pythonLocation }} + + - name: Prepare build + if: steps.build-cache.outputs.cache-hit != 'true' + run: | + & python.exe winbuild\build_prepare.py -v --python=$env:pythonLocation --srcdir shell: pwsh - name: Build dependencies / libjpeg-turbo + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_libjpeg.cmd" - name: Build dependencies / zlib + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_zlib.cmd" - name: Build dependencies / LibTiff + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_libtiff.cmd" - name: Build dependencies / WebP + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_libwebp.cmd" - name: Build dependencies / FreeType + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_freetype.cmd" - name: Build dependencies / LCMS2 + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_lcms2.cmd" - name: Build dependencies / OpenJPEG + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_openjpeg.cmd" # GPL licensed; skip if building wheels - name: Build dependencies / libimagequant - if: "github.event_name != 'push'" + if: false run: "& winbuild\\build\\build_dep_libimagequant.cmd" # Raqm dependencies - name: Build dependencies / HarfBuzz + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_harfbuzz.cmd" - name: Build dependencies / FriBidi + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_fribidi.cmd" - name: Build dependencies / Raqm + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_libraqm.cmd" + # trim ~150MB x 9 + - name: Optimize build cache + if: steps.build-cache.outputs.cache-hit != 'true' + run: rmdir /S /Q winbuild\build\src + shell: cmd + - name: Build Pillow run: | & winbuild\build\build_pillow.cmd install @@ -143,14 +171,12 @@ jobs: - name: Build wheel id: wheel - if: "github.event_name == 'push'" run: | for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a winbuild\\build\\build_pillow.cmd bdist_wheel" shell: cmd - uses: actions/upload-artifact@v2 - if: "github.event_name == 'push'" with: name: ${{ steps.wheel.outputs.dist }} path: dist\*.whl diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 0ba8a135c..97ed1e873 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -378,10 +378,10 @@ def extract_dep(url, filename): print("Extracting " + filename) if filename.endswith(".zip"): with zipfile.ZipFile(file) as zf: - zf.extractall(build_dir) + zf.extractall(sources_dir) elif filename.endswith(".tar.gz") or filename.endswith(".tgz"): with tarfile.open(file, "r:gz") as tgz: - tgz.extractall(build_dir) + tgz.extractall(sources_dir) else: raise RuntimeError("Unknown archive type: " + filename) @@ -416,7 +416,7 @@ def build_dep(name): extract_dep(dep["url"], dep["filename"]) for patch_file, patch_list in dep.get("patch", {}).items(): - patch_file = os.path.join(build_dir, dir, patch_file.format(**prefs)) + patch_file = os.path.join(sources_dir, dir, patch_file.format(**prefs)) with open(patch_file, "r") as f: text = f.read() for patch_from, patch_to in patch_list.items(): @@ -429,7 +429,7 @@ def build_dep(name): "@echo " + ("=" * 70), "@echo ==== {:<60} ====".format(banner), "@echo " + ("=" * 70), - "cd /D %s" % os.path.join(build_dir, dir), + "cd /D %s" % os.path.join(sources_dir, dir), *prefs["header"], *dep.get("build", []), *get_footer(dep), @@ -477,6 +477,7 @@ if __name__ == "__main__": "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64" ) build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")) + sources_dir = "" for arg in sys.argv[1:]: if arg == "-v": verbose = True @@ -494,6 +495,8 @@ if __name__ == "__main__": architecture = arg[15:] elif arg.startswith("--dir="): build_dir = arg[6:] + elif arg == "--srcdir": + sources_dir = os.path.sep + "src" else: raise ValueError("Unknown parameter: " + arg) @@ -524,10 +527,13 @@ if __name__ == "__main__": lib_dir = os.path.join(build_dir, "lib") # build directory for *.bin files bin_dir = os.path.join(build_dir, "bin") + # directory for storing project files + sources_dir = build_dir + sources_dir shutil.rmtree(build_dir, ignore_errors=True) - for path in [build_dir, inc_dir, lib_dir, bin_dir]: - os.makedirs(path) + os.makedirs(build_dir, exist_ok=False) + for path in [inc_dir, lib_dir, bin_dir, sources_dir]: + os.makedirs(path, exist_ok=True) prefs = { # Python paths / preferences @@ -543,6 +549,7 @@ if __name__ == "__main__": "inc_dir": inc_dir, "lib_dir": lib_dir, "bin_dir": bin_dir, + "src_dir": sources_dir, # Compilers / Tools **msvs, "cmake": "cmake.exe", # TODO find CMAKE automatically From eb83f0bac2009d894c3832316bea9517dfd56272 Mon Sep 17 00:00:00 2001 From: nulano Date: Sun, 21 Jun 2020 12:32:37 +0200 Subject: [PATCH 2/2] keep imagequant enabled for PRs --- .github/workflows/test-windows.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 05937c17b..5dcee2441 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -105,9 +105,9 @@ jobs: if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_openjpeg.cmd" - # GPL licensed; skip if building wheels + # GPL licensed - name: Build dependencies / libimagequant - if: false + if: steps.build-cache.outputs.cache-hit != 'true' run: "& winbuild\\build\\build_dep_libimagequant.cmd" # Raqm dependencies @@ -129,7 +129,9 @@ jobs: - name: Build Pillow run: | - & winbuild\build\build_pillow.cmd install + $FLAGS="" + if ('${{ github.event_name }}' -eq 'push') { $FLAGS="--disable-imagequant" } + & winbuild\build\build_pillow.cmd $FLAGS install & $env:pythonLocation\python.exe selftest.py --installed shell: pwsh @@ -171,12 +173,14 @@ jobs: - name: Build wheel id: wheel + if: "github.event_name == 'push'" run: | for /f "tokens=3 delims=/" %%a in ("${{ github.ref }}") do echo ::set-output name=dist::dist-%%a - winbuild\\build\\build_pillow.cmd bdist_wheel" + winbuild\\build\\build_pillow.cmd --disable-imagequant bdist_wheel shell: cmd - uses: actions/upload-artifact@v2 + if: "github.event_name == 'push'" with: name: ${{ steps.wheel.outputs.dist }} path: dist\*.whl