Pillow/winbuild/build_prepare.py

747 lines
26 KiB
Python
Raw Normal View History

2023-09-18 17:03:18 +03:00
from __future__ import annotations
import argparse
import os
import platform
import re
import shutil
2023-06-27 07:43:58 +03:00
import struct
import subprocess
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_cd(path: str) -> str:
2020-07-09 22:08:15 +03:00
return f"cd /D {path}"
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_set(name: str, value: str) -> str:
2020-07-09 22:08:15 +03:00
return f"set {name}={value}"
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_append(name: str, value: str) -> str:
2020-07-09 22:08:15 +03:00
op = "path " if name == "PATH" else f"set {name}="
return op + f"%{name}%;{value}"
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_copy(src: str, tgt: str) -> str:
2020-07-09 22:08:15 +03:00
return f'copy /Y /B "{src}" "{tgt}"'
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_xcopy(src: str, tgt: str) -> str:
2020-07-09 22:08:15 +03:00
return f'xcopy /Y /E "{src}" "{tgt}"'
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_mkdir(path: str) -> str:
2020-07-09 22:08:15 +03:00
return f'mkdir "{path}"'
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_rmdir(path: str) -> str:
2020-07-09 22:08:15 +03:00
return f'rmdir /S /Q "{path}"'
2020-01-01 03:48:13 +03:00
2023-09-18 17:03:18 +03:00
def cmd_nmake(
makefile: str | None = None,
target: str = "",
2023-09-20 12:24:43 +03:00
params: list[str] | None = None,
2023-09-18 17:03:18 +03:00
) -> str:
2023-09-20 12:24:43 +03:00
params = "" if params is None else " ".join(params)
2020-01-01 03:48:13 +03:00
return " ".join(
[
2020-07-09 22:08:15 +03:00
"{nmake}",
2020-01-01 03:48:13 +03:00
"-nologo",
2020-07-09 22:08:15 +03:00
f'-f "{makefile}"' if makefile is not None else "",
f"{params}",
f'"{target}"',
2020-01-01 03:48:13 +03:00
]
2020-07-09 22:08:15 +03:00
)
2020-01-01 03:48:13 +03:00
def cmds_cmake(
target: str | tuple[str, ...] | list[str], *params, build_dir: str = "."
) -> list[str]:
if not isinstance(target, str):
target = " ".join(target)
return [
" ".join(
[
"{cmake}",
"-DCMAKE_BUILD_TYPE=Release",
"-DCMAKE_VERBOSE_MAKEFILE=ON",
"-DCMAKE_RULE_MESSAGES:BOOL=OFF", # for NMake
"-DCMAKE_C_COMPILER=cl.exe", # for Ninja
"-DCMAKE_CXX_COMPILER=cl.exe", # for Ninja
"-DCMAKE_C_FLAGS=-nologo",
"-DCMAKE_CXX_FLAGS=-nologo",
*params,
'-G "{cmake_generator}"',
f'-B "{build_dir}"',
f"-S .",
]
),
f'{{cmake}} --build "{build_dir}" --clean-first --parallel --target {target}',
]
2020-01-01 03:48:13 +03:00
def cmd_msbuild(
2023-09-18 17:03:18 +03:00
file: str,
configuration: str = "Release",
target: str = "Build",
platform: str = "{msbuild_arch}",
) -> str:
2020-01-01 03:48:13 +03:00
return " ".join(
[
2020-07-09 22:08:15 +03:00
"{msbuild}",
f"{file}",
f'/t:"{target}"',
f'/p:Configuration="{configuration}"',
f"/p:Platform={platform}",
2020-01-01 03:48:13 +03:00
"/m",
]
2020-07-09 22:08:15 +03:00
)
2020-01-01 03:48:13 +03:00
SF_PROJECTS = "https://sourceforge.net/projects"
2023-09-18 17:06:30 +03:00
ARCHITECTURES = {
2023-06-27 07:43:58 +03:00
"x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"},
"x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"},
"ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"},
}
2020-04-12 11:18:49 +03:00
# dependencies, listed in order of compilation
2023-09-18 17:06:30 +03:00
DEPS = {
2019-12-31 20:13:16 +03:00
"libjpeg": {
"url": SF_PROJECTS
2023-10-16 23:51:56 +03:00
+ "/libjpeg-turbo/files/3.0.1/libjpeg-turbo-3.0.1.tar.gz/download",
"filename": "libjpeg-turbo-3.0.1.tar.gz",
"dir": "libjpeg-turbo-3.0.1",
"license": ["README.ijg", "LICENSE.md"],
"license_pattern": (
"(LEGAL ISSUES\n============\n\n.+?)\n\nREFERENCES\n=========="
".+(libjpeg-turbo Licenses\n======================\n\n.+)$"
),
2023-11-17 00:57:41 +03:00
"patch": {
r"CMakeLists.txt": {
# libjpeg-turbo does not detect MSVC x86_arm64 cross-compiler correctly
'if(MSVC_IDE AND CMAKE_GENERATOR_PLATFORM MATCHES "arm64")': "if({architecture} STREQUAL ARM64)", # noqa: E501
},
},
"build": [
*cmds_cmake(
("jpeg-static", "cjpeg-static", "djpeg-static"),
"-DENABLE_SHARED:BOOL=FALSE",
"-DWITH_JPEG8:BOOL=TRUE",
"-DWITH_CRT_DLL:BOOL=TRUE",
2019-12-31 20:13:16 +03:00
),
cmd_copy("jpeg-static.lib", "libjpeg.lib"),
cmd_copy("cjpeg-static.exe", "cjpeg.exe"),
cmd_copy("djpeg-static.exe", "djpeg.exe"),
],
"headers": ["j*.h"],
"libs": ["libjpeg.lib"],
"bins": ["cjpeg.exe", "djpeg.exe"],
},
2019-12-31 20:13:16 +03:00
"zlib": {
2023-08-18 15:22:51 +03:00
"url": "https://zlib.net/zlib13.zip",
"filename": "zlib13.zip",
"dir": "zlib-1.3",
"license": "README",
"license_pattern": "Copyright notice:\n\n(.+)$",
"build": [
cmd_nmake(r"win32\Makefile.msc", "clean"),
cmd_nmake(r"win32\Makefile.msc", "zlib.lib"),
cmd_copy("zlib.lib", "z.lib"),
],
"headers": [r"z*.h"],
"libs": [r"*.lib"],
},
"xz": {
2023-11-01 23:54:34 +03:00
"url": SF_PROJECTS + "/lzmautils/files/xz-5.4.5.tar.gz/download",
"filename": "xz-5.4.5.tar.gz",
"dir": "xz-5.4.5",
"license": "COPYING",
"build": [
*cmds_cmake("liblzma", "-DBUILD_SHARED_LIBS:BOOL=OFF"),
cmd_mkdir(r"{inc_dir}\lzma"),
cmd_copy(r"src\liblzma\api\lzma\*.h", r"{inc_dir}\lzma"),
],
"headers": [r"src\liblzma\api\lzma.h"],
"libs": [r"liblzma.lib"],
},
2019-12-31 20:13:16 +03:00
"libwebp": {
2023-09-14 08:32:33 +03:00
"url": "http://downloads.webmproject.org/releases/webp/libwebp-1.3.2.tar.gz",
"filename": "libwebp-1.3.2.tar.gz",
"dir": "libwebp-1.3.2",
2022-09-05 13:48:42 +03:00
"license": "COPYING",
"build": [
cmd_rmdir(r"output\release-static"), # clean
cmd_nmake(
"Makefile.vc",
"all",
2022-09-05 13:48:42 +03:00
[
"CFG=release-static",
"RTLIBCFG=dynamic",
2022-09-05 13:48:42 +03:00
"OBJDIR=output",
"ARCH={architecture}",
"LIBWEBP_BASENAME=webp",
],
),
cmd_mkdir(r"{inc_dir}\webp"),
cmd_copy(r"src\webp\*.h", r"{inc_dir}\webp"),
],
"libs": [r"output\release-static\{architecture}\lib\*.lib"],
},
2019-12-31 20:13:16 +03:00
"libtiff": {
2023-10-09 13:49:00 +03:00
"url": "https://download.osgeo.org/libtiff/tiff-4.6.0.tar.gz",
"filename": "tiff-4.6.0.tar.gz",
"dir": "tiff-4.6.0",
2022-12-24 06:19:32 +03:00
"license": "LICENSE.md",
"patch": {
r"libtiff\tif_lzma.c": {
# link against liblzma.lib
"#ifdef LZMA_SUPPORT": '#ifdef LZMA_SUPPORT\n#pragma comment(lib, "liblzma.lib")', # noqa: E501
},
2022-09-05 13:48:42 +03:00
r"libtiff\tif_webp.c": {
# link against webp.lib
"#ifdef WEBP_SUPPORT": '#ifdef WEBP_SUPPORT\n#pragma comment(lib, "webp.lib")', # noqa: E501
},
r"test\CMakeLists.txt": {
2023-06-27 06:51:17 +03:00
"add_executable(test_write_read_tags ../placeholder.h)": "",
"target_sources(test_write_read_tags PRIVATE test_write_read_tags.c)": "", # noqa: E501
"target_link_libraries(test_write_read_tags PRIVATE tiff)": "",
"list(APPEND simple_tests test_write_read_tags)": "",
},
},
"build": [
*cmds_cmake(
"tiff",
"-DBUILD_SHARED_LIBS:BOOL=OFF",
'-DCMAKE_C_FLAGS="-nologo -DLZMA_API_STATIC"',
)
],
"headers": [r"libtiff\tiff*.h"],
"libs": [r"libtiff\*.lib"],
},
"libpng": {
2022-11-21 07:42:44 +03:00
"url": SF_PROJECTS + "/libpng/files/libpng16/1.6.39/lpng1639.zip/download",
"filename": "lpng1639.zip",
"dir": "lpng1639",
"license": "LICENSE",
"build": [
*cmds_cmake("png_static", "-DPNG_SHARED:BOOL=OFF", "-DPNG_TESTS:BOOL=OFF"),
cmd_copy("libpng16_static.lib", "libpng16.lib"),
],
"headers": [r"png*.h"],
"libs": [r"libpng16.lib"],
},
2022-09-07 20:59:55 +03:00
"brotli": {
2023-10-09 12:14:42 +03:00
"url": "https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz",
"filename": "brotli-1.1.0.tar.gz",
"dir": "brotli-1.1.0",
2022-09-07 20:59:55 +03:00
"license": "LICENSE",
"build": [
2023-10-09 18:43:28 +03:00
*cmds_cmake(("brotlicommon", "brotlidec"), "-DBUILD_SHARED_LIBS:BOOL=OFF"),
2022-09-07 20:59:55 +03:00
cmd_xcopy(r"c\include", "{inc_dir}"),
],
"libs": ["*.lib"],
},
2019-12-31 20:13:16 +03:00
"freetype": {
2023-02-23 16:45:11 +03:00
"url": "https://download.savannah.gnu.org/releases/freetype/freetype-2.13.2.tar.gz",
2023-08-26 06:28:53 +03:00
"filename": "freetype-2.13.2.tar.gz",
"dir": "freetype-2.13.2",
"license": ["LICENSE.TXT", r"docs\FTL.TXT", r"docs\GPLv2.TXT"],
2019-12-31 19:34:11 +03:00
"patch": {
r"builds\windows\vc2010\freetype.vcxproj": {
# freetype setting is /MD for .dll and /MT for .lib, we need /MD
2020-04-12 11:18:49 +03:00
"<RuntimeLibrary>MultiThreaded</RuntimeLibrary>": "<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>", # noqa: E501
2020-02-25 02:00:57 +03:00
# freetype doesn't specify SDK version, MSBuild may guess incorrectly
2020-04-12 11:18:49 +03:00
'<PropertyGroup Label="Globals">': '<PropertyGroup Label="Globals">\n <WindowsTargetPlatformVersion>$(WindowsSDKVersion)</WindowsTargetPlatformVersion>', # noqa: E501
},
r"builds\windows\vc2010\freetype.user.props": {
2022-09-07 20:59:55 +03:00
"<UserDefines></UserDefines>": "<UserDefines>FT_CONFIG_OPTION_SYSTEM_ZLIB;FT_CONFIG_OPTION_USE_PNG;FT_CONFIG_OPTION_USE_HARFBUZZ;FT_CONFIG_OPTION_USE_BROTLI</UserDefines>", # noqa: E501
"<UserIncludeDirectories></UserIncludeDirectories>": r"<UserIncludeDirectories>{dir_harfbuzz}\src;{inc_dir}</UserIncludeDirectories>", # noqa: E501
"<UserLibraryDirectories></UserLibraryDirectories>": "<UserLibraryDirectories>{lib_dir}</UserLibraryDirectories>", # noqa: E501
2023-10-09 12:14:42 +03:00
"<UserDependencies></UserDependencies>": "<UserDependencies>zlib.lib;libpng16.lib;brotlicommon.lib;brotlidec.lib</UserDependencies>", # noqa: E501
},
r"src/autofit/afshaper.c": {
# link against harfbuzz.lib
"#ifdef FT_CONFIG_OPTION_USE_HARFBUZZ": '#ifdef FT_CONFIG_OPTION_USE_HARFBUZZ\n#pragma comment(lib, "harfbuzz.lib")', # noqa: E501
},
2019-12-31 19:34:11 +03:00
},
"build": [
cmd_rmdir("objs"),
2019-12-31 20:13:16 +03:00
cmd_msbuild(
r"builds\windows\vc2010\freetype.sln", "Release Static", "Clean"
),
cmd_msbuild(
r"builds\windows\vc2010\freetype.sln", "Release Static", "Build"
),
cmd_xcopy("include", "{inc_dir}"),
],
"libs": [r"objs\{msbuild_arch}\Release Static\freetype.lib"],
},
2019-12-31 20:13:16 +03:00
"lcms2": {
2023-12-04 00:32:39 +03:00
"url": SF_PROJECTS + "/lcms/files/lcms/2.16/lcms2-2.16.tar.gz/download",
"filename": "lcms2-2.16.tar.gz",
"dir": "lcms2-2.16",
"license": "LICENSE",
2019-12-31 19:34:11 +03:00
"patch": {
2022-08-24 08:39:43 +03:00
r"Projects\VC2022\lcms2_static\lcms2_static.vcxproj": {
2019-12-31 20:13:16 +03:00
# default is /MD for x86 and /MT for x64, we need /MD always
2020-04-12 11:18:49 +03:00
"<RuntimeLibrary>MultiThreaded</RuntimeLibrary>": "<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>", # noqa: E501
2019-12-31 20:13:16 +03:00
# retarget to default toolset (selected by vcvarsall.bat)
2022-08-24 08:39:43 +03:00
"<PlatformToolset>v143</PlatformToolset>": "<PlatformToolset>$(DefaultPlatformToolset)</PlatformToolset>", # noqa: E501
2019-12-31 20:13:16 +03:00
# retarget to latest (selected by vcvarsall.bat)
"<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>": "<WindowsTargetPlatformVersion>$(WindowsSDKVersion)</WindowsTargetPlatformVersion>", # noqa: E501
2019-12-31 20:13:16 +03:00
}
2019-12-31 19:34:11 +03:00
},
"build": [
cmd_rmdir("Lib"),
2022-08-24 08:39:43 +03:00
cmd_rmdir(r"Projects\VC2022\Release"),
cmd_msbuild(r"Projects\VC2022\lcms2.sln", "Release", "Clean"),
2021-06-04 18:53:20 +03:00
cmd_msbuild(
2022-08-24 08:39:43 +03:00
r"Projects\VC2022\lcms2.sln", "Release", "lcms2_static:Rebuild"
2021-06-04 18:53:20 +03:00
),
cmd_xcopy("include", "{inc_dir}"),
],
"libs": [r"Lib\MS\*.lib"],
},
2019-12-31 20:13:16 +03:00
"openjpeg": {
2022-05-14 03:11:42 +03:00
"url": "https://github.com/uclouvain/openjpeg/archive/v2.5.0.tar.gz",
"filename": "openjpeg-2.5.0.tar.gz",
"dir": "openjpeg-2.5.0",
"license": "LICENSE",
"patch": {
r"src\lib\openjp2\ht_dec.c": {
"#ifdef OPJ_COMPILER_MSVC\n return (OPJ_UINT32)__popcnt(val);": "#if defined(OPJ_COMPILER_MSVC) && (defined(_M_IX86) || defined(_M_AMD64))\n return (OPJ_UINT32)__popcnt(val);", # noqa: E501
}
},
"build": [
*cmds_cmake(
"openjp2", "-DBUILD_CODEC:BOOL=OFF", "-DBUILD_SHARED_LIBS:BOOL=OFF"
),
2022-05-14 03:11:42 +03:00
cmd_mkdir(r"{inc_dir}\openjpeg-2.5.0"),
cmd_copy(r"src\lib\openjp2\*.h", r"{inc_dir}\openjpeg-2.5.0"),
],
"libs": [r"bin\*.lib"],
},
2019-12-31 20:13:16 +03:00
"libimagequant": {
2021-12-30 07:42:30 +03:00
# commit: Merge branch 'master' into msvc (matches 2.17.0 tag)
2023-02-23 16:45:11 +03:00
"url": "https://github.com/ImageOptim/libimagequant/archive/e4c1334be0eff290af5e2b4155057c2953a313ab.zip",
2021-12-30 07:42:30 +03:00
"filename": "libimagequant-e4c1334be0eff290af5e2b4155057c2953a313ab.zip",
"dir": "libimagequant-e4c1334be0eff290af5e2b4155057c2953a313ab",
"license": "COPYRIGHT",
2019-12-31 20:13:16 +03:00
"patch": {
"CMakeLists.txt": {
2021-09-21 05:49:35 +03:00
"if(OPENMP_FOUND)": "if(false)",
"install": "#install",
2023-11-17 00:57:41 +03:00
# libimagequant does not detect MSVC x86_arm64 cross-compiler correctly
"if(${{CMAKE_SYSTEM_PROCESSOR}} STREQUAL ARM64)": "if({architecture} STREQUAL ARM64)", # noqa: E501
2019-12-31 20:13:16 +03:00
}
},
"build": [
*cmds_cmake("imagequant_a"),
2021-09-21 05:49:35 +03:00
cmd_copy("imagequant_a.lib", "imagequant.lib"),
2019-12-31 20:13:16 +03:00
],
"headers": [r"*.h"],
2021-09-21 05:49:35 +03:00
"libs": [r"imagequant.lib"],
2019-12-31 20:13:16 +03:00
},
"harfbuzz": {
2023-11-25 11:42:37 +03:00
"url": "https://github.com/harfbuzz/harfbuzz/archive/8.3.0.zip",
"filename": "harfbuzz-8.3.0.zip",
"dir": "harfbuzz-8.3.0",
"license": "COPYING",
2019-12-31 20:13:16 +03:00
"build": [
*cmds_cmake(
"harfbuzz",
"-DHB_HAVE_FREETYPE:BOOL=TRUE",
'-DCMAKE_CXX_FLAGS="-nologo -d2FH4-"',
),
2019-12-31 20:13:16 +03:00
],
"headers": [r"src\*.h"],
"libs": [r"*.lib"],
},
"fribidi": {
2023-05-17 01:21:59 +03:00
"url": "https://github.com/fribidi/fribidi/archive/v1.0.13.zip",
"filename": "fribidi-1.0.13.zip",
"dir": "fribidi-1.0.13",
"license": "COPYING",
2019-12-31 20:13:16 +03:00
"build": [
2023-05-17 01:21:59 +03:00
cmd_copy(r"COPYING", r"{bin_dir}\fribidi-1.0.13-COPYING"),
2020-01-01 15:02:27 +03:00
cmd_copy(r"{winbuild_dir}\fribidi.cmake", r"CMakeLists.txt"),
# generated tab.i files cannot be cross-compiled
" ^&^& ".join(
[
"if {architecture}==ARM64 cmd /c call {vcvarsall} x86",
*cmds_cmake("fribidi-gen", "-DARCH=x86", build_dir="build_x86"),
]
),
*cmds_cmake("fribidi", "-DARCH={architecture}"),
2019-12-31 20:13:16 +03:00
],
2020-11-25 14:21:42 +03:00
"bins": [r"*.dll"],
2019-12-31 20:13:16 +03:00
},
}
# based on distutils._msvccompiler from CPython 3.7.4
def find_msvs(architecture: str) -> dict[str, str] | None:
root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles")
if not root:
print("Program Files not found")
return None
requires = ["-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"]
2023-11-17 00:57:41 +03:00
if architecture == "ARM64":
requires += ["-requires", "Microsoft.VisualStudio.Component.VC.Tools.ARM64"]
2023-11-17 00:57:41 +03:00
try:
2019-12-31 20:13:16 +03:00
vspath = (
subprocess.check_output(
[
os.path.join(
root, "Microsoft Visual Studio", "Installer", "vswhere.exe"
),
"-latest",
"-prerelease",
*requires,
2019-12-31 20:13:16 +03:00
"-property",
"installationPath",
"-products",
"*",
]
)
.decode(encoding="mbcs")
.strip()
)
except (subprocess.CalledProcessError, OSError, UnicodeDecodeError):
print("vswhere not found")
return None
if not os.path.isdir(os.path.join(vspath, "VC", "Auxiliary", "Build")):
print("Visual Studio seems to be missing C compiler")
return None
2019-12-31 03:10:22 +03:00
# vs2017
msbuild = os.path.join(vspath, "MSBuild", "15.0", "Bin", "MSBuild.exe")
if not os.path.isfile(msbuild):
2019-12-31 03:10:22 +03:00
# vs2019
msbuild = os.path.join(vspath, "MSBuild", "Current", "Bin", "MSBuild.exe")
if not os.path.isfile(msbuild):
2019-12-31 03:10:22 +03:00
print("Visual Studio MSBuild not found")
return None
vcvarsall = os.path.join(vspath, "VC", "Auxiliary", "Build", "vcvarsall.bat")
if not os.path.isfile(vcvarsall):
print("Visual Studio vcvarsall not found")
return None
return {
"vs_dir": vspath,
"msbuild": f'"{msbuild}"',
"vcvarsall": f'"{vcvarsall}"',
"nmake": "nmake.exe", # nmake selected by vcvarsall
}
def download_dep(url: str, file: str) -> None:
2020-09-01 20:16:46 +03:00
import urllib.request
ex = None
for i in range(3):
try:
print(f"Fetching {url} (attempt {i + 1})...")
content = urllib.request.urlopen(url).read()
with open(file, "wb") as f:
f.write(content)
break
except urllib.error.URLError as e:
ex = e
else:
raise RuntimeError(ex)
def extract_dep(url: str, filename: str) -> None:
import tarfile
import zipfile
file = os.path.join(args.depends_dir, filename)
if not os.path.exists(file):
# First try our mirror
mirror_url = (
f"https://raw.githubusercontent.com/"
f"python-pillow/pillow-depends/main/{filename}"
)
try:
download_dep(mirror_url, file)
except RuntimeError as exc:
# Otherwise try upstream
print(exc)
download_dep(url, file)
2020-01-01 03:48:13 +03:00
print("Extracting " + filename)
2022-10-31 02:45:49 +03:00
sources_dir_abs = os.path.abspath(sources_dir)
if filename.endswith(".zip"):
with zipfile.ZipFile(file) as zf:
2022-10-31 02:45:49 +03:00
for member in zf.namelist():
member_abspath = os.path.abspath(os.path.join(sources_dir, member))
member_prefix = os.path.commonpath([sources_dir_abs, member_abspath])
2022-10-31 02:45:49 +03:00
if sources_dir_abs != member_prefix:
msg = "Attempted Path Traversal in Zip File"
raise RuntimeError(msg)
2020-09-01 20:16:46 +03:00
zf.extractall(sources_dir)
elif filename.endswith((".tar.gz", ".tgz")):
with tarfile.open(file, "r:gz") as tgz:
for member in tgz.getnames():
member_abspath = os.path.abspath(os.path.join(sources_dir, member))
member_prefix = os.path.commonpath([sources_dir_abs, member_abspath])
2022-10-31 02:45:49 +03:00
if sources_dir_abs != member_prefix:
msg = "Attempted Path Traversal in Tar File"
raise RuntimeError(msg)
2020-09-01 20:16:46 +03:00
tgz.extractall(sources_dir)
else:
msg = "Unknown archive type: " + filename
raise RuntimeError(msg)
2023-09-18 17:03:18 +03:00
def write_script(name: str, lines: list[str]) -> None:
name = os.path.join(args.build_dir, name)
lines = [line.format(**prefs) for line in lines]
print("Writing " + name)
with open(name, "w", newline="") as f:
f.write(os.linesep.join(lines))
if args.verbose:
2020-02-25 02:00:57 +03:00
for line in lines:
print(" " + line)
2023-09-18 17:03:18 +03:00
def get_footer(dep: dict) -> list[str]:
lines = []
for out in dep.get("headers", []):
lines.append(cmd_copy(out, "{inc_dir}"))
for out in dep.get("libs", []):
lines.append(cmd_copy(out, "{lib_dir}"))
for out in dep.get("bins", []):
lines.append(cmd_copy(out, "{bin_dir}"))
return lines
2023-09-18 17:03:18 +03:00
def build_env() -> None:
lines = [
"if defined DISTUTILS_USE_SDK goto end",
cmd_set("INCLUDE", "{inc_dir}"),
cmd_set("INCLIB", "{lib_dir}"),
cmd_set("LIB", "{lib_dir}"),
cmd_append("PATH", "{bin_dir}"),
2023-06-24 17:57:43 +03:00
"call {vcvarsall} {vcvars_arch}",
cmd_set("DISTUTILS_USE_SDK", "1"), # use same compiler to build Pillow
cmd_set("py_vcruntime_redist", "true"), # always use /MD, never /MT
":end",
"@echo on",
]
write_script("build_env.cmd", lines)
2023-09-18 17:03:18 +03:00
def build_dep(name: str) -> str:
2023-09-18 17:06:30 +03:00
dep = DEPS[name]
2019-12-31 20:13:16 +03:00
dir = dep["dir"]
2020-07-09 22:08:15 +03:00
file = f"build_dep_{name}.cmd"
extract_dep(dep["url"], dep["filename"])
licenses = dep["license"]
if isinstance(licenses, str):
licenses = [licenses]
license_text = ""
for license_file in licenses:
with open(os.path.join(sources_dir, dir, license_file)) as f:
license_text += f.read()
if "license_pattern" in dep:
match = re.search(dep["license_pattern"], license_text, re.DOTALL)
license_text = "\n".join(match.groups())
assert len(license_text) > 50
with open(os.path.join(license_dir, f"{dir}.txt"), "w") as f:
print(f"Writing license {dir}.txt")
f.write(license_text)
2019-12-31 19:34:11 +03:00
for patch_file, patch_list in dep.get("patch", {}).items():
2020-09-01 20:16:46 +03:00
patch_file = os.path.join(sources_dir, dir, patch_file.format(**prefs))
with open(patch_file) as f:
2019-12-31 19:34:11 +03:00
text = f.read()
for patch_from, patch_to in patch_list.items():
2020-09-01 20:16:46 +03:00
patch_from = patch_from.format(**prefs)
patch_to = patch_to.format(**prefs)
assert patch_from in text
text = text.replace(patch_from, patch_to)
2019-12-31 19:34:11 +03:00
with open(patch_file, "w") as f:
2021-09-21 05:49:35 +03:00
print(f"Patching {patch_file}")
2019-12-31 19:34:11 +03:00
f.write(text)
2020-07-09 22:08:15 +03:00
banner = f"Building {name} ({dir})"
2019-12-31 03:10:22 +03:00
lines = [
2023-06-24 17:57:43 +03:00
r'call "{build_dir}\build_env.cmd"',
2020-01-01 03:48:13 +03:00
"@echo " + ("=" * 70),
2020-07-09 22:08:15 +03:00
f"@echo ==== {banner:<60} ====",
2020-01-01 03:48:13 +03:00
"@echo " + ("=" * 70),
cmd_cd(os.path.join(sources_dir, dir)),
2019-12-31 03:10:22 +03:00
*dep.get("build", []),
*get_footer(dep),
]
write_script(file, lines)
return file
2023-09-18 17:03:18 +03:00
def build_dep_all() -> None:
lines = [r'call "{build_dir}\build_env.cmd"']
2023-11-16 02:04:36 +03:00
gha_groups = "GITHUB_ACTIONS" in os.environ
2023-09-18 17:06:30 +03:00
for dep_name in DEPS:
print()
2020-02-25 02:00:57 +03:00
if dep_name in disabled:
print(f"Skipping disabled dependency {dep_name}")
2020-02-25 02:00:57 +03:00
continue
2020-07-09 22:08:15 +03:00
script = build_dep(dep_name)
2023-11-16 02:04:36 +03:00
if gha_groups:
lines.append(f"@echo ::group::Running {script}")
2022-03-04 08:42:24 +03:00
lines.append(rf'cmd.exe /c "{{build_dir}}\{script}"')
2020-01-01 03:48:13 +03:00
lines.append("if errorlevel 1 echo Build failed! && exit /B 1")
2023-11-16 02:04:36 +03:00
if gha_groups:
lines.append("@echo ::endgroup::")
print()
2020-01-01 03:48:13 +03:00
lines.append("@echo All Pillow dependencies built successfully!")
write_script("build_dep_all.cmd", lines)
if __name__ == "__main__":
winbuild_dir = os.path.dirname(os.path.realpath(__file__))
pillow_dir = os.path.realpath(os.path.join(winbuild_dir, ".."))
parser = argparse.ArgumentParser(
prog="winbuild\\build_prepare.py",
description="Download and generate build scripts for Pillow dependencies.",
epilog="""Arguments can also be supplied using the environment variables
PILLOW_BUILD, PILLOW_DEPS, ARCHITECTURE. See winbuild\\build.rst
for more information.""",
)
parser.add_argument(
"-v", "--verbose", action="store_true", help="print generated scripts"
)
parser.add_argument(
"-d",
"--dir",
"--build-dir",
dest="build_dir",
metavar="PILLOW_BUILD",
default=os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")),
help="build directory (default: 'winbuild\\build')",
)
parser.add_argument(
"--depends",
dest="depends_dir",
metavar="PILLOW_DEPS",
default=os.environ.get("PILLOW_DEPS", os.path.join(winbuild_dir, "depends")),
help="directory used to store cached dependencies "
"(default: 'winbuild\\depends')",
)
parser.add_argument(
"--architecture",
2023-09-18 17:06:30 +03:00
choices=ARCHITECTURES,
default=os.environ.get(
"ARCHITECTURE",
2023-06-27 07:43:58 +03:00
(
"ARM64"
if platform.machine() == "ARM64"
else ("x86" if struct.calcsize("P") == 4 else "x64")
),
),
help="build architecture (default: same as host Python)",
)
parser.add_argument(
"--nmake",
dest="cmake_generator",
action="store_const",
const="NMake Makefiles",
default="Ninja",
help="build dependencies using NMake instead of Ninja",
)
parser.add_argument(
"--no-imagequant",
action="store_true",
help="skip GPL-licensed optional dependency libimagequant",
)
parser.add_argument(
"--no-fribidi",
"--no-raqm",
action="store_true",
help="skip LGPL-licensed optional dependency FriBiDi",
)
args = parser.parse_args()
2020-01-01 03:48:13 +03:00
2023-09-18 17:06:30 +03:00
arch_prefs = ARCHITECTURES[args.architecture]
print("Target architecture:", args.architecture)
2020-01-01 15:02:27 +03:00
2023-11-17 00:57:41 +03:00
msvs = find_msvs(args.architecture)
2019-12-31 22:57:17 +03:00
if msvs is None:
msg = "Visual Studio not found. Please install Visual Studio 2017 or newer."
raise RuntimeError(msg)
2020-01-01 03:48:13 +03:00
print("Found Visual Studio at:", msvs["vs_dir"])
# dependency cache directory
args.depends_dir = os.path.abspath(args.depends_dir)
os.makedirs(args.depends_dir, exist_ok=True)
print("Caching dependencies in:", args.depends_dir)
args.build_dir = os.path.abspath(args.build_dir)
print("Using output directory:", args.build_dir)
2020-01-01 03:48:13 +03:00
# build directory for *.h files
inc_dir = os.path.join(args.build_dir, "inc")
2020-01-01 03:48:13 +03:00
# build directory for *.lib files
lib_dir = os.path.join(args.build_dir, "lib")
2020-01-01 03:48:13 +03:00
# build directory for *.bin files
bin_dir = os.path.join(args.build_dir, "bin")
2020-09-01 20:16:46 +03:00
# directory for storing project files
sources_dir = os.path.join(args.build_dir, "src")
# copy dependency licenses to this directory
license_dir = os.path.join(args.build_dir, "license")
shutil.rmtree(args.build_dir, ignore_errors=True)
os.makedirs(args.build_dir, exist_ok=False)
for path in [inc_dir, lib_dir, bin_dir, sources_dir, license_dir]:
2020-09-01 20:16:46 +03:00
os.makedirs(path, exist_ok=True)
disabled = []
if args.no_imagequant:
disabled += ["libimagequant"]
if args.no_fribidi:
disabled += ["fribidi"]
prefs = {
"architecture": args.architecture,
2020-01-01 15:02:27 +03:00
**arch_prefs,
# Pillow paths
"pillow_dir": pillow_dir,
2020-01-01 15:02:27 +03:00
"winbuild_dir": winbuild_dir,
# Build paths
"build_dir": args.build_dir,
"inc_dir": inc_dir,
2020-01-01 15:02:27 +03:00
"lib_dir": lib_dir,
"bin_dir": bin_dir,
2020-09-01 20:16:46 +03:00
"src_dir": sources_dir,
"license_dir": license_dir,
2020-01-01 15:02:27 +03:00
# Compilers / Tools
**msvs,
"cmake": "cmake.exe", # TODO find CMAKE automatically
"cmake_generator": args.cmake_generator,
2020-01-01 15:02:27 +03:00
# TODO find NASM automatically
}
2023-09-18 17:06:30 +03:00
for k, v in DEPS.items():
prefs[f"dir_{k}"] = os.path.join(sources_dir, v["dir"])
2020-02-25 02:00:57 +03:00
print()
2020-09-01 20:16:46 +03:00
write_script(".gitignore", ["*"])
build_env()
build_dep_all()