From 1b6a8c6122ad6b5e5794b04ca37660126675a8a4 Mon Sep 17 00:00:00 2001 From: DWesl <22566757+DWesl@users.noreply.github.com> Date: Sat, 18 Dec 2021 10:00:14 -0500 Subject: [PATCH 01/21] TST: Parametrize numpy roundtrip to find failing case Segfaults are annoying to debug. --- Tests/test_numpy.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Tests/test_numpy.py b/Tests/test_numpy.py index def7adf3f..936474fe8 100644 --- a/Tests/test_numpy.py +++ b/Tests/test_numpy.py @@ -189,8 +189,9 @@ def test_putdata(): assert len(im.getdata()) == len(arr) -def test_roundtrip_eye(): - for dtype in ( +@pytest.mark.parametrize( + "dtype", + ( bool, numpy.bool8, numpy.int8, @@ -202,9 +203,11 @@ def test_roundtrip_eye(): float, numpy.float32, numpy.float64, - ): - arr = numpy.eye(10, dtype=dtype) - numpy.testing.assert_array_equal(arr, numpy.array(Image.fromarray(arr))) + ), +) +def test_roundtrip_eye(dtype): + arr = numpy.eye(10, dtype=dtype) + numpy.testing.assert_array_equal(arr, numpy.array(Image.fromarray(arr))) def test_zero_size(): From 7ab973c4c95494cae655825cc36c5ba9abbfabd2 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 30 Jan 2022 08:01:35 +1100 Subject: [PATCH 02/21] Updated lcms2 to 2.13 --- docs/installation.rst | 2 +- winbuild/build_prepare.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index ea0e78500..88aaf5834 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -169,7 +169,7 @@ Many of Pillow's features require external libraries: * **littlecms** provides color management * Pillow version 2.2.1 and below uses liblcms1, Pillow 2.3.0 and - above uses liblcms2. Tested with **1.19** and **2.7-2.12**. + above uses liblcms2. Tested with **1.19** and **2.7-2.13**. * **libwebp** provides the WebP format. diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 3092c5a2b..78916987c 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -219,9 +219,9 @@ deps = { # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], }, "lcms2": { - "url": SF_MIRROR + "/project/lcms/lcms/2.12/lcms2-2.12.tar.gz", - "filename": "lcms2-2.12.tar.gz", - "dir": "lcms2-2.12", + "url": SF_MIRROR + "/project/lcms/lcms/2.12/lcms2-2.13.tar.gz", + "filename": "lcms2-2.13.tar.gz", + "dir": "lcms2-2.13", "patch": { r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always From d8e94c206ef69989642e48aa2795a2a69f9d0577 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 30 Jan 2022 14:40:30 +1100 Subject: [PATCH 03/21] Switched from windows-2019 to windows-latest --- .github/workflows/test-mingw.yml | 2 +- .github/workflows/test-windows.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mingw.yml b/.github/workflows/test-mingw.yml index d94c7d537..8a9c1725d 100644 --- a/.github/workflows/test-mingw.yml +++ b/.github/workflows/test-mingw.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2019 + runs-on: windows-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index c768838eb..c78f9fd24 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2019 + runs-on: windows-latest strategy: fail-fast: false matrix: From 36f293071e76c9e0206708cf01087055f1dc0912 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 31 Jan 2022 08:05:01 +1100 Subject: [PATCH 04/21] Updated raqm to 0.9.0 --- depends/install_raqm.sh | 2 +- src/thirdparty/raqm/COPYING | 2 +- src/thirdparty/raqm/README.md | 5 +- src/thirdparty/raqm/raqm-version.h | 4 +- src/thirdparty/raqm/raqm.c | 472 ++++++++++++++++++----------- src/thirdparty/raqm/raqm.h | 11 +- 6 files changed, 311 insertions(+), 185 deletions(-) diff --git a/depends/install_raqm.sh b/depends/install_raqm.sh index 39b998d05..992503650 100755 --- a/depends/install_raqm.sh +++ b/depends/install_raqm.sh @@ -2,7 +2,7 @@ # install raqm -archive=libraqm-0.8.0 +archive=libraqm-0.9.0 ./download-and-extract.sh $archive https://raw.githubusercontent.com/python-pillow/pillow-depends/main/$archive.tar.gz diff --git a/src/thirdparty/raqm/COPYING b/src/thirdparty/raqm/COPYING index 78db981bb..c605a5dc6 100644 --- a/src/thirdparty/raqm/COPYING +++ b/src/thirdparty/raqm/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) Copyright © 2015 Information Technology Authority (ITA) -Copyright © 2016-2021 Khaled Hosny +Copyright © 2016-2022 Khaled Hosny Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/thirdparty/raqm/README.md b/src/thirdparty/raqm/README.md index 17ce2efc9..02e996e7a 100644 --- a/src/thirdparty/raqm/README.md +++ b/src/thirdparty/raqm/README.md @@ -68,6 +68,7 @@ Projects using Raqm 3. [FontView](https://github.com/googlei18n/fontview) 4. [Pillow](https://github.com/python-pillow) 5. [mplcairo](https://github.com/anntzer/mplcairo) +6. [CEGUI](https://github.com/cegui/cegui) The following projects have patches to support complex text layout using Raqm: @@ -77,8 +78,8 @@ The following projects have patches to support complex text layout using Raqm: -[1]: http://fribidi.org +[1]: https://github.com/fribidi/fribidi [2]: https://github.com/Tehreer/SheenBidi -[3]: http://harfbuzz.org +[3]: https://github.com/harfbuzz/harfbuzz [4]: https://www.freetype.org [5]: https://www.gtk.org/gtk-doc diff --git a/src/thirdparty/raqm/raqm-version.h b/src/thirdparty/raqm/raqm-version.h index e96b1aea5..78b70a561 100644 --- a/src/thirdparty/raqm/raqm-version.h +++ b/src/thirdparty/raqm/raqm-version.h @@ -32,10 +32,10 @@ #define _RAQM_VERSION_H_ #define RAQM_VERSION_MAJOR 0 -#define RAQM_VERSION_MINOR 8 +#define RAQM_VERSION_MINOR 9 #define RAQM_VERSION_MICRO 0 -#define RAQM_VERSION_STRING "0.8.0" +#define RAQM_VERSION_STRING "0.9.0" #define RAQM_VERSION_ATLEAST(major,minor,micro) \ ((major)*10000+(minor)*100+(micro) <= \ diff --git a/src/thirdparty/raqm/raqm.c b/src/thirdparty/raqm/raqm.c index 46890b9a7..f852542b4 100644 --- a/src/thirdparty/raqm/raqm.c +++ b/src/thirdparty/raqm/raqm.c @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016-2021 Khaled Hosny + * Copyright © 2016-2022 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -24,7 +24,6 @@ #ifdef HAVE_CONFIG_H #include "config.h" -#undef HAVE_CONFIG_H // Workaround for Fribidi 1.0.5 and earlier #endif #include @@ -38,9 +37,6 @@ #else #include "../fribidi-shim/fribidi.h" #endif -#if FRIBIDI_MAJOR_VERSION >= 1 -#define USE_FRIBIDI_EX_API -#endif #endif #include @@ -190,13 +186,9 @@ typedef FriBidiLevel _raqm_bidi_level_t; #endif -typedef enum { - RAQM_FLAG_NONE = 0, - RAQM_FLAG_UTF8 = 1 << 0 -} _raqm_flags_t; - typedef struct { FT_Face ftface; + int ftloadflags; hb_language_t lang; hb_script_t script; } _raqm_text_info; @@ -209,6 +201,7 @@ struct _raqm { uint32_t *text; char *text_utf8; size_t text_len; + size_t text_capacity_bytes; _raqm_text_info *text_info; @@ -219,17 +212,17 @@ struct _raqm { size_t features_len; raqm_run_t *runs; + raqm_run_t *runs_pool; + raqm_glyph_t *glyphs; + size_t glyphs_capacity; - _raqm_flags_t flags; - - int ft_loadflags; int invisible_glyph; }; struct _raqm_run { - int pos; - int len; + uint32_t pos; + uint32_t len; hb_direction_t direction; hb_script_t script; @@ -243,31 +236,21 @@ static uint32_t _raqm_u8_to_u32_index (raqm_t *rq, uint32_t index); -static bool +static void _raqm_init_text_info (raqm_t *rq) { - hb_language_t default_lang; - - if (rq->text_info) - return true; - - rq->text_info = malloc (sizeof (_raqm_text_info) * rq->text_len); - if (!rq->text_info) - return false; - - default_lang = hb_language_get_default (); + hb_language_t default_lang = hb_language_get_default (); for (size_t i = 0; i < rq->text_len; i++) { rq->text_info[i].ftface = NULL; + rq->text_info[i].ftloadflags = -1; rq->text_info[i].lang = default_lang; rq->text_info[i].script = HB_SCRIPT_INVALID; } - - return true; } static void -_raqm_free_text_info (raqm_t *rq) +_raqm_release_text_info (raqm_t *rq) { if (!rq->text_info) return; @@ -277,9 +260,6 @@ _raqm_free_text_info (raqm_t *rq) if (rq->text_info[i].ftface) FT_Done_Face (rq->text_info[i].ftface); } - - free (rq->text_info); - rq->text_info = NULL; } static bool @@ -289,6 +269,9 @@ _raqm_compare_text_info (_raqm_text_info a, if (a.ftface != b.ftface) return false; + if (a.ftloadflags != b.ftloadflags) + return false; + if (a.lang != b.lang) return false; @@ -298,6 +281,88 @@ _raqm_compare_text_info (_raqm_text_info a, return true; } +static void +_raqm_free_text(raqm_t* rq) +{ + free (rq->text); + rq->text = NULL; + rq->text_info = NULL; + rq->text_utf8 = NULL; + rq->text_len = 0; + rq->text_capacity_bytes = 0; +} + +static bool +_raqm_alloc_text(raqm_t *rq, + size_t len, + bool need_utf8) +{ + /* Allocate contiguous memory block for texts and text_info */ + size_t mem_size = (sizeof (uint32_t) + sizeof (_raqm_text_info)) * len; + if (need_utf8) + mem_size += sizeof (char) * len; + + if (mem_size > rq->text_capacity_bytes) + { + void* new_mem = realloc (rq->text, mem_size); + if (!new_mem) + { + _raqm_free_text (rq); + return false; + } + + rq->text_capacity_bytes = mem_size; + rq->text = new_mem; + } + + rq->text_info = (_raqm_text_info*)(rq->text + len); + rq->text_utf8 = need_utf8 ? (char*)(rq->text_info + len) : NULL; + + return true; +} + +static raqm_run_t* +_raqm_alloc_run (raqm_t *rq) +{ + raqm_run_t *run = rq->runs_pool; + if (run) + { + rq->runs_pool = run->next; + } + else + { + run = malloc (sizeof (raqm_run_t)); + run->font = NULL; + run->buffer = NULL; + } + + run->pos = 0; + run->len = 0; + run->direction = HB_DIRECTION_INVALID; + run->script = HB_SCRIPT_INVALID; + run->next = NULL; + + return run; +} + +static void +_raqm_free_runs (raqm_run_t *runs) +{ + while (runs) + { + raqm_run_t *run = runs; + runs = runs->next; + + if (run->buffer) + hb_buffer_destroy (run->buffer); + + if (run->font) + hb_font_destroy (run->font); + + free (run); + } +} + /** * raqm_create: * @@ -322,26 +387,26 @@ raqm_create (void) rq->ref_count = 1; - rq->text = NULL; - rq->text_utf8 = NULL; - rq->text_len = 0; - - rq->text_info = NULL; - rq->base_dir = RAQM_DIRECTION_DEFAULT; rq->resolved_dir = RAQM_DIRECTION_DEFAULT; rq->features = NULL; rq->features_len = 0; - rq->runs = NULL; - rq->glyphs = NULL; - - rq->flags = RAQM_FLAG_NONE; - - rq->ft_loadflags = -1; rq->invisible_glyph = 0; + rq->text = NULL; + rq->text_utf8 = NULL; + rq->text_info = NULL; + rq->text_capacity_bytes = 0; + rq->text_len = 0; + + rq->runs = NULL; + rq->runs_pool = NULL; + + rq->glyphs = NULL; + rq->glyphs_capacity = 0; + return rq; } @@ -366,28 +431,13 @@ raqm_reference (raqm_t *rq) return rq; } -static void -_raqm_free_runs (raqm_t *rq) -{ - raqm_run_t *runs = rq->runs; - while (runs) - { - raqm_run_t *run = runs; - runs = runs->next; - - hb_buffer_destroy (run->buffer); - hb_font_destroy (run->font); - free (run); - } -} - /** * raqm_destroy: * @rq: a #raqm_t. * * Decreases the reference count on @rq by one. If the result is zero, then @rq * and all associated resources are freed. - * See cairo_reference(). + * See raqm_reference(). * * Since: 0.1 */ @@ -397,14 +447,60 @@ raqm_destroy (raqm_t *rq) if (!rq || --rq->ref_count != 0) return; - free (rq->text); - free (rq->text_utf8); - _raqm_free_text_info (rq); - _raqm_free_runs (rq); + _raqm_release_text_info (rq); + _raqm_free_text (rq); + _raqm_free_runs (rq->runs); + _raqm_free_runs (rq->runs_pool); free (rq->glyphs); + free (rq->features); free (rq); } +/** + * raqm_clear_contents: + * @rq: a #raqm_t. + * + * Clears internal state of previously used raqm_t object, making it ready + * for reuse and keeping some of allocated memory to increase performance. + * + * Since: 0.9 + */ +void +raqm_clear_contents (raqm_t *rq) +{ + if (!rq) + return; + + _raqm_release_text_info (rq); + + /* Return allocated runs to the pool, keep hb buffers for reuse */ + raqm_run_t *run = rq->runs; + while (run) + { + if (run->buffer) + hb_buffer_reset (run->buffer); + + if (run->font) + { + hb_font_destroy (run->font); + run->font = NULL; + } + + if (!run->next) + { + run->next = rq->runs_pool; + rq->runs_pool = rq->runs; + rq->runs = NULL; + break; + } + + run = run->next; + } + + rq->text_len = 0; + rq->resolved_dir = RAQM_DIRECTION_DEFAULT; +} + /** * raqm_set_text: * @rq: a #raqm_t. @@ -429,23 +525,20 @@ raqm_set_text (raqm_t *rq, if (!rq || !text) return false; - rq->text_len = len; + /* Call raqm_clear_contents to reuse this raqm_t */ + if (rq->text_len) + return false; /* Empty string, don’t fail but do nothing */ if (!len) return true; - free (rq->text); + if (!_raqm_alloc_text(rq, len, false)) + return false; - rq->text = malloc (sizeof (uint32_t) * rq->text_len); - if (!rq->text) - return false; - - _raqm_free_text_info (rq); - if (!_raqm_init_text_info (rq)) - return false; - - memcpy (rq->text, text, sizeof (uint32_t) * rq->text_len); + rq->text_len = len; + memcpy (rq->text, text, sizeof (uint32_t) * len); + _raqm_init_text_info (rq); return true; } @@ -511,41 +604,29 @@ _raqm_u8_to_u32 (const char *text, size_t len, uint32_t *unicode) * Since: 0.1 */ bool -raqm_set_text_utf8 (raqm_t *rq, - const char *text, - size_t len) +raqm_set_text_utf8 (raqm_t *rq, + const char *text, + size_t len) { - uint32_t *unicode; - size_t ulen; - bool ok; - if (!rq || !text) return false; + /* Call raqm_clear_contents to reuse this raqm_t */ + if (rq->text_len) + return false; + /* Empty string, don’t fail but do nothing */ if (!len) - { - rq->text_len = len; return true; - } - rq->flags |= RAQM_FLAG_UTF8; - - rq->text_utf8 = malloc (sizeof (char) * len); - if (!rq->text_utf8) - return false; - - unicode = malloc (sizeof (uint32_t) * len); - if (!unicode) - return false; + if (!_raqm_alloc_text(rq, len, true)) + return false; + rq->text_len = _raqm_u8_to_u32 (text, len, rq->text); memcpy (rq->text_utf8, text, sizeof (char) * len); + _raqm_init_text_info (rq); - ulen = _raqm_u8_to_u32 (text, len, unicode); - ok = raqm_set_text (rq, unicode, ulen); - - free (unicode); - return ok; + return true; } /** @@ -561,7 +642,7 @@ raqm_set_text_utf8 (raqm_t *rq, * * The default is #RAQM_DIRECTION_DEFAULT, which determines the paragraph * direction based on the first character with strong bidi type (see [rule - * P2](http://unicode.org/reports/tr9/#P2) in Unicode Bidirectional Algorithm), + * P2](https://unicode.org/reports/tr9/#P2) in Unicode Bidirectional Algorithm), * which can be good enough for many cases but has problems when a mainly * right-to-left paragraph starts with a left-to-right character and vice versa * as the detected paragraph direction will be the wrong one, or when text does @@ -629,7 +710,7 @@ raqm_set_language (raqm_t *rq, if (!rq->text_len) return true; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { start = _raqm_u8_to_u32_index (rq, start); end = _raqm_u8_to_u32_index (rq, end); @@ -686,13 +767,14 @@ raqm_add_font_feature (raqm_t *rq, ok = hb_feature_from_string (feature, len, &fea); if (ok) { - rq->features_len++; - rq->features = realloc (rq->features, - sizeof (hb_feature_t) * (rq->features_len)); - if (!rq->features) + void* new_features = realloc (rq->features, + sizeof (hb_feature_t) * (rq->features_len + 1)); + if (!new_features) return false; - rq->features[rq->features_len - 1] = fea; + rq->features = new_features; + rq->features[rq->features_len] = fea; + rq->features_len++; } return ok; @@ -700,12 +782,13 @@ raqm_add_font_feature (raqm_t *rq, static hb_font_t * _raqm_create_hb_font (raqm_t *rq, - FT_Face face) + FT_Face face, + int loadflags) { hb_font_t *font = hb_ft_font_create_referenced (face); - if (rq->ft_loadflags >= 0) - hb_ft_font_set_load_flags (font, rq->ft_loadflags); + if (loadflags >= 0) + hb_ft_font_set_load_flags (font, loadflags); return font; } @@ -796,7 +879,7 @@ raqm_set_freetype_face_range (raqm_t *rq, if (!rq->text_len) return true; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { start = _raqm_u8_to_u32_index (rq, start); end = _raqm_u8_to_u32_index (rq, end); @@ -805,6 +888,30 @@ raqm_set_freetype_face_range (raqm_t *rq, return _raqm_set_freetype_face (rq, face, start, end); } +static bool +_raqm_set_freetype_load_flags (raqm_t *rq, + int flags, + size_t start, + size_t end) +{ + if (!rq) + return false; + + if (!rq->text_len) + return true; + + if (start >= rq->text_len || end > rq->text_len) + return false; + + if (!rq->text_info) + return false; + + for (size_t i = start; i < end; i++) + rq->text_info[i].ftloadflags = flags; + + return true; +} + /** * raqm_set_freetype_load_flags: * @rq: a #raqm_t. @@ -823,14 +930,59 @@ raqm_set_freetype_face_range (raqm_t *rq, */ bool raqm_set_freetype_load_flags (raqm_t *rq, - int flags) + int flags) { + return _raqm_set_freetype_load_flags(rq, flags, 0, rq->text_len); +} + +/** + * raqm_set_freetype_load_flags_range: + * @rq: a #raqm_t. + * @flags: FreeType load flags. + * @start: index of first character that should use @flags. + * @len: number of characters using @flags. + * + * Sets the load flags passed to FreeType when loading glyphs for @len-number + * of characters staring at @start. Flags should be the same as used by the + * client when rendering corresponding FreeType glyphs. The @start and @len + * are input string array indices (i.e. counting bytes in UTF-8 and scaler + * values in UTF-32). + * + * This method can be used repeatedly to set different flags for different + * parts of the text. It is the responsibility of the client to make sure that + * flag ranges cover the whole text. + * + * This requires version of HarfBuzz that has hb_ft_font_set_load_flags(), for + * older version the flags will be ignored. + * + * See also raqm_set_freetype_load_flags(). + * + * Return value: + * %true if no errors happened, %false otherwise. + * + * Since: 0.9 + */ +bool +raqm_set_freetype_load_flags_range (raqm_t *rq, + int flags, + size_t start, + size_t len) +{ + size_t end = start + len; + if (!rq) return false; - rq->ft_loadflags = flags; + if (!rq->text_len) + return true; - return true; + if (rq->text_utf8) + { + start = _raqm_u8_to_u32_index (rq, start); + end = _raqm_u8_to_u32_index (rq, end); + } + + return _raqm_set_freetype_load_flags (rq, flags, start, end); } /** @@ -841,17 +993,10 @@ raqm_set_freetype_load_flags (raqm_t *rq, * Sets the glyph id to be used for invisible glyhphs. * * If @gid is negative, invisible glyphs will be suppressed from the output. - * This requires HarfBuzz 1.8.0 or later. If raqm is used with an earlier - * HarfBuzz version, the return value will be %false and the shaping behavior - * does not change. * * If @gid is zero, invisible glyphs will be rendered as space. - * This works on all versions of HarfBuzz. * * If @gid is a positive number, it will be used for invisible glyphs. - * This requires a version of HarfBuzz that has - * hb_buffer_set_invisible_glyph(). For older versions, the return value - * will be %false and the shaping behavior does not change. * * Return value: * %true if no errors happened, %false otherwise. @@ -865,17 +1010,6 @@ raqm_set_invisible_glyph (raqm_t *rq, if (!rq) return false; -#ifndef HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH - if (gid > 0) - return false; -#endif - -#if !defined(HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES) || \ - !HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES - if (gid < 0) - return false; -#endif - rq->invisible_glyph = gid; return true; } @@ -961,18 +1095,21 @@ raqm_get_glyphs (raqm_t *rq, for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) count += hb_buffer_get_length (run->buffer); - *length = count; - - if (rq->glyphs) - free (rq->glyphs); - - rq->glyphs = malloc (sizeof (raqm_glyph_t) * count); - if (!rq->glyphs) + if (count > rq->glyphs_capacity) { - *length = 0; - return NULL; + void* new_mem = realloc (rq->glyphs, sizeof (raqm_glyph_t) * count); + if (!new_mem) + { + *length = 0; + return NULL; + } + + rq->glyphs = new_mem; + rq->glyphs_capacity = count; } + *length = count; + RAQM_TEST ("Glyph information:\n"); count = 0; @@ -1005,7 +1142,7 @@ raqm_get_glyphs (raqm_t *rq, count += len; } - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { #ifdef RAQM_TESTING RAQM_TEST ("\nUTF-32 clusters:"); @@ -1276,24 +1413,14 @@ _raqm_bidi_itemize (raqm_t *rq, size_t *run_count) FriBidiCharType *types; _raqm_bidi_level_t *levels; int max_level = 0; -#ifdef USE_FRIBIDI_EX_API FriBidiBracketType *btypes; -#endif types = calloc (rq->text_len, sizeof (FriBidiCharType)); -#ifdef USE_FRIBIDI_EX_API btypes = calloc (rq->text_len, sizeof (FriBidiBracketType)); -#endif levels = calloc (rq->text_len, sizeof (_raqm_bidi_level_t)); - if (!types || !levels -#ifdef USE_FRIBIDI_EX_API - || !btypes -#endif - ) - { + if (!types || !levels || !btypes) goto done; - } if (rq->base_dir == RAQM_DIRECTION_RTL) par_type = FRIBIDI_PAR_RTL; @@ -1301,15 +1428,10 @@ _raqm_bidi_itemize (raqm_t *rq, size_t *run_count) par_type = FRIBIDI_PAR_LTR; fribidi_get_bidi_types (rq->text, rq->text_len, types); -#ifdef USE_FRIBIDI_EX_API fribidi_get_bracket_types (rq->text, rq->text_len, types, btypes); max_level = fribidi_get_par_embedding_levels_ex (types, btypes, rq->text_len, &par_type, levels); -#else - max_level = fribidi_get_par_embedding_levels (types, rq->text_len, - &par_type, levels); -#endif if (par_type == FRIBIDI_PAR_LTR) rq->resolved_dir = RAQM_DIRECTION_LTR; @@ -1325,9 +1447,7 @@ _raqm_bidi_itemize (raqm_t *rq, size_t *run_count) done: free (types); free (levels); -#ifdef USE_FRIBIDI_EX_API free (btypes); -#endif return runs; } @@ -1403,7 +1523,7 @@ _raqm_itemize (raqm_t *rq) last = NULL; for (size_t i = 0; i < run_count; i++) { - raqm_run_t *run = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *run = _raqm_alloc_run (rq); if (!run) { ok = false; @@ -1422,13 +1542,14 @@ _raqm_itemize (raqm_t *rq) { run->pos = runs[i].pos + runs[i].len - 1; run->script = rq->text_info[run->pos].script; - run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface); + run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface, + rq->text_info[run->pos].ftloadflags); for (int j = runs[i].len - 1; j >= 0; j--) { _raqm_text_info info = rq->text_info[runs[i].pos + j]; if (!_raqm_compare_text_info (rq->text_info[run->pos], info)) { - raqm_run_t *newrun = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *newrun = _raqm_alloc_run (rq); if (!newrun) { ok = false; @@ -1438,7 +1559,8 @@ _raqm_itemize (raqm_t *rq) newrun->len = 1; newrun->direction = _raqm_hb_dir (rq, runs[i].level); newrun->script = info.script; - newrun->font = _raqm_create_hb_font (rq, info.ftface); + newrun->font = _raqm_create_hb_font (rq, info.ftface, + info.ftloadflags); run->next = newrun; run = newrun; } @@ -1453,13 +1575,14 @@ _raqm_itemize (raqm_t *rq) { run->pos = runs[i].pos; run->script = rq->text_info[run->pos].script; - run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface); + run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface, + rq->text_info[run->pos].ftloadflags); for (size_t j = 0; j < runs[i].len; j++) { _raqm_text_info info = rq->text_info[runs[i].pos + j]; if (!_raqm_compare_text_info (rq->text_info[run->pos], info)) { - raqm_run_t *newrun = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *newrun = _raqm_alloc_run (rq); if (!newrun) { ok = false; @@ -1469,7 +1592,8 @@ _raqm_itemize (raqm_t *rq) newrun->len = 1; newrun->direction = _raqm_hb_dir (rq, runs[i].level); newrun->script = info.script; - newrun->font = _raqm_create_hb_font (rq, info.ftface); + newrun->font = _raqm_create_hb_font (rq, info.ftface, + info.ftloadflags); run->next = newrun; run = newrun; } @@ -1758,7 +1882,6 @@ _raqm_resolve_scripts (raqm_t *rq) return true; } -#ifdef HAVE_FT_GET_TRANSFORM static void _raqm_ft_transform (int *x, int *y, @@ -1773,22 +1896,19 @@ _raqm_ft_transform (int *x, *x = vector.x; *y = vector.y; } -#endif static bool _raqm_shape (raqm_t *rq) { hb_buffer_flags_t hb_buffer_flags = HB_BUFFER_FLAG_BOT | HB_BUFFER_FLAG_EOT; -#if defined(HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES) && \ - HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES if (rq->invisible_glyph < 0) hb_buffer_flags |= HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES; -#endif for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) { - run->buffer = hb_buffer_create (); + if (!run->buffer) + run->buffer = hb_buffer_create (); hb_buffer_add_utf32 (run->buffer, rq->text, rq->text_len, run->pos, run->len); @@ -1797,15 +1917,12 @@ _raqm_shape (raqm_t *rq) hb_buffer_set_direction (run->buffer, run->direction); hb_buffer_set_flags (run->buffer, hb_buffer_flags); -#ifdef HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH if (rq->invisible_glyph > 0) hb_buffer_set_invisible_glyph (run->buffer, rq->invisible_glyph); -#endif hb_shape_full (run->font, run->buffer, rq->features, rq->features_len, NULL); -#ifdef HAVE_FT_GET_TRANSFORM { FT_Matrix matrix; hb_glyph_position_t *pos; @@ -1819,7 +1936,6 @@ _raqm_shape (raqm_t *rq) _raqm_ft_transform (&pos[i].x_offset, &pos[i].y_offset, matrix); } } -#endif } return true; @@ -1917,7 +2033,7 @@ raqm_index_to_position (raqm_t *rq, if (rq == NULL) return false; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) *index = _raqm_u8_to_u32_index (rq, *index); if (*index >= rq->text_len) @@ -1974,7 +2090,7 @@ raqm_index_to_position (raqm_t *rq, } found: - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) *index = _raqm_u32_to_u8_index (rq, *index); RAQM_TEST ("The position is %d at index %zu\n",*x ,*index); return true; diff --git a/src/thirdparty/raqm/raqm.h b/src/thirdparty/raqm/raqm.h index faab85591..bdb5a50d8 100644 --- a/src/thirdparty/raqm/raqm.h +++ b/src/thirdparty/raqm/raqm.h @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016-2021 Khaled Hosny + * Copyright © 2016-2022 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -106,6 +106,9 @@ raqm_reference (raqm_t *rq); RAQM_API void raqm_destroy (raqm_t *rq); +RAQM_API void +raqm_clear_contents (raqm_t *rq); + RAQM_API bool raqm_set_text (raqm_t *rq, const uint32_t *text, @@ -145,6 +148,12 @@ RAQM_API bool raqm_set_freetype_load_flags (raqm_t *rq, int flags); +RAQM_API bool +raqm_set_freetype_load_flags_range (raqm_t *rq, + int flags, + size_t start, + size_t len); + RAQM_API bool raqm_set_invisible_glyph (raqm_t *rq, int gid); From 999ef3e4eaf793612d9f3525fcbff2bac629e135 Mon Sep 17 00:00:00 2001 From: nulano Date: Mon, 31 Jan 2022 00:11:47 +0000 Subject: [PATCH 05/21] Revert "detect FreeType / HarfBuzz features" This reverts commit 6565d13275cead21dc6f369204f0dc3d0b43bc18. --- src/thirdparty/raqm/raqm.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/thirdparty/raqm/raqm.c b/src/thirdparty/raqm/raqm.c index f852542b4..13f6e1f02 100644 --- a/src/thirdparty/raqm/raqm.c +++ b/src/thirdparty/raqm/raqm.c @@ -42,21 +42,6 @@ #include #include -#if FREETYPE_MAJOR > 2 || \ - FREETYPE_MAJOR == 2 && FREETYPE_MINOR >= 11 -#define HAVE_FT_GET_TRANSFORM -#endif - -#if HB_VERSION_ATLEAST(2, 0, 0) -#define HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH -#endif - -#if HB_VERSION_ATLEAST(1, 8, 0) -#define HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES 1 -#else -#define HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES 0 -#endif - #include "raqm.h" /** From 3ac2da533a1e74f4a1bb38e8e812190c49fbdc53 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 18:56:52 +0200 Subject: [PATCH 06/21] Fix lcms2 URL --- winbuild/build_prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 78916987c..182146fbc 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -219,7 +219,7 @@ deps = { # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], }, "lcms2": { - "url": SF_MIRROR + "/project/lcms/lcms/2.12/lcms2-2.13.tar.gz", + "url": SF_MIRROR + "/project/lcms/lcms/2.13/lcms2-2.13.tar.gz", "filename": "lcms2-2.13.tar.gz", "dir": "lcms2-2.13", "patch": { From cc4f9a2a7dcb3e69dd4ca4ab752ebcd0fba0bbb2 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 20:38:47 +0200 Subject: [PATCH 07/21] Update harfbuzz to 3.3.0 --- winbuild/build_prepare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 3092c5a2b..e6bcbc6fc 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -278,9 +278,9 @@ deps = { "libs": [r"imagequant.lib"], }, "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/3.2.0.zip", - "filename": "harfbuzz-3.2.0.zip", - "dir": "harfbuzz-3.2.0", + "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.0.zip", + "filename": "harfbuzz-3.3.0.zip", + "dir": "harfbuzz-3.3.0", "build": [ cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), cmd_nmake(target="clean"), From 87ab18f804ed860f4315038980bcf20044ae352c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 23:27:35 +0200 Subject: [PATCH 08/21] Update harfbuzz to 3.3.1 --- winbuild/build_prepare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index e6bcbc6fc..220956adc 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -278,9 +278,9 @@ deps = { "libs": [r"imagequant.lib"], }, "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.0.zip", - "filename": "harfbuzz-3.3.0.zip", - "dir": "harfbuzz-3.3.0", + "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.1.zip", + "filename": "harfbuzz-3.3.1.zip", + "dir": "harfbuzz-3.3.1", "build": [ cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), cmd_nmake(target="clean"), From 54e9decde30915e1f89ed7cda0bc49870707a02e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 23:41:16 +0200 Subject: [PATCH 09/21] Remove EOL CentOS 8 --- .github/workflows/test-docker.yml | 1 - docs/installation.rst | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index df04d0a6c..656df5e91 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -19,7 +19,6 @@ jobs: amazon-2-amd64, arch, centos-7-amd64, - centos-8-amd64, centos-stream-8-amd64, debian-10-buster-x86, debian-11-bullseye-x86, diff --git a/docs/installation.rst b/docs/installation.rst index 88aaf5834..984a689c2 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -453,8 +453,6 @@ These platforms are built and tested for every change. +----------------------------------+----------------------------+---------------------+ | CentOS 7 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ -| CentOS 8 | 3.9 | x86-64 | -+----------------------------------+----------------------------+---------------------+ | CentOS Stream 8 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ | Debian 10 Buster | 3.7 | x86 | @@ -530,6 +528,8 @@ These platforms have been reported to work at the versions mentioned. +----------------------------------+---------------------------+------------------+--------------+ | CentOS 6.3 | 2.7, 3.3 | |x86 | +----------------------------------+---------------------------+------------------+--------------+ +| CentOS 8 | 3.9 | 9.0.0 |x86-64 | ++----------------------------------+---------------------------+------------------+--------------+ | Fedora 23 | 2.7, 3.4 | 3.1.0 |x86-64 | +----------------------------------+---------------------------+------------------+--------------+ | Ubuntu Linux 12.04 LTS (Precise) | | 2.6, 3.2, 3.3, 3.4, 3.5 | 3.4.1 |x86,x86-64 | From f22d3561fd764e9600fe4640856c50ad74a90eaf Mon Sep 17 00:00:00 2001 From: Gabor Kertesz Date: Fri, 29 Oct 2021 12:31:40 +0200 Subject: [PATCH 10/21] Windows: Enable ARM64 for MSVC This patch enables ARM64 as a new platform for Windows. Platform query and documentation is updated accordingly. --- winbuild/build.rst | 6 +++--- winbuild/build_prepare.py | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/winbuild/build.rst b/winbuild/build.rst index b30a94226..661c5a5ec 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -24,7 +24,7 @@ Download and install: * `CMake 3.12 or newer `_ (also available as Visual Studio component C++ CMake tools for Windows) -* `NASM `_ +* x86/x64: `NASM `_ Any version of Visual Studio 2017 or newer should be supported, including Visual Studio 2017 Community, or Build Tools for Visual Studio 2019. @@ -42,8 +42,8 @@ behaviour of ``build_prepare.py``: If ``PYTHON`` is unset, the version of Python used to run ``build_prepare.py`` will be used. If only ``PYTHON`` is set, ``EXECUTABLE`` defaults to ``python.exe``. -* ``ARCHITECTURE`` is used to select a ``x86`` or ``x64`` build. By default, - uses same architecture as the version of Python used to run ``build_prepare.py``. +* ``ARCHITECTURE`` is used to select a ``x86``, ``x64`` or ``ARM64``build. + By default, uses same architecture as the version of Python used to run ``build_prepare.py``. is used. * ``PILLOW_BUILD`` can be used to override the ``winbuild\build`` directory path, used to store generated build scripts and compiled libraries. diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 4d9f79211..bebafff9f 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -1,4 +1,5 @@ import os +import platform import shutil import struct import subprocess @@ -93,6 +94,7 @@ SF_MIRROR = "http://iweb.dl.sourceforge.net" architectures = { "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, + "ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"}, } header = [ @@ -490,7 +492,10 @@ if __name__ == "__main__": python_dir = os.environ.get("PYTHON") python_exe = os.environ.get("EXECUTABLE", "python.exe") architecture = os.environ.get( - "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64" + "ARCHITECTURE", + "ARM64" + if platform.machine() == "ARM64" + else ("x86" if struct.calcsize("P") == 4 else "x64"), ) build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")) sources_dir = "" From 22a1f6db5e097783885be99970ae45496fc296e8 Mon Sep 17 00:00:00 2001 From: Gabor Kertesz Date: Fri, 29 Oct 2021 12:20:56 +0200 Subject: [PATCH 11/21] lcms2: Update to VS2019 In order to enable win-arm64, VS2019 should be used, while other platforms should work with newer version as well. Tested on x64-win10. --- winbuild/build_prepare.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index bebafff9f..01f1bac29 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -225,21 +225,21 @@ deps = { "filename": "lcms2-2.13.tar.gz", "dir": "lcms2-2.13", "patch": { - r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { + r"Projects\VC2019\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always "MultiThreaded": "MultiThreadedDLL", # noqa: E501 # retarget to default toolset (selected by vcvarsall.bat) - "v141": "$(DefaultPlatformToolset)", # noqa: E501 + "v142": "$(DefaultPlatformToolset)", # noqa: E501 # retarget to latest (selected by vcvarsall.bat) - "10.0.17134.0": "$(WindowsSDKVersion)", # noqa: E501 + "10.0": "$(WindowsSDKVersion)", # noqa: E501 } }, "build": [ cmd_rmdir("Lib"), - cmd_rmdir(r"Projects\VC2017\Release"), - cmd_msbuild(r"Projects\VC2017\lcms2.sln", "Release", "Clean"), + cmd_rmdir(r"Projects\VC2019\Release"), + cmd_msbuild(r"Projects\VC2019\lcms2.sln", "Release", "Clean"), cmd_msbuild( - r"Projects\VC2017\lcms2.sln", "Release", "lcms2_static:Rebuild" + r"Projects\VC2019\lcms2.sln", "Release", "lcms2_static:Rebuild" ), cmd_xcopy("include", "{inc_dir}"), ], From fb7edfda68a8972ec487eab845506fc08f25cae8 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 2 Feb 2022 11:49:31 +1100 Subject: [PATCH 12/21] Improved consistency of returning an image access object from load() --- Tests/test_file_eps.py | 9 +++++++++ Tests/test_file_gbr.py | 22 +++++++++++++++------- Tests/test_file_icns.py | 8 ++++++++ Tests/test_file_ico.py | 5 +++++ Tests/test_file_wal.py | 16 ++++++++++------ Tests/test_file_wmf.py | 6 ++++++ src/PIL/EpsImagePlugin.py | 12 ++++++------ src/PIL/GbrImagePlugin.py | 10 ++++------ src/PIL/IcnsImagePlugin.py | 9 +++++---- src/PIL/IcoImagePlugin.py | 2 +- src/PIL/ImageFile.py | 1 + src/PIL/WalImageFile.py | 13 +++++-------- src/PIL/WmfImagePlugin.py | 2 +- 13 files changed, 76 insertions(+), 39 deletions(-) diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 4c0b96f73..1790f4f77 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -58,6 +58,15 @@ def test_sanity(): assert image2_scale2.format == "EPS" +@pytest.mark.skipif(not HAS_GHOSTSCRIPT, reason="Ghostscript not available") +def test_load(): + with Image.open(FILE1) as im: + assert im.load()[0, 0] == (255, 255, 255) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (255, 255, 255) + + def test_invalid_file(): invalid_file = "Tests/images/flower.jpg" diff --git a/Tests/test_file_gbr.py b/Tests/test_file_gbr.py index 8d7fcf147..1ea8af8ee 100644 --- a/Tests/test_file_gbr.py +++ b/Tests/test_file_gbr.py @@ -5,20 +5,28 @@ from PIL import GbrImagePlugin, Image from .helper import assert_image_equal_tofile -def test_invalid_file(): - invalid_file = "Tests/images/flower.jpg" - - with pytest.raises(SyntaxError): - GbrImagePlugin.GbrImageFile(invalid_file) - - def test_gbr_file(): with Image.open("Tests/images/gbr.gbr") as im: assert_image_equal_tofile(im, "Tests/images/gbr.png") +def test_load(): + with Image.open("Tests/images/gbr.gbr") as im: + assert im.load()[0, 0] == (0, 0, 0, 0) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (0, 0, 0, 0) + + def test_multiple_load_operations(): with Image.open("Tests/images/gbr.gbr") as im: im.load() im.load() assert_image_equal_tofile(im, "Tests/images/gbr.png") + + +def test_invalid_file(): + invalid_file = "Tests/images/flower.jpg" + + with pytest.raises(SyntaxError): + GbrImagePlugin.GbrImageFile(invalid_file) diff --git a/Tests/test_file_icns.py b/Tests/test_file_icns.py index 3afbbeaac..b492f6cb2 100644 --- a/Tests/test_file_icns.py +++ b/Tests/test_file_icns.py @@ -28,6 +28,14 @@ def test_sanity(): assert im.format == "ICNS" +def test_load(): + with Image.open(TEST_FILE) as im: + assert im.load()[0, 0] == (0, 0, 0, 0) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (0, 0, 0, 0) + + def test_save(tmp_path): temp_file = str(tmp_path / "temp.icns") diff --git a/Tests/test_file_ico.py b/Tests/test_file_ico.py index 317264db6..73ac6f742 100644 --- a/Tests/test_file_ico.py +++ b/Tests/test_file_ico.py @@ -18,6 +18,11 @@ def test_sanity(): assert im.get_format_mimetype() == "image/x-icon" +def test_load(): + with Image.open(TEST_ICO_FILE) as im: + assert im.load()[0, 0] == (1, 1, 9, 255) + + def test_mask(): with Image.open("Tests/images/hopper_mask.ico") as im: assert_image_equal_tofile(im, "Tests/images/hopper_mask.png") diff --git a/Tests/test_file_wal.py b/Tests/test_file_wal.py index f25b42fe0..4be46e9d6 100644 --- a/Tests/test_file_wal.py +++ b/Tests/test_file_wal.py @@ -2,15 +2,11 @@ from PIL import WalImageFile from .helper import assert_image_equal_tofile +TEST_FILE = "Tests/images/hopper.wal" + def test_open(): - # Arrange - TEST_FILE = "Tests/images/hopper.wal" - - # Act with WalImageFile.open(TEST_FILE) as im: - - # Assert assert im.format == "WAL" assert im.format_description == "Quake2 Texture" assert im.mode == "P" @@ -19,3 +15,11 @@ def test_open(): assert isinstance(im, WalImageFile.WalImageFile) assert_image_equal_tofile(im, "Tests/images/hopper_wal.png") + + +def test_load(): + with WalImageFile.open(TEST_FILE) as im: + assert im.load()[0, 0] == 122 + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == 122 diff --git a/Tests/test_file_wmf.py b/Tests/test_file_wmf.py index 3f8bc96cc..d6769a24b 100644 --- a/Tests/test_file_wmf.py +++ b/Tests/test_file_wmf.py @@ -24,6 +24,12 @@ def test_load_raw(): assert_image_similar_tofile(im, "Tests/images/drawing_wmf_ref.png", 2.0) +def test_load(): + with Image.open("Tests/images/drawing.emf") as im: + if hasattr(Image.core, "drawwmf"): + assert im.load()[0, 0] == (255, 255, 255) + + def test_register_handler(tmp_path): class TestHandler: methodCalled = False diff --git a/src/PIL/EpsImagePlugin.py b/src/PIL/EpsImagePlugin.py index 5d9920280..b67363beb 100644 --- a/src/PIL/EpsImagePlugin.py +++ b/src/PIL/EpsImagePlugin.py @@ -329,12 +329,12 @@ class EpsImageFile(ImageFile.ImageFile): def load(self, scale=1, transparency=False): # Load EPS via Ghostscript - if not self.tile: - return - self.im = Ghostscript(self.tile, self.size, self.fp, scale, transparency) - self.mode = self.im.mode - self._size = self.im.size - self.tile = [] + if self.tile: + self.im = Ghostscript(self.tile, self.size, self.fp, scale, transparency) + self.mode = self.im.mode + self._size = self.im.size + self.tile = [] + return Image.Image.load(self) def load_seek(self, *args, **kwargs): # we can't incrementally load, so force ImageFile.parser to diff --git a/src/PIL/GbrImagePlugin.py b/src/PIL/GbrImagePlugin.py index 0f230602d..3d8fc47b2 100644 --- a/src/PIL/GbrImagePlugin.py +++ b/src/PIL/GbrImagePlugin.py @@ -84,12 +84,10 @@ class GbrImageFile(ImageFile.ImageFile): self._data_size = width * height * color_depth def load(self): - if self.im: - # Already loaded - return - - self.im = Image.core.new(self.mode, self.size) - self.frombytes(self.fp.read(self._data_size)) + if not self.im: + self.im = Image.core.new(self.mode, self.size) + self.frombytes(self.fp.read(self._data_size)) + return Image.Image.load(self) # diff --git a/src/PIL/IcnsImagePlugin.py b/src/PIL/IcnsImagePlugin.py index 6412d1cfb..069aff96b 100644 --- a/src/PIL/IcnsImagePlugin.py +++ b/src/PIL/IcnsImagePlugin.py @@ -286,21 +286,22 @@ class IcnsImageFile(ImageFile.ImageFile): self.best_size[1] * self.best_size[2], ) - Image.Image.load(self) + px = Image.Image.load(self) if self.im and self.im.size == self.size: # Already loaded - return + return px self.load_prepare() # This is likely NOT the best way to do it, but whatever. im = self.icns.getimage(self.best_size) # If this is a PNG or JPEG 2000, it won't be loaded yet - im.load() + px = im.load() self.im = im.im self.mode = im.mode self.size = im.size - self.load_end() + + return px def _save(im, fp, filename): diff --git a/src/PIL/IcoImagePlugin.py b/src/PIL/IcoImagePlugin.py index d9ff9b5e7..82837f307 100644 --- a/src/PIL/IcoImagePlugin.py +++ b/src/PIL/IcoImagePlugin.py @@ -306,7 +306,7 @@ class IcoImageFile(ImageFile.ImageFile): def load(self): if self.im and self.im.size == self.size: # Already loaded - return + return Image.Image.load(self) im = self.ico.getimage(self.size) # if tile is PNG, it won't really be loaded yet im.load() diff --git a/src/PIL/ImageFile.py b/src/PIL/ImageFile.py index 3374a5b1d..331410f0e 100644 --- a/src/PIL/ImageFile.py +++ b/src/PIL/ImageFile.py @@ -328,6 +328,7 @@ class StubImageFile(ImageFile): # become the other object (!) self.__class__ = image.__class__ self.__dict__ = image.__dict__ + return image.load() def _load(self): """(Hook) Find actual image loader.""" diff --git a/src/PIL/WalImageFile.py b/src/PIL/WalImageFile.py index 1354ad32b..0dc695a88 100644 --- a/src/PIL/WalImageFile.py +++ b/src/PIL/WalImageFile.py @@ -51,14 +51,11 @@ class WalImageFile(ImageFile.ImageFile): self.info["next_name"] = next_name def load(self): - if self.im: - # Already loaded - return - - self.im = Image.core.new(self.mode, self.size) - self.frombytes(self.fp.read(self.size[0] * self.size[1])) - self.putpalette(quake2palette) - Image.Image.load(self) + if not self.im: + self.im = Image.core.new(self.mode, self.size) + self.frombytes(self.fp.read(self.size[0] * self.size[1])) + self.putpalette(quake2palette) + return Image.Image.load(self) def open(filename): diff --git a/src/PIL/WmfImagePlugin.py b/src/PIL/WmfImagePlugin.py index 27f5d2f87..c32cc52f8 100644 --- a/src/PIL/WmfImagePlugin.py +++ b/src/PIL/WmfImagePlugin.py @@ -158,7 +158,7 @@ class WmfStubImageFile(ImageFile.StubImageFile): (x1 - x0) * self.info["dpi"] // self._inch, (y1 - y0) * self.info["dpi"] // self._inch, ) - super().load() + return super().load() def _save(im, fp, filename): From 8e3878dec2b300f5b495dd81989a77089747abee Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 2 Feb 2022 14:33:19 +1100 Subject: [PATCH 13/21] Update CHANGES.rst [ci skip] --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index d6cd9d50f..66d417393 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,12 @@ Changelog (Pillow) 9.1.0 (unreleased) ------------------ +- Enable arm64 for MSVC on Windows #5811 + [gaborkertesz-linaro, gaborkertesz] + +- Keep IPython/Jupyter text/plain output stable #5891 + [shamrin, radarhere] + - Raise an error when performing a negative crop #5972 [radarhere, hugovk] From dd46100bdc7fbb6c2fb71008a49c40f081eb0c7c Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 10 Jan 2022 21:49:55 +1100 Subject: [PATCH 14/21] Restrict builtins within lambdas for ImageMath.eval --- Tests/test_imagemath.py | 12 ++++++++++-- src/PIL/ImageMath.py | 15 +++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Tests/test_imagemath.py b/Tests/test_imagemath.py index 25811aa89..39d91eade 100644 --- a/Tests/test_imagemath.py +++ b/Tests/test_imagemath.py @@ -52,9 +52,17 @@ def test_ops(): assert pixel(ImageMath.eval("float(B)**33", images)) == "F 8589934592.0" -def test_prevent_exec(): +@pytest.mark.parametrize( + "expression", + ( + "exec('pass')", + "(lambda: exec('pass'))()", + "(lambda: (lambda: exec('pass'))())()", + ), +) +def test_prevent_exec(expression): with pytest.raises(ValueError): - ImageMath.eval("exec('pass')") + ImageMath.eval(expression) def test_logical(): diff --git a/src/PIL/ImageMath.py b/src/PIL/ImageMath.py index 4b6e4ccda..09d9898d7 100644 --- a/src/PIL/ImageMath.py +++ b/src/PIL/ImageMath.py @@ -240,11 +240,18 @@ def eval(expression, _dict={}, **kw): if hasattr(v, "im"): args[k] = _Operand(v) - code = compile(expression, "", "eval") - for name in code.co_names: - if name not in args and name != "abs": - raise ValueError(f"'{name}' not allowed") + compiled_code = compile(expression, "", "eval") + def scan(code): + for const in code.co_consts: + if type(const) == type(compiled_code): + scan(const) + + for name in code.co_names: + if name not in args and name != "abs": + raise ValueError(f"'{name}' not allowed") + + scan(compiled_code) out = builtins.eval(expression, {"__builtins": {"abs": abs}}, args) try: return out.im From 8da80130dbc747f3954b4904247d26289fe722f9 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 17 Jan 2022 08:59:17 +1100 Subject: [PATCH 15/21] In show_file, use os.remove to remove temporary images --- Tests/test_imageshow.py | 6 +- src/PIL/ImageShow.py | 145 +++++++++++++++++++++++++++++++--------- 2 files changed, 119 insertions(+), 32 deletions(-) diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index 02edfdfa1..bf19a6033 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -85,11 +85,13 @@ def test_ipythonviewer(): not on_ci() or is_win32(), reason="Only run on CIs; hangs on Windows CIs", ) -def test_file_deprecated(): +def test_file_deprecated(tmp_path): + f = str(tmp_path / "temp.jpg") for viewer in ImageShow._viewers: + hopper().save(f) with pytest.warns(DeprecationWarning): try: - viewer.show_file(file="test.jpg") + viewer.show_file(file=f) except NotImplementedError: pass with pytest.raises(TypeError): diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index 2165da307..7212baa1c 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -15,7 +15,6 @@ import os import shutil import subprocess import sys -import tempfile import warnings from shlex import quote @@ -180,16 +179,15 @@ class MacViewer(Viewer): path = options.pop("file") else: raise TypeError("Missing required argument: 'path'") - fd, temp_path = tempfile.mkstemp() - with os.fdopen(fd, "w") as f: - f.write(path) - with open(temp_path) as f: - subprocess.Popen( - ["im=$(cat); open -a Preview.app $im; sleep 20; rm -f $im"], - shell=True, - stdin=f, - ) - os.remove(temp_path) + subprocess.call(["open", "-a", "Preview.app", path]) + subprocess.Popen( + [ + sys.executable, + "-c", + "import os, sys, time;time.sleep(20);os.remove(sys.argv[1])", + path, + ] + ) return 1 @@ -205,6 +203,16 @@ class UnixViewer(Viewer): command = self.get_command_ex(file, **options)[0] return f"({command} {quote(file)}; rm -f {quote(file)})&" + +class XDGViewer(UnixViewer): + """ + The freedesktop.org ``xdg-open`` command. + """ + + def get_command_ex(self, file, **options): + command = executable = "xdg-open" + return command, executable + def show_file(self, path=None, **options): """ Display given file. @@ -223,28 +231,11 @@ class UnixViewer(Viewer): path = options.pop("file") else: raise TypeError("Missing required argument: 'path'") - fd, temp_path = tempfile.mkstemp() - with os.fdopen(fd, "w") as f: - f.write(path) - with open(temp_path) as f: - command = self.get_command_ex(path, **options)[0] - subprocess.Popen( - ["im=$(cat);" + command + " $im; rm -f $im"], shell=True, stdin=f - ) - os.remove(temp_path) + subprocess.Popen(["xdg-open", path]) + os.remove(path) return 1 -class XDGViewer(UnixViewer): - """ - The freedesktop.org ``xdg-open`` command. - """ - - def get_command_ex(self, file, **options): - command = executable = "xdg-open" - return command, executable - - class DisplayViewer(UnixViewer): """ The ImageMagick ``display`` command. @@ -257,6 +248,32 @@ class DisplayViewer(UnixViewer): command += f" -name {quote(title)}" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + args = ["display"] + if "title" in options: + args += ["-name", options["title"]] + args.append(path) + + subprocess.Popen(args) + os.remove(path) + return 1 + class GmDisplayViewer(UnixViewer): """The GraphicsMagick ``gm display`` command.""" @@ -266,6 +283,27 @@ class GmDisplayViewer(UnixViewer): command = "gm display" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + subprocess.Popen(["gm", "display", path]) + os.remove(path) + return 1 + class EogViewer(UnixViewer): """The GNOME Image Viewer ``eog`` command.""" @@ -275,6 +313,27 @@ class EogViewer(UnixViewer): command = "eog -n" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + subprocess.Popen(["eog", "-n", path]) + os.remove(path) + return 1 + class XVViewer(UnixViewer): """ @@ -290,6 +349,32 @@ class XVViewer(UnixViewer): command += f" -name {quote(title)}" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + args = ["xv"] + if "title" in options: + args += ["-name", options["title"]] + args.append(path) + + subprocess.Popen(args) + os.remove(path) + return 1 + if sys.platform not in ("win32", "darwin"): # unixoids if shutil.which("xdg-open"): From 143032103c9f2d55a0a7960bd3e630cb72549e8a Mon Sep 17 00:00:00 2001 From: Andrew Murray <3112309+radarhere@users.noreply.github.com> Date: Tue, 18 Jan 2022 11:24:01 +1100 Subject: [PATCH 16/21] Updated formatting Co-authored-by: Hugo van Kemenade --- src/PIL/ImageShow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index 7212baa1c..ccdb0b2a0 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -184,7 +184,7 @@ class MacViewer(Viewer): [ sys.executable, "-c", - "import os, sys, time;time.sleep(20);os.remove(sys.argv[1])", + "import os, sys, time; time.sleep(20); os.remove(sys.argv[1])", path, ] ) From 10c4f75aaa383bd9671e923e3b91d391ea12d781 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 08:58:12 +1100 Subject: [PATCH 17/21] Added delay after opening image with xdg-open --- src/PIL/ImageShow.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index ccdb0b2a0..f8829fc21 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -126,6 +126,16 @@ class Viewer: os.system(self.get_command(path, **options)) return 1 + def _remove_path_after_delay(self, path): + subprocess.Popen( + [ + sys.executable, + "-c", + "import os, sys, time; time.sleep(20); os.remove(sys.argv[1])", + path, + ] + ) + # -------------------------------------------------------------------- @@ -180,14 +190,7 @@ class MacViewer(Viewer): else: raise TypeError("Missing required argument: 'path'") subprocess.call(["open", "-a", "Preview.app", path]) - subprocess.Popen( - [ - sys.executable, - "-c", - "import os, sys, time; time.sleep(20); os.remove(sys.argv[1])", - path, - ] - ) + self._remove_path_after_delay(path) return 1 @@ -232,7 +235,7 @@ class XDGViewer(UnixViewer): else: raise TypeError("Missing required argument: 'path'") subprocess.Popen(["xdg-open", path]) - os.remove(path) + self._remove_path_after_delay(path) return 1 From 596eaf35cc983fe73e2408e3ca7f3e1431fd06e3 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 09:46:57 +1100 Subject: [PATCH 18/21] Update CHANGES.rst [ci skip] --- CHANGES.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 66d417393..fc9455652 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -32,6 +32,15 @@ Changelog (Pillow) - Remove readonly from Image.__eq__ #5930 [hugovk] +9.0.1 (2022-02-03) +------------------ + +- In show_file, use os.remove to remove temporary images. CVE-2022-24303 #6010 + [radarhere, hugovk] + +- Restrict builtins within lambdas for ImageMath.eval. CVE-2022-22817 #6009 + [radarhere] + 9.0.0 (2022-01-02) ------------------ From 8ef2d987ab652346126cf7e567eac7d2c0754940 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 09:48:56 +1100 Subject: [PATCH 19/21] Added release notes for 9.0.1 --- docs/releasenotes/9.0.1.rst | 23 +++++++++++++++++++++++ docs/releasenotes/index.rst | 1 + 2 files changed, 24 insertions(+) create mode 100644 docs/releasenotes/9.0.1.rst diff --git a/docs/releasenotes/9.0.1.rst b/docs/releasenotes/9.0.1.rst new file mode 100644 index 000000000..5d1b246bc --- /dev/null +++ b/docs/releasenotes/9.0.1.rst @@ -0,0 +1,23 @@ +9.0.1 +----- + +Security +======== + +This release addresses several security problems. + +:cve:`CVE-2022-24303`: If the path to the temporary directory on Linux or macOS +contained a space, this would break removal of the temporary image file after +``im.show()`` (and related actions), and potentially remove an unrelated file. This +been present since PIL. + +:cve:`CVE-2022-22817`: While Pillow 9.0 restricted top-level builtins available to +:py:meth:`PIL.ImageMath.eval`, it did not prevent builtins available to lambda +expressions. These are now also restricted. + +Other Changes +============= + +Pillow 9.0 added support for ``xdg-open`` as an image viewer, but there have been +reports that the temporary image file was removed too quickly to be loaded into the +final application. A delay has been added. diff --git a/docs/releasenotes/index.rst b/docs/releasenotes/index.rst index 8d1ad7837..e9b11c220 100644 --- a/docs/releasenotes/index.rst +++ b/docs/releasenotes/index.rst @@ -14,6 +14,7 @@ expected to be backported to earlier versions. .. toctree:: :maxdepth: 2 + 9.0.1 9.0.0 8.4.0 8.3.2 From 9d8f173e39458bb73107f90813c745717d87ea78 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 18:05:50 +1100 Subject: [PATCH 20/21] Updated libimagequant to 4.0.0 --- depends/install_imagequant.sh | 11 ++++++----- docs/installation.rst | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/depends/install_imagequant.sh b/depends/install_imagequant.sh index 774f26767..31fc2adaa 100755 --- a/depends/install_imagequant.sh +++ b/depends/install_imagequant.sh @@ -1,14 +1,15 @@ #!/bin/bash # install libimagequant -archive=libimagequant-2.17.0 +archive=libimagequant-4.0.0 ./download-and-extract.sh $archive https://raw.githubusercontent.com/python-pillow/pillow-depends/main/$archive.tar.gz -pushd $archive +pushd $archive/imagequant-sys -make shared -sudo cp libimagequant.so* /usr/lib/ -sudo cp libimagequant.h /usr/include/ +cargo install cargo-c +cargo cinstall --prefix=/usr --destdir=. +sudo cp usr/lib/libimagequant.so* /usr/lib/ +sudo cp usr/include/libimagequant.h /usr/include/ popd diff --git a/docs/installation.rst b/docs/installation.rst index 984a689c2..a6ed8f681 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -187,7 +187,7 @@ Many of Pillow's features require external libraries: * **libimagequant** provides improved color quantization - * Pillow has been tested with libimagequant **2.6-2.17.0** + * Pillow has been tested with libimagequant **2.6-4.0** * Libimagequant is licensed GPLv3, which is more restrictive than the Pillow license, therefore we will not be distributing binaries with libimagequant support enabled. From caf6fc60cac8ed59b8b2c361df1e9705be3d0504 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Fri, 4 Feb 2022 03:19:49 +1100 Subject: [PATCH 21/21] Corrected sentence [ci skip] --- docs/releasenotes/9.0.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releasenotes/9.0.1.rst b/docs/releasenotes/9.0.1.rst index 5d1b246bc..c1feee088 100644 --- a/docs/releasenotes/9.0.1.rst +++ b/docs/releasenotes/9.0.1.rst @@ -9,7 +9,7 @@ This release addresses several security problems. :cve:`CVE-2022-24303`: If the path to the temporary directory on Linux or macOS contained a space, this would break removal of the temporary image file after ``im.show()`` (and related actions), and potentially remove an unrelated file. This -been present since PIL. +has been present since PIL. :cve:`CVE-2022-22817`: While Pillow 9.0 restricted top-level builtins available to :py:meth:`PIL.ImageMath.eval`, it did not prevent builtins available to lambda