From af52060e973063b259708a8e91bfbbf13376c247 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 10 Apr 2025 20:45:53 +1000 Subject: [PATCH 01/49] Mention that tobytes() with the raw encoder uses Pack.c --- src/PIL/Image.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 88ea6f3b5..b419405fb 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -767,18 +767,20 @@ class Image: .. warning:: - This method returns the raw image data from the internal - storage. For compressed image data (e.g. PNG, JPEG) use - :meth:`~.save`, with a BytesIO parameter for in-memory - data. + This method returns raw image data derived from Pillow's internal + storage. For compressed image data (e.g. PNG, JPEG) use + :meth:`~.save`, with a BytesIO parameter for in-memory data. - :param encoder_name: What encoder to use. The default is to - use the standard "raw" encoder. + :param encoder_name: What encoder to use. - A list of C encoders can be seen under - codecs section of the function array in - :file:`_imaging.c`. Python encoders are - registered within the relevant plugins. + The default is to use the standard "raw" encoder. + To see how this packs pixel data into the returned + bytes, see :file:`libImaging/Pack.c`. + + A list of C encoders can be seen under codecs + section of the function array in + :file:`_imaging.c`. Python encoders are registered + within the relevant plugins. :param args: Extra arguments to the encoder. :returns: A :py:class:`bytes` object. """ From 3d77723a0c9d245f61e124c58a11e3a1779b3c0d Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 17 Apr 2025 21:42:42 +0100 Subject: [PATCH 02/49] Added arrow support for a flat array of 4*uint8 for image32 modes --- Tests/test_pyarrow.py | 66 +++++++++++++++++++++++++++++++++++++--- src/libImaging/Storage.c | 14 +++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index ece9f8f26..e7f2bc5f9 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -18,18 +18,25 @@ TEST_IMAGE_SIZE = (10, 10) def _test_img_equals_pyarray( - img: Image.Image, arr: Any, mask: list[int] | None + img: Image.Image, arr: Any, mask: list[int] | None, elts_per_pixel: int = 1 ) -> None: - assert img.height * img.width == len(arr) + assert img.height * img.width * elts_per_pixel == len(arr) px = img.load() assert px is not None + if elts_per_pixel > 1 and mask is None: + # have to do element wise comparison when we're comparing + # flattened r,g,b,a to a pixel. + mask = list(range(elts_per_pixel)) for x in range(0, img.size[0], int(img.size[0] / 10)): for y in range(0, img.size[1], int(img.size[1] / 10)): if mask: + pixel = px[x, y] + assert isinstance(pixel, tuple) for ix, elt in enumerate(mask): - pixel = px[x, y] - assert isinstance(pixel, tuple) - assert pixel[ix] == arr[y * img.width + x].as_py()[elt] + if elts_per_pixel == 1: + assert pixel[ix] == arr[y * img.width + x].as_py()[elt] + else: + assert pixel[ix] == arr[(y * img.width + x) * elts_per_pixel + elt].as_py() else: assert_deep_equal(px[x, y], arr[y * img.width + x].as_py()) @@ -110,3 +117,52 @@ def test_lifetime2() -> None: px = img2.load() assert px # make mypy happy assert isinstance(px[0, 0], int) + + +UINT_ARR = ( + fl_uint8_4_type, + [1,2,3,4], + 1 +) +UINT = ( + pyarrow.uint8(), + 3, + 4 +) + + + +@pytest.mark.parametrize( + "mode, data_tp, mask", + ( + ("L", (pyarrow.uint8(), 3, 1), None), + ("I", (pyarrow.int32(), 1<<24, 1), None), + ("F", (pyarrow.float32(), 3.14159, 1), None), + ("LA", UINT_ARR, [0, 3]), + ("LA", UINT, [0, 3]), + ("RGB", UINT_ARR, [0, 1, 2]), + ("RGBA", UINT_ARR, None), + ("RGBA", UINT_ARR, None), + ("CMYK", UINT_ARR, None), + ("YCbCr", UINT_ARR, [0, 1, 2]), + ("HSV", UINT_ARR, [0, 1, 2]), + ("RGB", UINT, [0, 1, 2]), + ("RGBA", UINT, None), + ("RGBA", UINT, None), + ("CMYK", UINT, None), + ("YCbCr", UINT, [0, 1, 2]), + ("HSV", UINT, [0, 1, 2]), + ), +) +def test_fromarray(mode: str, + data_tp: tuple, + mask:list[int] | None) -> None: + (dtype, + elt, + elts_per_pixel) = data_tp + + ct_pixels = TEST_IMAGE_SIZE[0] * TEST_IMAGE_SIZE[1] + arr = pyarrow.array([elt]*(ct_pixels*elts_per_pixel), type=dtype) + img = Image.fromarrow(arr, mode, TEST_IMAGE_SIZE) + + _test_img_equals_pyarray(img, arr, mask, elts_per_pixel) diff --git a/src/libImaging/Storage.c b/src/libImaging/Storage.c index 4fa4ecd1c..7f8d9c4a0 100644 --- a/src/libImaging/Storage.c +++ b/src/libImaging/Storage.c @@ -723,6 +723,8 @@ ImagingNewArrow( int64_t pixels = (int64_t)xsize * (int64_t)ysize; // fmt:off // don't reformat this + // stored as a single array, one element per pixel, either single band + // or multiband, where each pixel is an I32. if (((strcmp(schema->format, "I") == 0 // int32 && im->pixelsize == 4 // 4xchar* storage && im->bands >= 2) // INT32 into any INT32 Storage mode @@ -735,6 +737,7 @@ ImagingNewArrow( return im; } } + // Stored as [[r,g,b,a],....] if (strcmp(schema->format, "+w:4") == 0 // 4 up array && im->pixelsize == 4 // storage as 32 bpc && schema->n_children > 0 // make sure schema is well formed. @@ -750,6 +753,17 @@ ImagingNewArrow( return im; } } + // Stored as [r,g,b,a,r,g,b,a....] + if (strcmp(schema->format, "C") == 0 // uint8 + && im->pixelsize == 4 // storage as 32 bpc + && schema->n_children == 0 // make sure schema is well formed. + && strcmp(im->arrow_band_format, "C") == 0 // Expected Format + && 4* pixels == external_array->length) { // expected length + // single flat array, interleaved storage. + if (ImagingBorrowArrow(im, external_array, 1, array_capsule)) { + return im; + } + } // fmt: on ImagingDelete(im); return NULL; From c729d4e2085b96662b89ad09f99327f4516ce4ed Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 17 Apr 2025 22:16:27 +0100 Subject: [PATCH 03/49] Test uint32 array creation -> image32 images --- Tests/test_pyarrow.py | 61 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index e7f2bc5f9..92bc4c807 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -9,6 +9,7 @@ from PIL import Image from .helper import ( assert_deep_equal, assert_image_equal, + is_big_endian, hopper, ) @@ -41,6 +42,34 @@ def _test_img_equals_pyarray( assert_deep_equal(px[x, y], arr[y * img.width + x].as_py()) +def _test_img_equals_int32_pyarray( + img: Image.Image, arr: Any, mask: list[int] | None, elts_per_pixel: int = 1 +) -> None: + assert img.height * img.width * elts_per_pixel == len(arr) + px = img.load() + assert px is not None + if mask is None: + # have to do element wise comparison when we're comparing + # flattened rgba in an uint32 to a pixel. + mask = list(range(elts_per_pixel)) + for x in range(0, img.size[0], int(img.size[0] / 10)): + for y in range(0, img.size[1], int(img.size[1] / 10)): + pixel = px[x, y] + assert isinstance(pixel, tuple) + arr_pixel_int = arr[y * img.width + x].as_py() + arr_pixel_tuple = ( + arr_pixel_int % 256, + (arr_pixel_int // 256) % 256, + (arr_pixel_int // 256**2) % 256, + (arr_pixel_int // 256**3), + ) + if is_big_endian(): + arr_pixel_tuple = arr_pixel_tuple[::-1] + + for ix, elt in enumerate(mask): + assert pixel[ix] == arr_pixel_tuple[elt] + + # really hard to get a non-nullable list type fl_uint8_4_type = pyarrow.field( "_", pyarrow.list_(pyarrow.field("_", pyarrow.uint8()).with_nullable(False), 4) @@ -129,7 +158,11 @@ UINT = ( 3, 4 ) - +INT32 = ( + pyarrow.uint32(), + 0xabcdef45, + 1 +) @pytest.mark.parametrize( @@ -166,3 +199,29 @@ def test_fromarray(mode: str, img = Image.fromarrow(arr, mode, TEST_IMAGE_SIZE) _test_img_equals_pyarray(img, arr, mask, elts_per_pixel) + + +@pytest.mark.parametrize( + "mode, data_tp, mask", + ( + ("LA", INT32, [0, 3]), + ("RGB", INT32, [0, 1, 2]), + ("RGBA", INT32, None), + ("RGBA", INT32, None), + ("CMYK", INT32, None), + ("YCbCr", INT32, [0, 1, 2]), + ("HSV", INT32, [0, 1, 2]), + ), +) +def test_from_int32array(mode: str, + data_tp: tuple, + mask:list[int] | None) -> None: + (dtype, + elt, + elts_per_pixel) = data_tp + + ct_pixels = TEST_IMAGE_SIZE[0] * TEST_IMAGE_SIZE[1] + arr = pyarrow.array([elt]*(ct_pixels*elts_per_pixel), type=dtype) + img = Image.fromarrow(arr, mode, TEST_IMAGE_SIZE) + + _test_img_equals_int32_pyarray(img, arr, mask, elts_per_pixel) From ac500460dfc6ddaa7c0660de3f0233d05e207852 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 17 Apr 2025 22:22:31 +0100 Subject: [PATCH 04/49] lint --- Tests/test_pyarrow.py | 44 ++++++++++++++++------------------------ src/libImaging/Storage.c | 10 ++++----- 2 files changed, 23 insertions(+), 31 deletions(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index 92bc4c807..bcdd7ddc9 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -9,8 +9,8 @@ from PIL import Image from .helper import ( assert_deep_equal, assert_image_equal, - is_big_endian, hopper, + is_big_endian, ) pyarrow = pytest.importorskip("pyarrow", reason="PyArrow not installed") @@ -37,7 +37,10 @@ def _test_img_equals_pyarray( if elts_per_pixel == 1: assert pixel[ix] == arr[y * img.width + x].as_py()[elt] else: - assert pixel[ix] == arr[(y * img.width + x) * elts_per_pixel + elt].as_py() + assert ( + pixel[ix] + == arr[(y * img.width + x) * elts_per_pixel + elt].as_py() + ) else: assert_deep_equal(px[x, y], arr[y * img.width + x].as_py()) @@ -169,33 +172,27 @@ INT32 = ( "mode, data_tp, mask", ( ("L", (pyarrow.uint8(), 3, 1), None), - ("I", (pyarrow.int32(), 1<<24, 1), None), + ("I", (pyarrow.int32(), 1 << 24, 1), None), ("F", (pyarrow.float32(), 3.14159, 1), None), ("LA", UINT_ARR, [0, 3]), ("LA", UINT, [0, 3]), ("RGB", UINT_ARR, [0, 1, 2]), ("RGBA", UINT_ARR, None), - ("RGBA", UINT_ARR, None), ("CMYK", UINT_ARR, None), - ("YCbCr", UINT_ARR, [0, 1, 2]), - ("HSV", UINT_ARR, [0, 1, 2]), + ("YCbCr", UINT_ARR, [0, 1, 2]), + ("HSV", UINT_ARR, [0, 1, 2]), ("RGB", UINT, [0, 1, 2]), ("RGBA", UINT, None), - ("RGBA", UINT, None), ("CMYK", UINT, None), - ("YCbCr", UINT, [0, 1, 2]), - ("HSV", UINT, [0, 1, 2]), + ("YCbCr", UINT, [0, 1, 2]), + ("HSV", UINT, [0, 1, 2]), ), ) -def test_fromarray(mode: str, - data_tp: tuple, - mask:list[int] | None) -> None: - (dtype, - elt, - elts_per_pixel) = data_tp +def test_fromarray(mode: str, data_tp: tuple, mask: list[int] | None) -> None: + (dtype, elt, elts_per_pixel) = data_tp ct_pixels = TEST_IMAGE_SIZE[0] * TEST_IMAGE_SIZE[1] - arr = pyarrow.array([elt]*(ct_pixels*elts_per_pixel), type=dtype) + arr = pyarrow.array([elt] * (ct_pixels * elts_per_pixel), type=dtype) img = Image.fromarrow(arr, mode, TEST_IMAGE_SIZE) _test_img_equals_pyarray(img, arr, mask, elts_per_pixel) @@ -207,21 +204,16 @@ def test_fromarray(mode: str, ("LA", INT32, [0, 3]), ("RGB", INT32, [0, 1, 2]), ("RGBA", INT32, None), - ("RGBA", INT32, None), ("CMYK", INT32, None), - ("YCbCr", INT32, [0, 1, 2]), - ("HSV", INT32, [0, 1, 2]), + ("YCbCr", INT32, [0, 1, 2]), + ("HSV", INT32, [0, 1, 2]), ), ) -def test_from_int32array(mode: str, - data_tp: tuple, - mask:list[int] | None) -> None: - (dtype, - elt, - elts_per_pixel) = data_tp +def test_from_int32array(mode: str, data_tp: tuple, mask: list[int] | None) -> None: + (dtype, elt, elts_per_pixel) = data_tp ct_pixels = TEST_IMAGE_SIZE[0] * TEST_IMAGE_SIZE[1] - arr = pyarrow.array([elt]*(ct_pixels*elts_per_pixel), type=dtype) + arr = pyarrow.array([elt] * (ct_pixels * elts_per_pixel), type=dtype) img = Image.fromarrow(arr, mode, TEST_IMAGE_SIZE) _test_img_equals_int32_pyarray(img, arr, mask, elts_per_pixel) diff --git a/src/libImaging/Storage.c b/src/libImaging/Storage.c index 7f8d9c4a0..2c57165c1 100644 --- a/src/libImaging/Storage.c +++ b/src/libImaging/Storage.c @@ -754,11 +754,11 @@ ImagingNewArrow( } } // Stored as [r,g,b,a,r,g,b,a....] - if (strcmp(schema->format, "C") == 0 // uint8 - && im->pixelsize == 4 // storage as 32 bpc - && schema->n_children == 0 // make sure schema is well formed. - && strcmp(im->arrow_band_format, "C") == 0 // Expected Format - && 4* pixels == external_array->length) { // expected length + if (strcmp(schema->format, "C") == 0 // uint8 + && im->pixelsize == 4 // storage as 32 bpc + && schema->n_children == 0 // make sure schema is well formed. + && strcmp(im->arrow_band_format, "C") == 0 // Expected Format + && 4 * pixels == external_array->length) { // expected length // single flat array, interleaved storage. if (ImagingBorrowArrow(im, external_array, 1, array_capsule)) { return im; From 6bf791a3e7b2490bcb34ae9eb44419ee65c3caee Mon Sep 17 00:00:00 2001 From: wiredfool Date: Mon, 21 Apr 2025 10:27:49 +0100 Subject: [PATCH 05/49] Use a named tuple for the packed parameters --- Tests/test_pyarrow.py | 56 ++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index bcdd7ddc9..822cd18ac 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any # undone +from typing import Any, NamedTuple import pytest @@ -151,29 +151,37 @@ def test_lifetime2() -> None: assert isinstance(px[0, 0], int) -UINT_ARR = ( - fl_uint8_4_type, - [1,2,3,4], - 1 +class DataShape(NamedTuple): + dtype: Any + elt: Any + elts_per_pixel: int + + +UINT_ARR = DataShape( + dtype=fl_uint8_4_type, + elt=[1, 2, 3, 4], # array of 4 uint 8 per pixel + elts_per_pixel=1, # only one array per pixel ) -UINT = ( - pyarrow.uint8(), - 3, - 4 + +UINT = DataShape( + dtype=pyarrow.uint8(), + elt=3, # one uint8, + elts_per_pixel=4, # but repeated 4x per pixel ) -INT32 = ( - pyarrow.uint32(), - 0xabcdef45, - 1 + +UINT32 = DataShape( + dtype=pyarrow.uint32(), + elt=0xABCDEF45, # one packed int, doesn't fit in a int32 > 0x80000000 + elts_per_pixel=1, # one per pixel ) @pytest.mark.parametrize( "mode, data_tp, mask", ( - ("L", (pyarrow.uint8(), 3, 1), None), - ("I", (pyarrow.int32(), 1 << 24, 1), None), - ("F", (pyarrow.float32(), 3.14159, 1), None), + ("L", DataShape(pyarrow.uint8(), 3, 1), None), + ("I", DataShape(pyarrow.int32(), 1 << 24, 1), None), + ("F", DataShape(pyarrow.float32(), 3.14159, 1), None), ("LA", UINT_ARR, [0, 3]), ("LA", UINT, [0, 3]), ("RGB", UINT_ARR, [0, 1, 2]), @@ -188,7 +196,7 @@ INT32 = ( ("HSV", UINT, [0, 1, 2]), ), ) -def test_fromarray(mode: str, data_tp: tuple, mask: list[int] | None) -> None: +def test_fromarray(mode: str, data_tp: DataShape, mask: list[int] | None) -> None: (dtype, elt, elts_per_pixel) = data_tp ct_pixels = TEST_IMAGE_SIZE[0] * TEST_IMAGE_SIZE[1] @@ -201,15 +209,15 @@ def test_fromarray(mode: str, data_tp: tuple, mask: list[int] | None) -> None: @pytest.mark.parametrize( "mode, data_tp, mask", ( - ("LA", INT32, [0, 3]), - ("RGB", INT32, [0, 1, 2]), - ("RGBA", INT32, None), - ("CMYK", INT32, None), - ("YCbCr", INT32, [0, 1, 2]), - ("HSV", INT32, [0, 1, 2]), + ("LA", UINT32, [0, 3]), + ("RGB", UINT32, [0, 1, 2]), + ("RGBA", UINT32, None), + ("CMYK", UINT32, None), + ("YCbCr", UINT32, [0, 1, 2]), + ("HSV", UINT32, [0, 1, 2]), ), ) -def test_from_int32array(mode: str, data_tp: tuple, mask: list[int] | None) -> None: +def test_from_int32array(mode: str, data_tp: DataShape, mask: list[int] | None) -> None: (dtype, elt, elts_per_pixel) = data_tp ct_pixels = TEST_IMAGE_SIZE[0] * TEST_IMAGE_SIZE[1] From ce204f47f45f2ecdc831faac9c1b7ad8192a9fc7 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Mon, 21 Apr 2025 10:37:32 +0100 Subject: [PATCH 06/49] lint --- src/libImaging/Storage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libImaging/Storage.c b/src/libImaging/Storage.c index 2c57165c1..1a9171a0c 100644 --- a/src/libImaging/Storage.c +++ b/src/libImaging/Storage.c @@ -737,7 +737,7 @@ ImagingNewArrow( return im; } } - // Stored as [[r,g,b,a],....] + // Stored as [[r,g,b,a],...] if (strcmp(schema->format, "+w:4") == 0 // 4 up array && im->pixelsize == 4 // storage as 32 bpc && schema->n_children > 0 // make sure schema is well formed. @@ -753,7 +753,7 @@ ImagingNewArrow( return im; } } - // Stored as [r,g,b,a,r,g,b,a....] + // Stored as [r,g,b,a,r,g,b,a,...] if (strcmp(schema->format, "C") == 0 // uint8 && im->pixelsize == 4 // storage as 32 bpc && schema->n_children == 0 // make sure schema is well formed. From bc4b664b7094a311eb516e7eab1b88acf7496b67 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Mon, 21 Apr 2025 10:46:45 +0100 Subject: [PATCH 07/49] Add integer range tests --- Tests/test_pyarrow.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index 822cd18ac..6eedcafe7 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -153,7 +153,10 @@ def test_lifetime2() -> None: class DataShape(NamedTuple): dtype: Any - elt: Any + elt: Any # Strictly speaking, this should be a pixel or pixel component, + # so list[uint8][4], float, int, uint32, uint8, etc. + # But more correctly, it should be exactly the dtype from the + # line above. elts_per_pixel: int @@ -175,6 +178,12 @@ UINT32 = DataShape( elts_per_pixel=1, # one per pixel ) +INT32 = DataShape( + dtype=pyarrow.uint32(), + elt=0x12CDEF45, # one packed int + elts_per_pixel=1, # one per pixel +) + @pytest.mark.parametrize( "mode, data_tp, mask", @@ -215,6 +224,12 @@ def test_fromarray(mode: str, data_tp: DataShape, mask: list[int] | None) -> Non ("CMYK", UINT32, None), ("YCbCr", UINT32, [0, 1, 2]), ("HSV", UINT32, [0, 1, 2]), + ("LA", INT32, [0, 3]), + ("RGB", INT32, [0, 1, 2]), + ("RGBA", INT32, None), + ("CMYK", INT32, None), + ("YCbCr", INT32, [0, 1, 2]), + ("HSV", INT32, [0, 1, 2]), ), ) def test_from_int32array(mode: str, data_tp: DataShape, mask: list[int] | None) -> None: From 45e24e429f7d443000a6955d228fa00055104414 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Mon, 21 Apr 2025 10:54:00 +0100 Subject: [PATCH 08/49] Rearrance so black doesn't screw up the formatting --- Tests/test_pyarrow.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index 6eedcafe7..e7fce1e33 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -153,10 +153,10 @@ def test_lifetime2() -> None: class DataShape(NamedTuple): dtype: Any - elt: Any # Strictly speaking, this should be a pixel or pixel component, - # so list[uint8][4], float, int, uint32, uint8, etc. - # But more correctly, it should be exactly the dtype from the - # line above. + # Strictly speaking, elt should be a pixel or pixel component, so + # list[uint8][4], float, int, uint32, uint8, etc. But more + # correctly, it should be exactly the dtype from the line above. + elt: Any elts_per_pixel: int From 4d56b90f38eda564ce8913bdc9b5222c3407652f Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 5 May 2025 07:12:20 +1000 Subject: [PATCH 09/49] Updated docstring --- src/PIL/DdsImagePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PIL/DdsImagePlugin.py b/src/PIL/DdsImagePlugin.py index 26307817c..f9ade18f9 100644 --- a/src/PIL/DdsImagePlugin.py +++ b/src/PIL/DdsImagePlugin.py @@ -1,5 +1,5 @@ """ -A Pillow loader for .dds files (S3TC-compressed aka DXTC) +A Pillow plugin for .dds files (S3TC-compressed aka DXTC) Jerome Leclanche Documentation: From 78887f6114e68d4208a6d5e8f3d5134a6da6831a Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Fri, 9 May 2025 23:52:18 +1000 Subject: [PATCH 10/49] Corrected comment --- Tests/test_pyarrow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index e7fce1e33..c5872231b 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -162,7 +162,7 @@ class DataShape(NamedTuple): UINT_ARR = DataShape( dtype=fl_uint8_4_type, - elt=[1, 2, 3, 4], # array of 4 uint 8 per pixel + elt=[1, 2, 3, 4], # array of 4 uint8 per pixel elts_per_pixel=1, # only one array per pixel ) From 74ab5ac4cda564714545eee52ab789d4bddf1516 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Sun, 11 May 2025 23:46:21 +0200 Subject: [PATCH 11/49] Fix memory leak in arrow export using array structure --- src/libImaging/Arrow.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libImaging/Arrow.c b/src/libImaging/Arrow.c index 33ff2ce77..36f4554fc 100644 --- a/src/libImaging/Arrow.c +++ b/src/libImaging/Arrow.c @@ -127,9 +127,7 @@ static void release_const_array(struct ArrowArray *array) { Imaging im = (Imaging)array->private_data; - if (array->n_children == 0) { - ImagingDelete(im); - } + ImagingDelete(im); // Free the buffers and the buffers array if (array->buffers) { From 4984c45da2f6b854cb49dc81fc56372f335d43a0 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 10:27:38 +0200 Subject: [PATCH 12/49] valgrind memory leak check --- Makefile | 6 ++++++ Tests/oss-fuzz/python.supp | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/Makefile b/Makefile index 53164b08a..fd124d124 100644 --- a/Makefile +++ b/Makefile @@ -99,6 +99,12 @@ valgrind: --log-file=/tmp/valgrind-output \ python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output +.PHONY: valgrind-leak +valgrind-leak: + PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite \ + --log-file=/tmp/valgrind-output \ + python3 -m pytest -vv --valgrind --valgrind-log=/tmp/valgrind-output Tests/ + .PHONY: readme readme: python3 -c "import markdown2" > /dev/null 2>&1 || python3 -m pip install markdown2 diff --git a/Tests/oss-fuzz/python.supp b/Tests/oss-fuzz/python.supp index 36385d672..1ea2a8eb5 100644 --- a/Tests/oss-fuzz/python.supp +++ b/Tests/oss-fuzz/python.supp @@ -14,3 +14,23 @@ fun:_TIFFReadEncodedTileAndAllocBuffer ... } + +{ + + Memcheck:Leak + match-leak-kinds: possible + fun:malloc + fun:_PyMem_RawMalloc + fun:PyObject_Malloc + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: possible + fun:malloc + fun:_PyMem_RawRealloc + fun:PyMem_Realloc + ... +} From fdfba982c8d514240435f3ecef540939fb97f120 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 10:28:09 +0200 Subject: [PATCH 13/49] fix memory leak in arrow schema --- src/libImaging/Arrow.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libImaging/Arrow.c b/src/libImaging/Arrow.c index 36f4554fc..7d3306123 100644 --- a/src/libImaging/Arrow.c +++ b/src/libImaging/Arrow.c @@ -37,6 +37,10 @@ ReleaseExportedSchema(struct ArrowSchema *array) { child->release = NULL; } // UNDONE -- should I be releasing the children? + free(array->children[i]); + } + if (array->children) { + free(array->children); } // Release dictionary @@ -117,6 +121,7 @@ export_imaging_schema(Imaging im, struct ArrowSchema *schema) { retval = export_named_type(schema->children[0], im->arrow_band_format, "pixel"); if (retval != 0) { free(schema->children[0]); + free(schema->children); schema->release(schema); return retval; } From 84b88a9fbc9c4cfd2bfb7d7a8d18225ee43efedb Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 10:58:12 +0200 Subject: [PATCH 14/49] Suppress all python level leaks for now --- Tests/oss-fuzz/python.supp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/oss-fuzz/python.supp b/Tests/oss-fuzz/python.supp index 1ea2a8eb5..4803497ad 100644 --- a/Tests/oss-fuzz/python.supp +++ b/Tests/oss-fuzz/python.supp @@ -18,7 +18,7 @@ { Memcheck:Leak - match-leak-kinds: possible + match-leak-kinds: all fun:malloc fun:_PyMem_RawMalloc fun:PyObject_Malloc @@ -28,7 +28,7 @@ { Memcheck:Leak - match-leak-kinds: possible + match-leak-kinds: all fun:malloc fun:_PyMem_RawRealloc fun:PyMem_Realloc From eaab43540344e26889116262651001f4e42b1630 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 10:58:37 +0200 Subject: [PATCH 15/49] Fix leak in webp_encode * Free the output buffer on webp encode error --- src/_webp.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/_webp.c b/src/_webp.c index 3aa4c408b..a7809c40e 100644 --- a/src/_webp.c +++ b/src/_webp.c @@ -641,6 +641,10 @@ WebPEncode_wrapper(PyObject *self, PyObject *args) { ImagingSectionLeave(&cookie); WebPPictureFree(&pic); + + output = writer.mem; + ret_size = writer.size; + if (!ok) { int error_code = (&pic)->error_code; char message[50] = ""; @@ -652,10 +656,9 @@ WebPEncode_wrapper(PyObject *self, PyObject *args) { ); } PyErr_Format(PyExc_ValueError, "encoding error %d%s", error_code, message); + free(output); return NULL; } - output = writer.mem; - ret_size = writer.size; { /* I want to truncate the *_size items that get passed into WebP From a9bcd7db884d89bbfe1966c0611f7f3dda1f8f08 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 19:50:55 +0200 Subject: [PATCH 16/49] Fix leak of destination image in ImagingUnsharpMask when an error occurs --- src/_imaging.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_imaging.c b/src/_imaging.c index 72f122143..79e0a2b23 100644 --- a/src/_imaging.c +++ b/src/_imaging.c @@ -2226,6 +2226,7 @@ _unsharp_mask(ImagingObject *self, PyObject *args) { } if (!ImagingUnsharpMask(imOut, imIn, radius, percent, threshold)) { + ImagingDelete(imOut); return NULL; } From e2e40c54568236d2504921eb0b335cdab734a7d5 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 22:33:27 +0200 Subject: [PATCH 17/49] Fix memory leak in TiffEncode * If setimage errors out, the tiff client state was not freed. --- src/encode.c | 2 ++ src/libImaging/TiffDecode.c | 42 ++++++++++++++++++------------------- src/libImaging/TiffDecode.h | 2 ++ 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/encode.c b/src/encode.c index 7c365a74f..e56494036 100644 --- a/src/encode.c +++ b/src/encode.c @@ -703,6 +703,8 @@ PyImaging_LibTiffEncoderNew(PyObject *self, PyObject *args) { return NULL; } + encoder->cleanup = ImagingLibTiffEncodeCleanup; + num_core_tags = sizeof(core_tags) / sizeof(int); for (pos = 0; pos < tags_size; pos++) { item = PyList_GetItemRef(tags, pos); diff --git a/src/libImaging/TiffDecode.c b/src/libImaging/TiffDecode.c index 9a2db95b4..173eca160 100644 --- a/src/libImaging/TiffDecode.c +++ b/src/libImaging/TiffDecode.c @@ -929,6 +929,27 @@ ImagingLibTiffSetField(ImagingCodecState state, ttag_t tag, ...) { return status; } +int +ImagingLibTiffEncodeCleanup(ImagingCodecState state) { + TIFFSTATE *clientstate = (TIFFSTATE *)state->context; + TIFF *tiff = clientstate->tiff; + + if (!tiff) { + return 0; + } + // TIFFClose in libtiff calls tif_closeproc and TIFFCleanup + if (clientstate->fp) { + // Python will manage the closing of the file rather than libtiff + // So only call TIFFCleanup + TIFFCleanup(tiff); + } else { + // When tif_closeproc refers to our custom _tiffCloseProc though, + // that is fine, as it does not close the file + TIFFClose(tiff); + } + return 0; +} + int ImagingLibTiffEncode(Imaging im, ImagingCodecState state, UINT8 *buffer, int bytes) { /* One shot encoder. Encode everything to the tiff in the clientstate. @@ -1010,16 +1031,6 @@ ImagingLibTiffEncode(Imaging im, ImagingCodecState state, UINT8 *buffer, int byt TRACE(("Encode Error, row %d\n", state->y)); state->errcode = IMAGING_CODEC_BROKEN; - // TIFFClose in libtiff calls tif_closeproc and TIFFCleanup - if (clientstate->fp) { - // Python will manage the closing of the file rather than libtiff - // So only call TIFFCleanup - TIFFCleanup(tiff); - } else { - // When tif_closeproc refers to our custom _tiffCloseProc though, - // that is fine, as it does not close the file - TIFFClose(tiff); - } if (!clientstate->fp) { free(clientstate->data); } @@ -1036,22 +1047,11 @@ ImagingLibTiffEncode(Imaging im, ImagingCodecState state, UINT8 *buffer, int byt TRACE(("Error flushing the tiff")); // likely reason is memory. state->errcode = IMAGING_CODEC_MEMORY; - if (clientstate->fp) { - TIFFCleanup(tiff); - } else { - TIFFClose(tiff); - } if (!clientstate->fp) { free(clientstate->data); } return -1; } - TRACE(("Closing \n")); - if (clientstate->fp) { - TIFFCleanup(tiff); - } else { - TIFFClose(tiff); - } // reset the clientstate metadata to use it to read out the buffer. clientstate->loc = 0; clientstate->size = clientstate->eof; // redundant? diff --git a/src/libImaging/TiffDecode.h b/src/libImaging/TiffDecode.h index 22361210d..77808b543 100644 --- a/src/libImaging/TiffDecode.h +++ b/src/libImaging/TiffDecode.h @@ -40,6 +40,8 @@ ImagingLibTiffInit(ImagingCodecState state, int fp, uint32_t offset); extern int ImagingLibTiffEncodeInit(ImagingCodecState state, char *filename, int fp); extern int +ImagingLibTiffEncodeCleanup(ImagingCodecState state); +extern int ImagingLibTiffMergeFieldInfo( ImagingCodecState state, TIFFDataType field_type, int key, int is_var_length ); From f792e0b1ef4f25e0df33e8e971056142f9f5248d Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 22:48:36 +0200 Subject: [PATCH 18/49] Fix memory leak * Return after setting the error for advanced features without libraqm. Not returning here leads to an alloc that's never freed. --- src/_imagingft.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_imagingft.c b/src/_imagingft.c index 62dab73e5..ca8e556f0 100644 --- a/src/_imagingft.c +++ b/src/_imagingft.c @@ -425,6 +425,7 @@ text_layout_fallback( "setting text direction, language or font features is not supported " "without libraqm" ); + return 0; } if (PyUnicode_Check(string)) { From 789631c60c3760beeb623cd1728a737502fd9ca3 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 23:31:09 +0200 Subject: [PATCH 19/49] Fix memory leak when JpegEncode returns an error. --- src/libImaging/JpegEncode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libImaging/JpegEncode.c b/src/libImaging/JpegEncode.c index 3c11eac22..79a38e12f 100644 --- a/src/libImaging/JpegEncode.c +++ b/src/libImaging/JpegEncode.c @@ -131,6 +131,7 @@ ImagingJpegEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) { break; default: state->errcode = IMAGING_CODEC_CONFIG; + jpeg_destroy_compress(&context->cinfo); return -1; } @@ -161,6 +162,7 @@ ImagingJpegEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) { /* Would subsample the green and blue channels, which doesn't make sense */ state->errcode = IMAGING_CODEC_CONFIG; + jpeg_destroy_compress(&context->cinfo); return -1; } jpeg_set_colorspace(&context->cinfo, JCS_RGB); From 7aa6a61d430c585cd10c91c5a73ce13f9f851b9e Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Tue, 13 May 2025 23:50:52 +0200 Subject: [PATCH 20/49] Wrap Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fd124d124..15f03ba45 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,8 @@ valgrind: .PHONY: valgrind-leak valgrind-leak: - PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite \ + PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp \ + --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite \ --log-file=/tmp/valgrind-output \ python3 -m pytest -vv --valgrind --valgrind-log=/tmp/valgrind-output Tests/ From fb126af7a6a12e0870e56187257f75f35fe8558b Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Thu, 15 May 2025 21:10:48 +0200 Subject: [PATCH 21/49] Adding pytest-valgrind install --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 15f03ba45..bdddecda5 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ valgrind: .PHONY: valgrind-leak valgrind-leak: + python3 -c "import pytest_valgrind" > /dev/null 2>&1 || python3 -m pip install pytest-valgrind PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp \ --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite \ --log-file=/tmp/valgrind-output \ From d5449d576013566100d8a0d41bbc1a756df86da5 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Thu, 15 May 2025 21:11:31 +0200 Subject: [PATCH 22/49] Guess so. --- src/libImaging/Arrow.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libImaging/Arrow.c b/src/libImaging/Arrow.c index 7d3306123..0b8c89a07 100644 --- a/src/libImaging/Arrow.c +++ b/src/libImaging/Arrow.c @@ -36,7 +36,6 @@ ReleaseExportedSchema(struct ArrowSchema *array) { child->release(child); child->release = NULL; } - // UNDONE -- should I be releasing the children? free(array->children[i]); } if (array->children) { From 218f055865a4f0abd05ac221c48cf86127907ca9 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Thu, 15 May 2025 21:59:02 +0200 Subject: [PATCH 23/49] Add github workflow/test-script --- .github/workflows/test-valgrind-memory.yml | 60 ++++++++++++++++++++++ depends/docker-test-valgrind-memory.sh | 11 ++++ 2 files changed, 71 insertions(+) create mode 100644 .github/workflows/test-valgrind-memory.yml create mode 100644 depends/docker-test-valgrind-memory.sh diff --git a/.github/workflows/test-valgrind-memory.yml b/.github/workflows/test-valgrind-memory.yml new file mode 100644 index 000000000..e6a5f6e77 --- /dev/null +++ b/.github/workflows/test-valgrind-memory.yml @@ -0,0 +1,60 @@ +name: Test Valgrind Memory Leaks + +# like the Docker tests, but running valgrind only on *.c/*.h changes. + +# this is very expensive. Only run on the pull request. +on: + # push: + # branches: + # - "**" + # paths: + # - ".github/workflows/test-valgrind.yml" + # - "**.c" + # - "**.h" + pull_request: + paths: + - ".github/workflows/test-valgrind.yml" + - "**.c" + - "**.h" + - "depends/docker-test-valgrind-memory.sh" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + docker: [ + ubuntu-22.04-jammy-amd64-valgrind, + ] + dockerTag: [main] + + name: ${{ matrix.docker }} + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Build system information + run: python3 .github/workflows/system-info.py + + - name: Docker pull + run: | + docker pull pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }} + + - name: Build and Run Valgrind + run: | + # The Pillow user in the docker container is UID 1001 + sudo chown -R 1001 $GITHUB_WORKSPACE + docker run --name pillow_container -e "PILLOW_VALGRIND_TEST=true" -v $GITHUB_WORKSPACE:/Pillow pythonpillow/${{ matrix.docker }}:${{ matrix.dockerTag }} /Pillow/depends/docker-test-valgrind-memory.sh + sudo chown -R runner $GITHUB_WORKSPACE diff --git a/depends/docker-test-valgrind-memory.sh b/depends/docker-test-valgrind-memory.sh new file mode 100644 index 000000000..4fd6652d8 --- /dev/null +++ b/depends/docker-test-valgrind-memory.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +## Run this as the test script in the docker valgrind image. +## Note -- can be included directly into the docker image, +## but requires the currnet python.supp. + +source /vpy3/bin/activate +cd /Pillow +make clean +make install +make valgrind-memory From a6b8b3af7709081d8c53818e68b8bc15e9a48f34 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Thu, 15 May 2025 22:04:14 +0200 Subject: [PATCH 24/49] executable --- depends/docker-test-valgrind-memory.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 depends/docker-test-valgrind-memory.sh diff --git a/depends/docker-test-valgrind-memory.sh b/depends/docker-test-valgrind-memory.sh old mode 100644 new mode 100755 From 2d506f6f5a478b4a798b3ce71f31b5e5f6f6b60f Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Thu, 15 May 2025 22:06:35 +0200 Subject: [PATCH 25/49] correct target --- depends/docker-test-valgrind-memory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/docker-test-valgrind-memory.sh b/depends/docker-test-valgrind-memory.sh index 4fd6652d8..29fc6f230 100755 --- a/depends/docker-test-valgrind-memory.sh +++ b/depends/docker-test-valgrind-memory.sh @@ -8,4 +8,4 @@ source /vpy3/bin/activate cd /Pillow make clean make install -make valgrind-memory +make valgrind-leak From f1957b49b2d01a9d063aed4000f985b220e30fa0 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Fri, 16 May 2025 12:08:45 +0200 Subject: [PATCH 26/49] Xfail timouts in Valgrind tests * ensure that the env variable is set in the makefile --- Makefile | 4 ++-- Tests/test_file_jpeg.py | 5 +++++ Tests/test_imagefontpil.py | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bdddecda5..82c2c085f 100644 --- a/Makefile +++ b/Makefile @@ -95,14 +95,14 @@ test: .PHONY: valgrind valgrind: python3 -c "import pytest_valgrind" > /dev/null 2>&1 || python3 -m pip install pytest-valgrind - PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \ + PILLOW_VALGRIND_TEST=true PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \ --log-file=/tmp/valgrind-output \ python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output .PHONY: valgrind-leak valgrind-leak: python3 -c "import pytest_valgrind" > /dev/null 2>&1 || python3 -m pip install pytest-valgrind - PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp \ + PILLOW_VALGRIND_TEST=true PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp \ --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite \ --log-file=/tmp/valgrind-output \ python3 -m pytest -vv --valgrind --valgrind-log=/tmp/valgrind-output Tests/ diff --git a/Tests/test_file_jpeg.py b/Tests/test_file_jpeg.py index 79f0ec1a8..fb9f26fc7 100644 --- a/Tests/test_file_jpeg.py +++ b/Tests/test_file_jpeg.py @@ -1034,6 +1034,11 @@ class TestFileJpeg: im.save(f, xmp=b"1" * 65505) @pytest.mark.timeout(timeout=1) + @pytest.mark.xfail( + "PILLOW_VALGRIND_TEST" in os.environ, + reason="Valgrind is slower", + raises=TimeoutError + ) def test_eof(self, monkeypatch: pytest.MonkeyPatch) -> None: # Even though this decoder never says that it is finished # the image should still end when there is no new data diff --git a/Tests/test_imagefontpil.py b/Tests/test_imagefontpil.py index 695aecbde..adce4a75c 100644 --- a/Tests/test_imagefontpil.py +++ b/Tests/test_imagefontpil.py @@ -2,6 +2,7 @@ from __future__ import annotations import struct from io import BytesIO +import os import pytest @@ -73,6 +74,11 @@ def test_decompression_bomb() -> None: @pytest.mark.timeout(4) +@pytest.mark.xfail( + "PILLOW_VALGRIND_TEST" in os.environ, + reason="Valgrind is slower", + raises=TimeoutError +) def test_oom() -> None: glyph = struct.pack( ">hhhhhhhhhh", 1, 0, -32767, -32767, 32767, 32767, -32767, -32767, 32767, 32767 From ff50e30d3e9f1425ca6af95ac044d365c63719d1 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Fri, 16 May 2025 12:47:22 +0200 Subject: [PATCH 27/49] Fix memory leak in text_layout_raqm on 0 length string --- src/_imagingft.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/_imagingft.c b/src/_imagingft.c index ca8e556f0..0d70544a5 100644 --- a/src/_imagingft.c +++ b/src/_imagingft.c @@ -275,6 +275,7 @@ text_layout_raqm( if (!text || !size) { /* return 0 and clean up, no glyphs==no size, and raqm fails with empty strings */ + PyMem_Free(text); goto failed; } set_text = raqm_set_text(rq, text, size); From 20b49a332bd0f0f39660fbb3587cfc4b6d539f0c Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Sat, 17 May 2025 10:45:43 +0200 Subject: [PATCH 28/49] Remove timeout as the specific reason, pytest-timeout doesn't raise a timeout error. --- Tests/test_file_jpeg.py | 3 +-- Tests/test_imagefontpil.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Tests/test_file_jpeg.py b/Tests/test_file_jpeg.py index fb9f26fc7..d923020c8 100644 --- a/Tests/test_file_jpeg.py +++ b/Tests/test_file_jpeg.py @@ -1036,8 +1036,7 @@ class TestFileJpeg: @pytest.mark.timeout(timeout=1) @pytest.mark.xfail( "PILLOW_VALGRIND_TEST" in os.environ, - reason="Valgrind is slower", - raises=TimeoutError + reason="Valgrind is slower" ) def test_eof(self, monkeypatch: pytest.MonkeyPatch) -> None: # Even though this decoder never says that it is finished diff --git a/Tests/test_imagefontpil.py b/Tests/test_imagefontpil.py index adce4a75c..bd9bafb55 100644 --- a/Tests/test_imagefontpil.py +++ b/Tests/test_imagefontpil.py @@ -76,8 +76,7 @@ def test_decompression_bomb() -> None: @pytest.mark.timeout(4) @pytest.mark.xfail( "PILLOW_VALGRIND_TEST" in os.environ, - reason="Valgrind is slower", - raises=TimeoutError + reason="Valgrind is slower" ) def test_oom() -> None: glyph = struct.pack( From c35082b619899d5351ba249e8ea23a4412d0c728 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 17 May 2025 08:47:59 +0000 Subject: [PATCH 29/49] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Tests/test_file_jpeg.py | 3 +-- Tests/test_imagefontpil.py | 7 ++----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/Tests/test_file_jpeg.py b/Tests/test_file_jpeg.py index d923020c8..7c33c7517 100644 --- a/Tests/test_file_jpeg.py +++ b/Tests/test_file_jpeg.py @@ -1035,8 +1035,7 @@ class TestFileJpeg: @pytest.mark.timeout(timeout=1) @pytest.mark.xfail( - "PILLOW_VALGRIND_TEST" in os.environ, - reason="Valgrind is slower" + "PILLOW_VALGRIND_TEST" in os.environ, reason="Valgrind is slower" ) def test_eof(self, monkeypatch: pytest.MonkeyPatch) -> None: # Even though this decoder never says that it is finished diff --git a/Tests/test_imagefontpil.py b/Tests/test_imagefontpil.py index bd9bafb55..e5b770745 100644 --- a/Tests/test_imagefontpil.py +++ b/Tests/test_imagefontpil.py @@ -1,8 +1,8 @@ from __future__ import annotations +import os import struct from io import BytesIO -import os import pytest @@ -74,10 +74,7 @@ def test_decompression_bomb() -> None: @pytest.mark.timeout(4) -@pytest.mark.xfail( - "PILLOW_VALGRIND_TEST" in os.environ, - reason="Valgrind is slower" -) +@pytest.mark.xfail("PILLOW_VALGRIND_TEST" in os.environ, reason="Valgrind is slower") def test_oom() -> None: glyph = struct.pack( ">hhhhhhhhhh", 1, 0, -32767, -32767, 32767, 32767, -32767, -32767, 32767, 32767 From 2603a249df9223133b74c671acfcdc6a51567843 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 23 May 2025 10:57:03 +0100 Subject: [PATCH 30/49] Update depends/docker-test-valgrind-memory.sh Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- depends/docker-test-valgrind-memory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/docker-test-valgrind-memory.sh b/depends/docker-test-valgrind-memory.sh index 29fc6f230..5f7805421 100755 --- a/depends/docker-test-valgrind-memory.sh +++ b/depends/docker-test-valgrind-memory.sh @@ -2,7 +2,7 @@ ## Run this as the test script in the docker valgrind image. ## Note -- can be included directly into the docker image, -## but requires the currnet python.supp. +## but requires the current python.supp. source /vpy3/bin/activate cd /Pillow From 9526d949b07bbddfc7e515810dc23738b778bee4 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 23 May 2025 10:58:28 +0100 Subject: [PATCH 31/49] Update Tests/test_pyarrow.py Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- Tests/test_pyarrow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index e7fce1e33..c5872231b 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -162,7 +162,7 @@ class DataShape(NamedTuple): UINT_ARR = DataShape( dtype=fl_uint8_4_type, - elt=[1, 2, 3, 4], # array of 4 uint 8 per pixel + elt=[1, 2, 3, 4], # array of 4 uint8 per pixel elts_per_pixel=1, # only one array per pixel ) From 6807bd3d70cc5873b3cad29d598e08a34fdc1fa0 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 10 May 2025 00:03:08 +1000 Subject: [PATCH 32/49] Added type hints --- .ci/requirements-mypy.txt | 1 + Tests/test_pyarrow.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.ci/requirements-mypy.txt b/.ci/requirements-mypy.txt index 2e3610478..86ac2e0b2 100644 --- a/.ci/requirements-mypy.txt +++ b/.ci/requirements-mypy.txt @@ -4,6 +4,7 @@ IceSpringPySideStubs-PySide6 ipython numpy packaging +pyarrow-stubs pytest sphinx types-atheris diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index c5872231b..2029f96f5 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -13,7 +13,11 @@ from .helper import ( is_big_endian, ) -pyarrow = pytest.importorskip("pyarrow", reason="PyArrow not installed") +TYPE_CHECKING = False +if TYPE_CHECKING: + import pyarrow +else: + pyarrow = pytest.importorskip("pyarrow", reason="PyArrow not installed") TEST_IMAGE_SIZE = (10, 10) @@ -94,14 +98,14 @@ fl_uint8_4_type = pyarrow.field( ("HSV", fl_uint8_4_type, [0, 1, 2]), ), ) -def test_to_array(mode: str, dtype: Any, mask: list[int] | None) -> None: +def test_to_array(mode: str, dtype: pyarrow.DataType, mask: list[int] | None) -> None: img = hopper(mode) # Resize to non-square img = img.crop((3, 0, 124, 127)) assert img.size == (121, 127) - arr = pyarrow.array(img) + arr = pyarrow.array(img) # type: ignore[call-overload] _test_img_equals_pyarray(img, arr, mask) assert arr.type == dtype @@ -118,8 +122,8 @@ def test_lifetime() -> None: img = hopper("L") - arr_1 = pyarrow.array(img) - arr_2 = pyarrow.array(img) + arr_1 = pyarrow.array(img) # type: ignore[call-overload] + arr_2 = pyarrow.array(img) # type: ignore[call-overload] del img @@ -136,8 +140,8 @@ def test_lifetime2() -> None: img = hopper("L") - arr_1 = pyarrow.array(img) - arr_2 = pyarrow.array(img) + arr_1 = pyarrow.array(img) # type: ignore[call-overload] + arr_2 = pyarrow.array(img) # type: ignore[call-overload] assert arr_1.sum().as_py() > 0 del arr_1 @@ -152,7 +156,7 @@ def test_lifetime2() -> None: class DataShape(NamedTuple): - dtype: Any + dtype: pyarrow.DataType # Strictly speaking, elt should be a pixel or pixel component, so # list[uint8][4], float, int, uint32, uint8, etc. But more # correctly, it should be exactly the dtype from the line above. From 60a1a20536fe18cfe936e90140ed56c3eb31bd81 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Fri, 23 May 2025 15:32:46 +0200 Subject: [PATCH 33/49] add timeouts to two more tests --- Tests/test_file_tiff.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tests/test_file_tiff.py b/Tests/test_file_tiff.py index 502d9df9a..050bfe578 100644 --- a/Tests/test_file_tiff.py +++ b/Tests/test_file_tiff.py @@ -990,6 +990,10 @@ class TestFileTiff: @pytest.mark.timeout(6) @pytest.mark.filterwarnings("ignore:Truncated File Read") + @pytest.mark.xfail( + "PILLOW_VALGRIND_TEST" in os.environ, + reason="Valgrind is slower" + ) def test_timeout(self, monkeypatch: pytest.MonkeyPatch) -> None: with Image.open("Tests/images/timeout-6646305047838720") as im: monkeypatch.setattr(ImageFile, "LOAD_TRUNCATED_IMAGES", True) @@ -1002,6 +1006,10 @@ class TestFileTiff: ], ) @pytest.mark.timeout(2) + @pytest.mark.xfail( + "PILLOW_VALGRIND_TEST" in os.environ, + reason="Valgrind is slower" + ) def test_oom(self, test_file: str) -> None: with pytest.raises(UnidentifiedImageError): with pytest.warns(UserWarning): From c63db77db3850c51df38af8f4a96f5c13f286b42 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 May 2025 13:37:02 +0000 Subject: [PATCH 34/49] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Tests/test_file_tiff.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tests/test_file_tiff.py b/Tests/test_file_tiff.py index 050bfe578..b6985b83b 100644 --- a/Tests/test_file_tiff.py +++ b/Tests/test_file_tiff.py @@ -991,8 +991,7 @@ class TestFileTiff: @pytest.mark.timeout(6) @pytest.mark.filterwarnings("ignore:Truncated File Read") @pytest.mark.xfail( - "PILLOW_VALGRIND_TEST" in os.environ, - reason="Valgrind is slower" + "PILLOW_VALGRIND_TEST" in os.environ, reason="Valgrind is slower" ) def test_timeout(self, monkeypatch: pytest.MonkeyPatch) -> None: with Image.open("Tests/images/timeout-6646305047838720") as im: @@ -1007,8 +1006,7 @@ class TestFileTiff: ) @pytest.mark.timeout(2) @pytest.mark.xfail( - "PILLOW_VALGRIND_TEST" in os.environ, - reason="Valgrind is slower" + "PILLOW_VALGRIND_TEST" in os.environ, reason="Valgrind is slower" ) def test_oom(self, test_file: str) -> None: with pytest.raises(UnidentifiedImageError): From 4d0678ca33b65af2686fe93be5b77c2b28027959 Mon Sep 17 00:00:00 2001 From: Eric Soroos Date: Fri, 23 May 2025 16:35:57 +0200 Subject: [PATCH 35/49] Add parallel test target, using pytest-xdist --- Makefile | 8 +++++++- pyproject.toml | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5a8152454..a56fe8fec 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,13 @@ sdist: .PHONY: test test: python3 -c "import pytest" > /dev/null 2>&1 || python3 -m pip install pytest - python3 -m pytest -qq + python3 -m pytest -qq Tests/ + +.PHONY: test-p +test-p: + python3 -c "import xdist" > /dev/null 2>&1 || python3 -m pip install pytest-xdist + python3 -m pytest -qq -n auto Tests/ + .PHONY: valgrind valgrind: diff --git a/pyproject.toml b/pyproject.toml index a3ff9723b..683ab24ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ optional-dependencies.tests = [ "pytest", "pytest-cov", "pytest-timeout", + "pytest-xdist", "trove-classifiers>=2024.10.12", ] From 4eb89f8e5bcd19ad64ed2328c9566061a7116cc2 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 21 May 2025 20:36:19 +1000 Subject: [PATCH 36/49] Reduced number of bytes read for header --- src/PIL/WmfImagePlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PIL/WmfImagePlugin.py b/src/PIL/WmfImagePlugin.py index f709d026b..d569cb4b8 100644 --- a/src/PIL/WmfImagePlugin.py +++ b/src/PIL/WmfImagePlugin.py @@ -81,7 +81,7 @@ class WmfStubImageFile(ImageFile.StubImageFile): def _open(self) -> None: # check placable header - s = self.fp.read(80) + s = self.fp.read(44) if s.startswith(b"\xd7\xcd\xc6\x9a\x00\x00"): # placeable windows metafile From 57b77bde96484d4a1d6f92adec9a2c2b86485f55 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 24 May 2025 11:55:18 +1000 Subject: [PATCH 37/49] Removed CMAKE_POLICY_VERSION_MINIMUM=3.5 --- .ci/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install.sh b/.ci/install.sh index d065e7ab5..acb84f046 100755 --- a/.ci/install.sh +++ b/.ci/install.sh @@ -66,7 +66,7 @@ if [[ $(uname) != CYGWIN* ]]; then pushd depends && ./install_raqm.sh && popd # libavif - pushd depends && CMAKE_POLICY_VERSION_MINIMUM=3.5 ./install_libavif.sh && popd + pushd depends && ./install_libavif.sh && popd # extra test images pushd depends && ./install_extra_test_images.sh && popd From 5a04b9581b16a7f1e1109f1e31a206a6550f314c Mon Sep 17 00:00:00 2001 From: Andrew Murray <3112309+radarhere@users.noreply.github.com> Date: Wed, 28 May 2025 08:20:35 +1000 Subject: [PATCH 38/49] Run slow tests on valgrind, but without timeout (#8975) --- Tests/helper.py | 6 ++++++ Tests/test_file_eps.py | 3 ++- Tests/test_file_fli.py | 9 +++++++-- Tests/test_file_jpeg.py | 3 ++- Tests/test_file_pdf.py | 10 +++++++--- Tests/test_file_tiff.py | 5 +++-- Tests/test_image.py | 6 ++---- Tests/test_imagefontpil.py | 4 ++-- 8 files changed, 31 insertions(+), 15 deletions(-) diff --git a/Tests/helper.py b/Tests/helper.py index 909fff879..ec61cd263 100644 --- a/Tests/helper.py +++ b/Tests/helper.py @@ -161,6 +161,12 @@ def assert_tuple_approx_equal( pytest.fail(msg + ": " + repr(actuals) + " != " + repr(targets)) +def timeout_unless_slower_valgrind(timeout: float) -> pytest.MarkDecorator: + if "PILLOW_VALGRIND_TEST" in os.environ: + return pytest.mark.pil_noop_mark() + return pytest.mark.timeout(timeout) + + def skip_unless_feature(feature: str) -> pytest.MarkDecorator: reason = f"{feature} not available" return pytest.mark.skipif(not features.check(feature), reason=reason) diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index b484a8cfa..d94de7287 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -15,6 +15,7 @@ from .helper import ( is_win32, mark_if_feature_version, skip_unless_feature, + timeout_unless_slower_valgrind, ) HAS_GHOSTSCRIPT = EpsImagePlugin.has_ghostscript() @@ -398,7 +399,7 @@ def test_emptyline() -> None: assert image.format == "EPS" -@pytest.mark.timeout(timeout=5) +@timeout_unless_slower_valgrind(5) @pytest.mark.parametrize( "test_file", ["Tests/images/eps/timeout-d675703545fee17acab56e5fec644c19979175de.eps"], diff --git a/Tests/test_file_fli.py b/Tests/test_file_fli.py index 81df1ab0b..0fadd01d0 100644 --- a/Tests/test_file_fli.py +++ b/Tests/test_file_fli.py @@ -7,7 +7,12 @@ import pytest from PIL import FliImagePlugin, Image, ImageFile -from .helper import assert_image_equal, assert_image_equal_tofile, is_pypy +from .helper import ( + assert_image_equal, + assert_image_equal_tofile, + is_pypy, + timeout_unless_slower_valgrind, +) # created as an export of a palette image from Gimp2.6 # save as...-> hopper.fli, default options. @@ -189,7 +194,7 @@ def test_seek() -> None: "Tests/images/timeout-bff0a9dc7243a8e6ede2408d2ffa6a9964698b87.fli", ], ) -@pytest.mark.timeout(timeout=3) +@timeout_unless_slower_valgrind(3) def test_timeouts(test_file: str) -> None: with open(test_file, "rb") as f: with Image.open(f) as im: diff --git a/Tests/test_file_jpeg.py b/Tests/test_file_jpeg.py index 79f0ec1a8..b9eec591d 100644 --- a/Tests/test_file_jpeg.py +++ b/Tests/test_file_jpeg.py @@ -32,6 +32,7 @@ from .helper import ( is_win32, mark_if_feature_version, skip_unless_feature, + timeout_unless_slower_valgrind, ) ElementTree: ModuleType | None @@ -1033,7 +1034,7 @@ class TestFileJpeg: with pytest.raises(ValueError): im.save(f, xmp=b"1" * 65505) - @pytest.mark.timeout(timeout=1) + @timeout_unless_slower_valgrind(1) def test_eof(self, monkeypatch: pytest.MonkeyPatch) -> None: # Even though this decoder never says that it is finished # the image should still end when there is no new data diff --git a/Tests/test_file_pdf.py b/Tests/test_file_pdf.py index bde1e3ab8..a2218673b 100644 --- a/Tests/test_file_pdf.py +++ b/Tests/test_file_pdf.py @@ -13,7 +13,12 @@ import pytest from PIL import Image, PdfParser, features -from .helper import hopper, mark_if_feature_version, skip_unless_feature +from .helper import ( + hopper, + mark_if_feature_version, + skip_unless_feature, + timeout_unless_slower_valgrind, +) def helper_save_as_pdf(tmp_path: Path, mode: str, **kwargs: Any) -> str: @@ -339,8 +344,7 @@ def test_pdf_append_to_bytesio() -> None: assert len(f.getvalue()) > initial_size -@pytest.mark.timeout(1) -@pytest.mark.skipif("PILLOW_VALGRIND_TEST" in os.environ, reason="Valgrind is slower") +@timeout_unless_slower_valgrind(1) @pytest.mark.parametrize("newline", (b"\r", b"\n")) def test_redos(newline: bytes) -> None: malicious = b" trailer<<>>" + newline * 3456 diff --git a/Tests/test_file_tiff.py b/Tests/test_file_tiff.py index 502d9df9a..d0d394aa9 100644 --- a/Tests/test_file_tiff.py +++ b/Tests/test_file_tiff.py @@ -26,6 +26,7 @@ from .helper import ( hopper, is_pypy, is_win32, + timeout_unless_slower_valgrind, ) ElementTree: ModuleType | None @@ -988,7 +989,7 @@ class TestFileTiff: with pytest.raises(OSError): im.load() - @pytest.mark.timeout(6) + @timeout_unless_slower_valgrind(6) @pytest.mark.filterwarnings("ignore:Truncated File Read") def test_timeout(self, monkeypatch: pytest.MonkeyPatch) -> None: with Image.open("Tests/images/timeout-6646305047838720") as im: @@ -1001,7 +1002,7 @@ class TestFileTiff: "Tests/images/oom-225817ca0f8c663be7ab4b9e717b02c661e66834.tif", ], ) - @pytest.mark.timeout(2) + @timeout_unless_slower_valgrind(2) def test_oom(self, test_file: str) -> None: with pytest.raises(UnidentifiedImageError): with pytest.warns(UserWarning): diff --git a/Tests/test_image.py b/Tests/test_image.py index 7e6118d52..14a067127 100644 --- a/Tests/test_image.py +++ b/Tests/test_image.py @@ -34,6 +34,7 @@ from .helper import ( is_win32, mark_if_feature_version, skip_unless_feature, + timeout_unless_slower_valgrind, ) ElementTree: ModuleType | None @@ -572,10 +573,7 @@ class TestImage: i = Image.new("RGB", [1, 1]) assert isinstance(i.size, tuple) - @pytest.mark.timeout(0.75) - @pytest.mark.skipif( - "PILLOW_VALGRIND_TEST" in os.environ, reason="Valgrind is slower" - ) + @timeout_unless_slower_valgrind(0.75) @pytest.mark.parametrize("size", ((0, 100000000), (100000000, 0))) def test_empty_image(self, size: tuple[int, int]) -> None: Image.new("RGB", size) diff --git a/Tests/test_imagefontpil.py b/Tests/test_imagefontpil.py index 695aecbde..3eb98d379 100644 --- a/Tests/test_imagefontpil.py +++ b/Tests/test_imagefontpil.py @@ -7,7 +7,7 @@ import pytest from PIL import Image, ImageDraw, ImageFont, _util, features -from .helper import assert_image_equal_tofile +from .helper import assert_image_equal_tofile, timeout_unless_slower_valgrind fonts = [ImageFont.load_default_imagefont()] if not features.check_module("freetype2"): @@ -72,7 +72,7 @@ def test_decompression_bomb() -> None: font.getmask("A" * 1_000_000) -@pytest.mark.timeout(4) +@timeout_unless_slower_valgrind(4) def test_oom() -> None: glyph = struct.pack( ">hhhhhhhhhh", 1, 0, -32767, -32767, 32767, 32767, -32767, -32767, 32767, 32767 From 6a60b2e6dd0909f627d093cbc431891a79d2b987 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 30 May 2025 10:27:11 +0100 Subject: [PATCH 39/49] Remove Tests/ path arg, this is already configured --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a56fe8fec..1f9d2ce13 100644 --- a/Makefile +++ b/Makefile @@ -95,12 +95,12 @@ sdist: .PHONY: test test: python3 -c "import pytest" > /dev/null 2>&1 || python3 -m pip install pytest - python3 -m pytest -qq Tests/ + python3 -m pytest -qq .PHONY: test-p test-p: python3 -c "import xdist" > /dev/null 2>&1 || python3 -m pip install pytest-xdist - python3 -m pytest -qq -n auto Tests/ + python3 -m pytest -qq -n auto .PHONY: valgrind From 98cf15e9e487cbc53b101498d43cc0cc141ee7e7 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 30 May 2025 10:35:13 +0100 Subject: [PATCH 40/49] Update depends/docker-test-valgrind-memory.sh Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- depends/docker-test-valgrind-memory.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/depends/docker-test-valgrind-memory.sh b/depends/docker-test-valgrind-memory.sh index 5f7805421..f0d1d851d 100755 --- a/depends/docker-test-valgrind-memory.sh +++ b/depends/docker-test-valgrind-memory.sh @@ -1,7 +1,7 @@ #!/bin/bash -## Run this as the test script in the docker valgrind image. -## Note -- can be included directly into the docker image, +## Run this as the test script in the Docker valgrind image. +## Note -- can be included directly into the Docker image, ## but requires the current python.supp. source /vpy3/bin/activate From 399b6c1045ff2387e7db8206e72baec33f996030 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 30 May 2025 10:40:07 +0100 Subject: [PATCH 41/49] Update Makefile Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4f63cfe02..27d70dcb7 100644 --- a/Makefile +++ b/Makefile @@ -110,7 +110,7 @@ valgrind-leak: PILLOW_VALGRIND_TEST=true PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp \ --leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite \ --log-file=/tmp/valgrind-output \ - python3 -m pytest -vv --valgrind --valgrind-log=/tmp/valgrind-output Tests/ + python3 -m pytest -vv --valgrind --valgrind-log=/tmp/valgrind-output .PHONY: readme readme: From 506691729a2f9d33228f8693cdbe90418e1b321a Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 30 May 2025 10:40:35 +0100 Subject: [PATCH 42/49] Apply suggestions from code review Co-authored-by: Andrew Murray <3112309+radarhere@users.noreply.github.com> --- Tests/test_pyarrow.py | 4 ++-- src/libImaging/Storage.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/test_pyarrow.py b/Tests/test_pyarrow.py index 2029f96f5..8dad94fe0 100644 --- a/Tests/test_pyarrow.py +++ b/Tests/test_pyarrow.py @@ -29,7 +29,7 @@ def _test_img_equals_pyarray( px = img.load() assert px is not None if elts_per_pixel > 1 and mask is None: - # have to do element wise comparison when we're comparing + # have to do element-wise comparison when we're comparing # flattened r,g,b,a to a pixel. mask = list(range(elts_per_pixel)) for x in range(0, img.size[0], int(img.size[0] / 10)): @@ -56,7 +56,7 @@ def _test_img_equals_int32_pyarray( px = img.load() assert px is not None if mask is None: - # have to do element wise comparison when we're comparing + # have to do element-wise comparison when we're comparing # flattened rgba in an uint32 to a pixel. mask = list(range(elts_per_pixel)) for x in range(0, img.size[0], int(img.size[0] / 10)): diff --git a/src/libImaging/Storage.c b/src/libImaging/Storage.c index 1a9171a0c..6f0a1bfa3 100644 --- a/src/libImaging/Storage.c +++ b/src/libImaging/Storage.c @@ -757,7 +757,7 @@ ImagingNewArrow( if (strcmp(schema->format, "C") == 0 // uint8 && im->pixelsize == 4 // storage as 32 bpc && schema->n_children == 0 // make sure schema is well formed. - && strcmp(im->arrow_band_format, "C") == 0 // Expected Format + && strcmp(im->arrow_band_format, "C") == 0 // expected format && 4 * pixels == external_array->length) { // expected length // single flat array, interleaved storage. if (ImagingBorrowArrow(im, external_array, 1, array_capsule)) { From 3944db288a5b54ea6171fd1334e517fbcc3c9136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=93=E9=BC=A0?= Date: Sat, 31 May 2025 09:10:45 +0800 Subject: [PATCH 43/49] Update MinGW package names (#8987) --- docs/installation/building-from-source.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/installation/building-from-source.rst b/docs/installation/building-from-source.rst index c72568b20..8988a92ce 100644 --- a/docs/installation/building-from-source.rst +++ b/docs/installation/building-from-source.rst @@ -194,9 +194,9 @@ Many of Pillow's features require external libraries: pacman -S \ mingw-w64-x86_64-gcc \ - mingw-w64-x86_64-python3 \ - mingw-w64-x86_64-python3-pip \ - mingw-w64-x86_64-python3-setuptools + mingw-w64-x86_64-python \ + mingw-w64-x86_64-python-pip \ + mingw-w64-x86_64-python-setuptools Prerequisites are installed on **MSYS2 MinGW 64-bit** with:: From bc4138f1692d718ec9fe7b3b7449dc20d0e2d85e Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 31 May 2025 11:48:49 +1000 Subject: [PATCH 44/49] ubuntu-latest now uses Ubuntu 24.04 --- docs/installation/platform-support.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/installation/platform-support.rst b/docs/installation/platform-support.rst index 93486d034..1071380fd 100644 --- a/docs/installation/platform-support.rst +++ b/docs/installation/platform-support.rst @@ -42,11 +42,13 @@ These platforms are built and tested for every change. | macOS 14 Sonoma | 3.10, 3.11, 3.12, 3.13, | arm64 | | | PyPy3 | | +----------------------------------+----------------------------+---------------------+ -| Ubuntu Linux 22.04 LTS (Jammy) | 3.9, 3.10, 3.11, | x86-64 | -| | 3.12, 3.13, PyPy3 | | +| Ubuntu Linux 22.04 LTS (Jammy) | 3.10 | x86-64 | +----------------------------------+----------------------------+---------------------+ -| Ubuntu Linux 24.04 LTS (Noble) | 3.12 | x86-64, arm64v8, | -| | | ppc64le, s390x | +| Ubuntu Linux 24.04 LTS (Noble) | 3.9, 3.10, 3.11, | x86-64 | +| | 3.12, 3.13, PyPy3 | | +| +----------------------------+---------------------+ +| | 3.12 | arm64v8, ppc64le, | +| | | s390x | +----------------------------------+----------------------------+---------------------+ | Windows Server 2019 | 3.9 | x86 | +----------------------------------+----------------------------+---------------------+ From 9327e425ba77523ec9d98eb9558806ecf29b9365 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 31 May 2025 12:02:16 +1000 Subject: [PATCH 45/49] Stop testing deprecated Windows Server 2019 --- .github/workflows/test-windows.yml | 5 ++--- docs/installation/platform-support.rst | 6 +++--- winbuild/README.md | 3 +-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index bfa4c7cd3..6b76351b0 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -31,16 +31,15 @@ env: jobs: build: - runs-on: ${{ matrix.os }} + runs-on: windows-latest strategy: fail-fast: false matrix: python-version: ["pypy3.11", "pypy3.10", "3.10", "3.11", "3.12", "3.13", "3.14"] architecture: ["x64"] - os: ["windows-latest"] include: # Test the oldest Python on 32-bit - - { python-version: "3.9", architecture: "x86", os: "windows-2019" } + - { python-version: "3.9", architecture: "x86" } timeout-minutes: 45 diff --git a/docs/installation/platform-support.rst b/docs/installation/platform-support.rst index 93486d034..f262d861c 100644 --- a/docs/installation/platform-support.rst +++ b/docs/installation/platform-support.rst @@ -48,9 +48,9 @@ These platforms are built and tested for every change. | Ubuntu Linux 24.04 LTS (Noble) | 3.12 | x86-64, arm64v8, | | | | ppc64le, s390x | +----------------------------------+----------------------------+---------------------+ -| Windows Server 2019 | 3.9 | x86 | -+----------------------------------+----------------------------+---------------------+ -| Windows Server 2022 | 3.10, 3.11, 3.12, 3.13, | x86-64 | +| Windows Server 2022 | 3.9 | x86 | +| +----------------------------+---------------------+ +| | 3.10, 3.11, 3.12, 3.13, | x86-64 | | | PyPy3 | | | +----------------------------+---------------------+ | | 3.12 (MinGW) | x86-64 | diff --git a/winbuild/README.md b/winbuild/README.md index c474f12ce..0d3ec8d8a 100644 --- a/winbuild/README.md +++ b/winbuild/README.md @@ -11,8 +11,7 @@ For more extensive info, see the [Windows build instructions](build.rst). * Requires Microsoft Visual Studio 2017 or newer with C++ component. * Requires NASM for libjpeg-turbo, a required dependency when using this script. * Requires CMake 3.15 or newer (available as Visual Studio component). -* Tested on Windows Server 2022 with Visual Studio 2022 Enterprise and Windows Server - 2019 with Visual Studio 2019 Enterprise (GitHub Actions). +* Tested on Windows Server 2022 with Visual Studio 2022 Enterprise (GitHub Actions). Here's an example script to build on Windows: From 892fd2c2affa4980121a059bc2b7875834571804 Mon Sep 17 00:00:00 2001 From: Andrew Murray <3112309+radarhere@users.noreply.github.com> Date: Sun, 1 Jun 2025 15:41:48 +1000 Subject: [PATCH 46/49] Removed unreachable code (#8918) --- src/PIL/MpegImagePlugin.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/PIL/MpegImagePlugin.py b/src/PIL/MpegImagePlugin.py index 5aa00d05b..47ebe9d62 100644 --- a/src/PIL/MpegImagePlugin.py +++ b/src/PIL/MpegImagePlugin.py @@ -33,11 +33,7 @@ class BitStream: def peek(self, bits: int) -> int: while self.bits < bits: - c = self.next() - if c < 0: - self.bits = 0 - continue - self.bitbuffer = (self.bitbuffer << 8) + c + self.bitbuffer = (self.bitbuffer << 8) + self.next() self.bits += 8 return self.bitbuffer >> (self.bits - bits) & (1 << bits) - 1 From 95603e9717c81d3492933c3a8d094bfbb7e90340 Mon Sep 17 00:00:00 2001 From: Andrew Murray <3112309+radarhere@users.noreply.github.com> Date: Mon, 2 Jun 2025 20:14:11 +1000 Subject: [PATCH 47/49] Use ImageFile.MAXBLOCK in tobytes() (#8906) --- src/PIL/Image.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/PIL/Image.py b/src/PIL/Image.py index aaa3332ee..ed2f728aa 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -802,7 +802,9 @@ class Image: e = _getencoder(self.mode, encoder_name, encoder_args) e.setimage(self.im) - bufsize = max(65536, self.size[0] * 4) # see RawEncode.c + from . import ImageFile + + bufsize = max(ImageFile.MAXBLOCK, self.size[0] * 4) # see RawEncode.c output = [] while True: From 070e1eba626736a5cfa4a90a8a97dfbbf6278b91 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Jun 2025 14:08:24 +1000 Subject: [PATCH 48/49] [pre-commit.ci] pre-commit autoupdate (#8993) --- .pre-commit-config.yaml | 8 ++++---- src/_imaging.c | 9 +++++---- src/display.c | 8 ++++---- src/libImaging/Fill.c | 5 +++-- src/libImaging/Filter.c | 6 ++++-- src/libImaging/Jpeg2KEncode.c | 8 ++++---- src/libImaging/Point.c | 5 +++-- src/libImaging/Resample.c | 6 ++++-- src/libImaging/Storage.c | 5 +++-- src/libImaging/TiffDecode.c | 3 ++- 10 files changed, 36 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e15e6f639..a1a054e00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: v0.11.12 hooks: - id: ruff args: [--exit-non-zero-on-fix] @@ -24,7 +24,7 @@ repos: exclude: (Makefile$|\.bat$|\.cmake$|\.eps$|\.fits$|\.gd$|\.opt$) - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v20.1.3 + rev: v20.1.5 hooks: - id: clang-format types: [c] @@ -58,7 +58,7 @@ repos: - id: check-renovate - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.6.0 + rev: v1.9.0 hooks: - id: zizmor @@ -68,7 +68,7 @@ repos: - id: sphinx-lint - repo: https://github.com/tox-dev/pyproject-fmt - rev: v2.5.1 + rev: v2.6.0 hooks: - id: pyproject-fmt diff --git a/src/_imaging.c b/src/_imaging.c index 79e0a2b23..9213ba13d 100644 --- a/src/_imaging.c +++ b/src/_imaging.c @@ -308,9 +308,9 @@ _new_arrow(PyObject *self, PyObject *args) { } // ImagingBorrowArrow is responsible for retaining the array_capsule - ret = - PyImagingNew(ImagingNewArrow(mode, xsize, ysize, schema_capsule, array_capsule) - ); + ret = PyImagingNew( + ImagingNewArrow(mode, xsize, ysize, schema_capsule, array_capsule) + ); if (!ret) { return ImagingError_ValueError("Invalid Arrow array mode or size mismatch"); } @@ -1665,7 +1665,8 @@ _putdata(ImagingObject *self, PyObject *args) { int bigendian = 0; if (image->type == IMAGING_TYPE_SPECIAL) { // I;16* - if (strcmp(image->mode, "I;16B") == 0 + if ( + strcmp(image->mode, "I;16B") == 0 #ifdef WORDS_BIGENDIAN || strcmp(image->mode, "I;16N") == 0 #endif diff --git a/src/display.c b/src/display.c index 11742a895..3215f6691 100644 --- a/src/display.c +++ b/src/display.c @@ -327,11 +327,11 @@ PyImaging_GrabScreenWin32(PyObject *self, PyObject *args) { // added in Windows 10 (1607) // loaded dynamically to avoid link errors user32 = LoadLibraryA("User32.dll"); - SetThreadDpiAwarenessContext_function = (Func_SetThreadDpiAwarenessContext - )GetProcAddress(user32, "SetThreadDpiAwarenessContext"); + SetThreadDpiAwarenessContext_function = (Func_SetThreadDpiAwarenessContext) + GetProcAddress(user32, "SetThreadDpiAwarenessContext"); if (SetThreadDpiAwarenessContext_function != NULL) { - GetWindowDpiAwarenessContext_function = (Func_GetWindowDpiAwarenessContext - )GetProcAddress(user32, "GetWindowDpiAwarenessContext"); + GetWindowDpiAwarenessContext_function = (Func_GetWindowDpiAwarenessContext) + GetProcAddress(user32, "GetWindowDpiAwarenessContext"); if (screens == -1 && GetWindowDpiAwarenessContext_function != NULL) { dpiAwareness = GetWindowDpiAwarenessContext_function(wnd); } diff --git a/src/libImaging/Fill.c b/src/libImaging/Fill.c index 8fb481e7e..28f427370 100644 --- a/src/libImaging/Fill.c +++ b/src/libImaging/Fill.c @@ -118,8 +118,9 @@ ImagingFillRadialGradient(const char *mode) { for (y = 0; y < 256; y++) { for (x = 0; x < 256; x++) { - d = (int - )sqrt((double)((x - 128) * (x - 128) + (y - 128) * (y - 128)) * 2.0); + d = (int)sqrt( + (double)((x - 128) * (x - 128) + (y - 128) * (y - 128)) * 2.0 + ); if (d >= 255) { d = 255; } diff --git a/src/libImaging/Filter.c b/src/libImaging/Filter.c index 7b7b2e429..c46dd3cd1 100644 --- a/src/libImaging/Filter.c +++ b/src/libImaging/Filter.c @@ -155,7 +155,8 @@ ImagingFilter3x3(Imaging imOut, Imaging im, const float *kernel, float offset) { } else { int bigendian = 0; if (im->type == IMAGING_TYPE_SPECIAL) { - if (strcmp(im->mode, "I;16B") == 0 + if ( + strcmp(im->mode, "I;16B") == 0 #ifdef WORDS_BIGENDIAN || strcmp(im->mode, "I;16N") == 0 #endif @@ -308,7 +309,8 @@ ImagingFilter5x5(Imaging imOut, Imaging im, const float *kernel, float offset) { } else { int bigendian = 0; if (im->type == IMAGING_TYPE_SPECIAL) { - if (strcmp(im->mode, "I;16B") == 0 + if ( + strcmp(im->mode, "I;16B") == 0 #ifdef WORDS_BIGENDIAN || strcmp(im->mode, "I;16N") == 0 #endif diff --git a/src/libImaging/Jpeg2KEncode.c b/src/libImaging/Jpeg2KEncode.c index 34d1a2294..61e095ad6 100644 --- a/src/libImaging/Jpeg2KEncode.c +++ b/src/libImaging/Jpeg2KEncode.c @@ -207,8 +207,8 @@ j2k_set_cinema_params(Imaging im, int components, opj_cparameters_t *params) { if (params->cp_cinema == OPJ_CINEMA4K_24) { float max_rate = - ((float)(components * im->xsize * im->ysize * 8) / (CINEMA_24_CS_LENGTH * 8) - ); + ((float)(components * im->xsize * im->ysize * 8) / + (CINEMA_24_CS_LENGTH * 8)); params->POC[0].tile = 1; params->POC[0].resno0 = 0; @@ -243,8 +243,8 @@ j2k_set_cinema_params(Imaging im, int components, opj_cparameters_t *params) { params->max_comp_size = COMP_24_CS_MAX_LENGTH; } else { float max_rate = - ((float)(components * im->xsize * im->ysize * 8) / (CINEMA_48_CS_LENGTH * 8) - ); + ((float)(components * im->xsize * im->ysize * 8) / + (CINEMA_48_CS_LENGTH * 8)); for (n = 0; n < params->tcp_numlayers; ++n) { rate = 0; diff --git a/src/libImaging/Point.c b/src/libImaging/Point.c index 6a4060b4b..b11ea62ed 100644 --- a/src/libImaging/Point.c +++ b/src/libImaging/Point.c @@ -197,8 +197,9 @@ ImagingPoint(Imaging imIn, const char *mode, const void *table) { return imOut; mode_mismatch: - return (Imaging - )ImagingError_ValueError("point operation not supported for this mode"); + return (Imaging)ImagingError_ValueError( + "point operation not supported for this mode" + ); } Imaging diff --git a/src/libImaging/Resample.c b/src/libImaging/Resample.c index f5e386dc2..b114e0023 100644 --- a/src/libImaging/Resample.c +++ b/src/libImaging/Resample.c @@ -470,7 +470,8 @@ ImagingResampleHorizontal_16bpc( double *k; int bigendian = 0; - if (strcmp(imIn->mode, "I;16N") == 0 + if ( + strcmp(imIn->mode, "I;16N") == 0 #ifdef WORDS_BIGENDIAN || strcmp(imIn->mode, "I;16B") == 0 #endif @@ -509,7 +510,8 @@ ImagingResampleVertical_16bpc( double *k; int bigendian = 0; - if (strcmp(imIn->mode, "I;16N") == 0 + if ( + strcmp(imIn->mode, "I;16N") == 0 #ifdef WORDS_BIGENDIAN || strcmp(imIn->mode, "I;16B") == 0 #endif diff --git a/src/libImaging/Storage.c b/src/libImaging/Storage.c index 6f0a1bfa3..11d6c06cc 100644 --- a/src/libImaging/Storage.c +++ b/src/libImaging/Storage.c @@ -602,8 +602,9 @@ ImagingBorrowArrow( } if (!borrowed_buffer) { - return (Imaging - )ImagingError_ValueError("Arrow Array, exactly 2 buffers required"); + return (Imaging)ImagingError_ValueError( + "Arrow Array, exactly 2 buffers required" + ); } for (y = i = 0; y < im->ysize; y++) { diff --git a/src/libImaging/TiffDecode.c b/src/libImaging/TiffDecode.c index 173eca160..2e83fb847 100644 --- a/src/libImaging/TiffDecode.c +++ b/src/libImaging/TiffDecode.c @@ -557,7 +557,8 @@ _decodeStrip( (tdata_t)state->buffer, strip_size ) == -1) { - TRACE(("Decode Error, strip %d\n", TIFFComputeStrip(tiff, state->y, 0)) + TRACE( + ("Decode Error, strip %d\n", TIFFComputeStrip(tiff, state->y, 0)) ); state->errcode = IMAGING_CODEC_BROKEN; return -1; From eb0256acc082e362b4172f3256a551a412ef4b09 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 3 Jun 2025 22:44:26 +1000 Subject: [PATCH 49/49] Fixed test --- Tests/test_deprecate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/test_deprecate.py b/Tests/test_deprecate.py index 82ff14181..88479ff0d 100644 --- a/Tests/test_deprecate.py +++ b/Tests/test_deprecate.py @@ -47,7 +47,6 @@ def test_unknown_version() -> None: ], ) def test_old_version(deprecated: str, plural: bool, expected: str) -> None: - expected = r"" with pytest.raises(RuntimeError, match=expected): _deprecate.deprecate(deprecated, 1, plural=plural)