Merge branch 'master' into core_tags

This commit is contained in:
Andrew Murray 2021-04-11 12:46:37 +10:00
commit 8be655bec6
36 changed files with 198 additions and 54 deletions

View File

@ -2,6 +2,12 @@
Changelog (Pillow)
==================
8.3.0 (unreleased)
------------------
- Fixed reading uncompressed RGB data from DDS #5383
[radarhere]
8.2.0 (2021-04-01)
------------------

View File

@ -102,6 +102,13 @@ sdist:
test:
pytest -qq
.PHONY: valgrind
valgrind:
python3 -c "import pytest_valgrind" || pip3 install pytest-valgrind
PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \
--log-file=/tmp/valgrind-output \
python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output
.PHONY: readme
readme:
python3 setup.py --long-description | markdown2 > .long-description.html && open .long-description.html

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import sys

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
from PIL import Image

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import pytest
from PIL import Image

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Reproductions/tests for OOB read errors in FliDecode.c

View File

@ -13,6 +13,11 @@ def pytest_report_header(config):
def pytest_configure(config):
config.addinivalue_line(
"markers",
"pil_noop_mark: A conditional mark where nothing special happens",
)
# We're marking some tests to ignore valgrind errors and XFAIL them.
# Ensure that the mark is defined
# even in cases where pytest-valgrind isn't installed

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import base64
import os

View File

@ -173,6 +173,21 @@ def skip_unless_feature_version(feature, version_required, reason=None):
return pytest.mark.skipif(version_available < version_required, reason=reason)
def mark_if_feature_version(mark, feature, version_blacklist, reason=None):
if not features.check(feature):
return pytest.mark.pil_noop_mark()
if reason is None:
reason = f"{feature} is {version_blacklist}"
version_required = parse_version(version_blacklist)
version_available = parse_version(features.version(feature))
if (
version_available.major == version_required.major
and version_available.minor == version_required.minor
):
return mark(reason=reason)
return pytest.mark.pil_noop_mark()
@pytest.mark.skipif(sys.platform.startswith("win32"), reason="Requires Unix or macOS")
class PillowLeakTestCase:
# requires unix/macOS

BIN
Tests/images/hopper.dds Normal file

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 953 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 143 KiB

After

Width:  |  Height:  |  Size: 143 KiB

View File

@ -0,0 +1,16 @@
{
<py3_8_encode_current_locale>
Memcheck:Cond
...
fun:encode_current_locale
}
{
<libtiff_zlib>
Memcheck:Cond
fun:inflate
fun:ZIPDecode
fun:_TIFFReadEncodedTileAndAllocBuffer
...
}

View File

@ -2,12 +2,19 @@ import subprocess
import sys
import fuzzers
import packaging
import pytest
from PIL import Image
from PIL import Image, features
if sys.platform.startswith("win32"):
pytest.skip("Fuzzer is linux only", allow_module_level=True)
if features.check("libjpeg_turbo"):
version = packaging.version.parse(features.version("libjpeg_turbo"))
if version.major == 2 and version.minor == 0:
pytestmark = pytest.mark.valgrind_known_error(
reason="Known failing with libjpeg_turbo 2.0"
)
@pytest.mark.parametrize(

View File

@ -14,7 +14,8 @@ TEST_FILE_DX10_BC7 = "Tests/images/bc7-argb-8bpp_MipMaps-1.dds"
TEST_FILE_DX10_BC7_UNORM_SRGB = "Tests/images/DXGI_FORMAT_BC7_UNORM_SRGB.dds"
TEST_FILE_DX10_R8G8B8A8 = "Tests/images/argb-32bpp_MipMaps-1.dds"
TEST_FILE_DX10_R8G8B8A8_UNORM_SRGB = "Tests/images/DXGI_FORMAT_R8G8B8A8_UNORM_SRGB.dds"
TEST_FILE_UNCOMPRESSED_RGB = "Tests/images/uncompressed_rgb.dds"
TEST_FILE_UNCOMPRESSED_RGB = "Tests/images/hopper.dds"
TEST_FILE_UNCOMPRESSED_RGB_WITH_ALPHA = "Tests/images/uncompressed_rgb.dds"
def test_sanity_dxt1():
@ -124,15 +125,22 @@ def test_unimplemented_dxgi_format():
def test_uncompressed_rgb():
"""Check uncompressed RGB images can be opened"""
# convert -format dds -define dds:compression=none hopper.jpg hopper.dds
with Image.open(TEST_FILE_UNCOMPRESSED_RGB) as im:
im.load()
assert im.format == "DDS"
assert im.mode == "RGB"
assert im.size == (128, 128)
assert_image_equal_tofile(im, "Tests/images/hopper.png")
# Test image with alpha
with Image.open(TEST_FILE_UNCOMPRESSED_RGB_WITH_ALPHA) as im:
assert im.format == "DDS"
assert im.mode == "RGBA"
assert im.size == (800, 600)
assert_image_equal_tofile(
im, TEST_FILE_UNCOMPRESSED_RGB.replace(".dds", ".png")
im, TEST_FILE_UNCOMPRESSED_RGB_WITH_ALPHA.replace(".dds", ".png")
)

View File

@ -8,6 +8,7 @@ from .helper import (
assert_image_similar,
assert_image_similar_tofile,
hopper,
mark_if_feature_version,
skip_unless_feature,
)
@ -64,7 +65,9 @@ def test_invalid_file():
EpsImagePlugin.EpsImageFile(invalid_file)
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
@pytest.mark.skipif(not HAS_GHOSTSCRIPT, reason="Ghostscript not available")
def test_cmyk():
with Image.open("Tests/images/pil_sample_cmyk.eps") as cmyk_image:

View File

@ -24,6 +24,7 @@ from .helper import (
djpeg_available,
hopper,
is_win32,
mark_if_feature_version,
skip_unless_feature,
)
@ -116,7 +117,9 @@ class TestFileJpeg:
assert test(100, 200) == (100, 200)
assert test(0) is None # square pixels
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_icc(self, tmp_path):
# Test ICC support
with Image.open("Tests/images/rgb.jpg") as im1:
@ -156,7 +159,9 @@ class TestFileJpeg:
test(ImageFile.MAXBLOCK + 1) # full buffer block plus one byte
test(ImageFile.MAXBLOCK * 4 + 3) # large block
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_large_icc_meta(self, tmp_path):
# https://github.com/python-pillow/Pillow/issues/148
# Sometimes the meta data on the icc_profile block is bigger than
@ -423,7 +428,9 @@ class TestFileJpeg:
with Image.open(filename):
pass
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_truncated_jpeg_should_read_all_the_data(self):
filename = "Tests/images/truncated_jpeg.jpg"
ImageFile.LOAD_TRUNCATED_IMAGES = True
@ -442,7 +449,9 @@ class TestFileJpeg:
with pytest.raises(OSError):
im.load()
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_qtables(self, tmp_path):
def _n_qtables_helper(n, test_file):
with Image.open(test_file) as im:
@ -726,7 +735,9 @@ class TestFileJpeg:
# OSError for unidentified image.
assert im.info.get("dpi") == (72, 72)
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_exif_x_resolution(self, tmp_path):
with Image.open("Tests/images/flower.jpg") as im:
exif = im.getexif()
@ -757,7 +768,9 @@ class TestFileJpeg:
# Act / Assert
assert im._getexif()[306] == "2017:03:13 23:03:09"
@pytest.mark.valgrind_known_error(reason="Backtrace in Python Core")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_photoshop(self):
with Image.open("Tests/images/photoshop-200dpi.jpg") as im:
assert im.info["photoshop"][0x03ED] == {

View File

@ -17,6 +17,7 @@ from .helper import (
assert_image_similar,
assert_image_similar_tofile,
hopper,
mark_if_feature_version,
skip_unless_feature,
)
@ -822,13 +823,17 @@ class TestFileLibTiff(LibTiffTestCase):
with Image.open(infile) as im:
assert_image_similar_tofile(im, "Tests/images/pil_sample_cmyk.jpg", 0.5)
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_strip_ycbcr_jpeg_2x2_sampling(self):
infile = "Tests/images/tiff_strip_ycbcr_jpeg_2x2_sampling.tif"
with Image.open(infile) as im:
assert_image_similar_tofile(im, "Tests/images/flower.jpg", 0.5)
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_strip_ycbcr_jpeg_1x1_sampling(self):
infile = "Tests/images/tiff_strip_ycbcr_jpeg_1x1_sampling.tif"
with Image.open(infile) as im:
@ -839,13 +844,17 @@ class TestFileLibTiff(LibTiffTestCase):
with Image.open(infile) as im:
assert_image_similar_tofile(im, "Tests/images/pil_sample_cmyk.jpg", 0.5)
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_tiled_ycbcr_jpeg_1x1_sampling(self):
infile = "Tests/images/tiff_tiled_ycbcr_jpeg_1x1_sampling.tif"
with Image.open(infile) as im:
assert_image_equal_tofile(im, "Tests/images/flower2.jpg")
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_tiled_ycbcr_jpeg_2x2_sampling(self):
infile = "Tests/images/tiff_tiled_ycbcr_jpeg_2x2_sampling.tif"
with Image.open(infile) as im:

View File

@ -8,7 +8,7 @@ import pytest
from PIL import Image, PdfParser
from .helper import hopper
from .helper import hopper, mark_if_feature_version
def helper_save_as_pdf(tmp_path, mode, **kwargs):
@ -85,7 +85,9 @@ def test_unsupported_mode(tmp_path):
im.save(outfile)
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_save_all(tmp_path):
# Single frame image
helper_save_as_pdf(tmp_path, "RGB", save_all=True)
@ -286,3 +288,13 @@ def test_pdf_append_to_bytesio():
f = io.BytesIO(f.getvalue())
im.save(f, format="PDF", append=True)
assert len(f.getvalue()) > initial_size
@pytest.mark.timeout(1)
def test_redos():
malicious = b" trailer<<>>" + b"\n" * 3456
# This particular exception isn't relevant here.
# The important thing is it doesn't timeout, cause a ReDoS (CVE-2021-25292).
with pytest.raises(PdfParser.PdfFormatError):
PdfParser.PdfParser(buf=malicious)

View File

@ -13,6 +13,7 @@ from .helper import (
hopper,
is_big_endian,
is_win32,
mark_if_feature_version,
skip_unless_feature,
)
@ -679,7 +680,9 @@ class TestFilePng:
exif = reloaded._getexif()
assert exif[274] == 1
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_exif_from_jpg(self, tmp_path):
with Image.open("Tests/images/pil_sample_rgb.jpg") as im:
test_file = str(tmp_path / "temp.png")

View File

@ -4,7 +4,7 @@ import pytest
from PIL import Image
from .helper import skip_unless_feature
from .helper import mark_if_feature_version, skip_unless_feature
pytestmark = [
skip_unless_feature("webp"),
@ -41,7 +41,9 @@ def test_read_exif_metadata_without_prefix():
assert exif[305] == "Adobe Photoshop CS6 (Macintosh)"
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_write_exif_metadata():
file_path = "Tests/images/flower.jpg"
test_buffer = BytesIO()
@ -74,7 +76,9 @@ def test_read_icc_profile():
assert icc == expected_icc
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_write_icc_metadata():
file_path = "Tests/images/flower2.jpg"
test_buffer = BytesIO()
@ -92,7 +96,9 @@ def test_write_icc_metadata():
assert webp_icc_profile == expected_icc_profile, "Webp ICC didn't match"
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_read_no_exif():
file_path = "Tests/images/flower.jpg"
test_buffer = BytesIO()

View File

@ -16,6 +16,7 @@ from .helper import (
assert_not_all_same,
hopper,
is_win32,
mark_if_feature_version,
skip_unless_feature,
)
@ -662,7 +663,9 @@ class TestImage:
assert not fp.closed
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_exif_jpeg(self, tmp_path):
with Image.open("Tests/images/exif-72dpi-int.jpg") as im: # Little endian
exif = im.getexif()

View File

@ -4,7 +4,12 @@ import pytest
from PIL import Image, ImageDraw
from .helper import assert_image_equal, assert_image_similar, hopper
from .helper import (
assert_image_equal,
assert_image_similar,
hopper,
mark_if_feature_version,
)
class TestImagingResampleVulnerability:
@ -455,7 +460,9 @@ class TestCoreResampleBox:
tiled.paste(tile, (x0, y0))
return tiled
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_tiles(self):
with Image.open("Tests/images/flower.jpg") as im:
assert im.size == (480, 360)
@ -466,7 +473,9 @@ class TestCoreResampleBox:
tiled = self.resize_tiled(im, dst_size, *tiles)
assert_image_similar(reference, tiled, 0.01)
@pytest.mark.valgrind_known_error(reason="Known Failing")
@mark_if_feature_version(
pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
)
def test_subsample(self):
# This test shows advantages of the subpixel resizing
# after supersampling (e.g. during JPEG decoding).

View File

@ -88,6 +88,7 @@ def test_no_resize():
assert im.size == (64, 64)
# valgrind test is failing with memory allocated in libjpeg
@pytest.mark.valgrind_known_error(reason="Known Failing")
def test_DCT_scaling_edges():
# Make an image with red borders and size (N * 8) + 1 to cross DCT grid

View File

@ -722,6 +722,29 @@ def test_rounded_rectangle(xy):
assert_image_equal_tofile(im, "Tests/images/imagedraw_rounded_rectangle.png")
@pytest.mark.parametrize(
"xy, radius, type",
[
((10, 20, 190, 180), 30.5, "given"),
((10, 10, 181, 190), 90, "width"),
((10, 20, 190, 181), 85, "height"),
],
)
def test_rounded_rectangle_non_integer_radius(xy, radius, type):
# Arrange
im = Image.new("RGB", (200, 200))
draw = ImageDraw.Draw(im)
# Act
draw.rounded_rectangle(xy, radius, fill="red", outline="green", width=5)
# Assert
assert_image_equal_tofile(
im,
"Tests/images/imagedraw_rounded_rectangle_non_integer_radius_" + type + ".png",
)
def test_rounded_rectangle_zero_radius():
# Arrange
im = Image.new("RGB", (W, H))

View File

@ -1,4 +1,3 @@
#!/usr/bin/env python
import pytest
from PIL import Image

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
from livereload.compiler import shell
from livereload.task import Task

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# minimal sanity check
import sys

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# > pyroma .
# ------------------------------
# Checking .

View File

@ -133,9 +133,11 @@ class DdsImageFile(ImageFile.ImageFile):
rawmode = ""
if bitcount == 32:
rawmode += masks[0xFF000000]
else:
self.mode = "RGB"
rawmode += masks[0xFF0000] + masks[0xFF00] + masks[0xFF]
self.tile = [("raw", (0, 0) + self.size, 0, (rawmode, 0, 1))]
self.tile = [("raw", (0, 0) + self.size, 0, (rawmode[::-1], 0, 1))]
else:
data_start = header_size + 4
n = 0

View File

@ -282,6 +282,7 @@ class ImageDraw:
# If the corners have no curve, that is a rectangle
return self.rectangle(xy, fill, outline, width)
r = d // 2
ink, fill = self._getink(outline, fill)
def draw_corners(pieslice):
@ -315,36 +316,28 @@ class ImageDraw:
draw_corners(True)
if full_x:
self.draw.draw_rectangle(
(x0, y0 + d / 2 + 1, x1, y1 - d / 2 - 1), fill, 1
)
self.draw.draw_rectangle((x0, y0 + r + 1, x1, y1 - r - 1), fill, 1)
else:
self.draw.draw_rectangle(
(x0 + d / 2 + 1, y0, x1 - d / 2 - 1, y1), fill, 1
)
self.draw.draw_rectangle((x0 + r + 1, y0, x1 - r - 1, y1), fill, 1)
if not full_x and not full_y:
self.draw.draw_rectangle(
(x0, y0 + d / 2 + 1, x0 + d / 2, y1 - d / 2 - 1), fill, 1
)
self.draw.draw_rectangle(
(x1 - d / 2, y0 + d / 2 + 1, x1, y1 - d / 2 - 1), fill, 1
)
self.draw.draw_rectangle((x0, y0 + r + 1, x0 + r, y1 - r - 1), fill, 1)
self.draw.draw_rectangle((x1 - r, y0 + r + 1, x1, y1 - r - 1), fill, 1)
if ink is not None and ink != fill and width != 0:
draw_corners(False)
if not full_x:
self.draw.draw_rectangle(
(x0 + d / 2 + 1, y0, x1 - d / 2 - 1, y0 + width - 1), ink, 1
(x0 + r + 1, y0, x1 - r - 1, y0 + width - 1), ink, 1
)
self.draw.draw_rectangle(
(x0 + d / 2 + 1, y1 - width + 1, x1 - d / 2 - 1, y1), ink, 1
(x0 + r + 1, y1 - width + 1, x1 - r - 1, y1), ink, 1
)
if not full_y:
self.draw.draw_rectangle(
(x0, y0 + d / 2 + 1, x0 + width - 1, y1 - d / 2 - 1), ink, 1
(x0, y0 + r + 1, x0 + width - 1, y1 - r - 1), ink, 1
)
self.draw.draw_rectangle(
(x1 - width + 1, y0 + d / 2 + 1, x1, y1 - d / 2 - 1), ink, 1
(x1 - width + 1, y0 + r + 1, x1, y1 - r - 1), ink, 1
)
def _multiline_check(self, text):

View File

@ -199,7 +199,7 @@ _setimage(ImagingDecoderObject *decoder, PyObject *args) {
state->bytes = (state->bits * state->xsize + 7) / 8;
}
/* malloc check ok, overflow checked above */
state->buffer = (UINT8 *)malloc(state->bytes);
state->buffer = (UINT8 *)calloc(1, state->bytes);
if (!state->buffer) {
return ImagingError_MemoryError();
}

View File

@ -264,7 +264,7 @@ _setimage(ImagingEncoderObject *encoder, PyObject *args) {
}
state->bytes = (state->bits * state->xsize + 7) / 8;
/* malloc check ok, overflow checked above */
state->buffer = (UINT8 *)malloc(state->bytes);
state->buffer = (UINT8 *)calloc(1, state->bytes);
if (!state->buffer) {
return ImagingError_MemoryError();
}

View File

@ -861,6 +861,10 @@ j2k_decode_entry(Imaging im, ImagingCodecState state) {
state->state = J2K_STATE_FAILED;
goto quick_exit;
}
/* Undefined behavior, sometimes decode_tile_data doesn't
fill the buffer and we do things with it later, leading
to valgrind errors. */
memset(new, 0, tile_info.data_size);
state->buffer = new;
buffer_size = tile_info.data_size;
}