From 9f6c708aa3961f9bc768d3bd5b95c1821d5ec6d1 Mon Sep 17 00:00:00 2001 From: Pedro Lacerda Date: Mon, 9 Jul 2018 04:33:00 -0300 Subject: [PATCH 001/164] Simplify library dirs search with ldconfig --- setup.py | 144 +++++++++++++++++++++++-------------------------------- 1 file changed, 61 insertions(+), 83 deletions(-) diff --git a/setup.py b/setup.py index 9529787f9..dd31e874d 100755 --- a/setup.py +++ b/setup.py @@ -74,6 +74,57 @@ def _dbg(s, tp=None): print(s) +def _find_library_dirs_ldconfig(): + # Based on ctypes.util from Python 2 + + if sys.platform.startswith("linux") or sys.platform.startswith("gnu"): + if struct.calcsize('l') == 4: + machine = os.uname()[4] + '-32' + else: + machine = os.uname()[4] + '-64' + mach_map = { + 'x86_64-64': 'libc6,x86-64', + 'ppc64-64': 'libc6,64bit', + 'sparc64-64': 'libc6,64bit', + 's390x-64': 'libc6,64bit', + 'ia64-64': 'libc6,IA-64', + } + abi_type = mach_map.get(machine, 'libc6') + + # XXX assuming GLIBC's ldconfig (with option -p) + # XXX Alpine Linux uses musl that can't print cache + args = ['/sbin/ldconfig', '-p'] + expr = r'.*\(%s.*\) => (.*)' % abi_type + env = dict(os.environ) + env['LC_ALL'] = 'C' + env['LANG'] = 'C' + + elif sys.platform.startswith("freebsd"): + args = ['/sbin/ldconfig', '-r'] + expr = r'.* => (.*)' + env = {} + + null = open(os.devnull, 'wb') + try: + with null: + p = subprocess.Popen(args, + stderr=null, + stdout=subprocess.PIPE, + env=env) + except OSError: # E.g. command not found + return None + [data, _] = p.communicate() + if isinstance(data, bytes): + data = data.decode() + + dirs = [] + for dll in re.findall(expr, data): + dir = os.path.dirname(dll) + if dir not in dirs: + dirs.append(dir) + return dirs + + def _add_directory(path, subdir, where=None): if subdir is None: return @@ -333,68 +384,20 @@ class pil_build_ext(build_ext): _add_directory(library_dirs, "/usr/X11/lib") _add_directory(include_dirs, "/usr/X11/include") - elif sys.platform.startswith("linux"): - arch_tp = (plat.processor(), plat.architecture()[0]) - # This should be correct on debian derivatives. - if os.path.exists('/etc/debian_version'): - # If this doesn't work, don't just silently patch - # downstream because it's going to break when people - # try to build pillow from source instead of - # installing from the system packages. - self.add_multiarch_paths() - - elif arch_tp == ("x86_64", "32bit"): - # Special Case: 32-bit build on 64-bit machine. - _add_directory(library_dirs, "/usr/lib/i386-linux-gnu") - else: - libdirs = { - 'x86_64': ["/lib64", "/usr/lib64", - "/usr/lib/x86_64-linux-gnu"], - '64bit': ["/lib64", "/usr/lib64", - "/usr/lib/x86_64-linux-gnu"], - 'i386': ["/usr/lib/i386-linux-gnu"], - 'i686': ["/usr/lib/i386-linux-gnu"], - '32bit': ["/usr/lib/i386-linux-gnu"], - 'aarch64': ["/usr/lib64", "/usr/lib/aarch64-linux-gnu"], - 'arm': ["/usr/lib/arm-linux-gnueabi"], - 'armv71': ["/usr/lib/arm-linux-gnueabi"], - 'armv7l': ["/usr/lib"], - 'ppc64': ["/usr/lib64", "/usr/lib/ppc64-linux-gnu", - "/usr/lib/powerpc64-linux-gnu"], - 'ppc64le': ["/usr/lib64"], - 'ppc': ["/usr/lib/ppc-linux-gnu", - "/usr/lib/powerpc-linux-gnu"], - 's390x': ["/usr/lib64", "/usr/lib/s390x-linux-gnu"], - 's390': ["/usr/lib/s390-linux-gnu"], - } - - for platform_ in arch_tp: - dirs = libdirs.get(platform_, None) - if not dirs: - continue - for path in dirs: - _add_directory(library_dirs, path) - break - - else: - raise ValueError( - "Unable to identify Linux platform: `%s`" % platform_) - + elif sys.platform.startswith("linux") or \ + sys.platform.startswith("gnu") or \ + sys.platform.startswith("freebsd"): + for dirname in _find_library_dirs_ldconfig(): + _add_directory(library_dirs, dirname) + if sys.platform.startswith("linux") and \ + os.environ.get('ANDROID_ROOT', None): # termux support for android. # system libraries (zlib) are installed in /system/lib # headers are at $PREFIX/include # user libs are at $PREFIX/lib - if os.environ.get('ANDROID_ROOT', None): - _add_directory(library_dirs, - os.path.join(os.environ['ANDROID_ROOT'], - 'lib')) - - elif sys.platform.startswith("gnu"): - self.add_multiarch_paths() - - elif sys.platform.startswith("freebsd"): - _add_directory(library_dirs, "/usr/local/lib") - _add_directory(include_dirs, "/usr/local/include") + _add_directory(library_dirs, + os.path.join(os.environ['ANDROID_ROOT'], + 'lib')) elif sys.platform.startswith("netbsd"): _add_directory(library_dirs, "/usr/pkg/lib") @@ -728,31 +731,6 @@ class pil_build_ext(build_ext): print("To check the build, run the selftest.py script.") print("") - # https://hg.python.org/users/barry/rev/7e8deab93d5a - def add_multiarch_paths(self): - # Debian/Ubuntu multiarch support. - # https://wiki.ubuntu.com/MultiarchSpec - # self.build_temp - tmpfile = os.path.join(self.build_temp, 'multiarch') - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - with open(tmpfile, 'wb') as fp: - try: - ret = subprocess.call(['dpkg-architecture', - '-qDEB_HOST_MULTIARCH'], stdout=fp) - except Exception: - return - try: - if ret >> 8 == 0: - with open(tmpfile, 'r') as fp: - multiarch_path_component = fp.readline().strip() - _add_directory(self.compiler.library_dirs, - '/usr/lib/' + multiarch_path_component) - _add_directory(self.compiler.include_dirs, - '/usr/include/' + multiarch_path_component) - finally: - os.unlink(tmpfile) - def debug_build(): return hasattr(sys, 'gettotalrefcount') From de2fbc9dffedb69ebb8045dbfcb1614c2711a114 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 16 Jun 2018 19:47:57 +1000 Subject: [PATCH 002/164] Wait until mode is normalized to copy im.info into encoderinfo --- Tests/test_file_gif.py | 21 +++++++++++++++++++++ src/PIL/GifImagePlugin.py | 17 ++++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/Tests/test_file_gif.py b/Tests/test_file_gif.py index 086a0f5d0..75b0c9746 100644 --- a/Tests/test_file_gif.py +++ b/Tests/test_file_gif.py @@ -491,6 +491,27 @@ class TestFileGif(PillowTestCase): self.assertEqual(reloaded.info['transparency'], 253) + def test_rgb_transparency(self): + out = self.tempfile('temp.gif') + + # Single frame + im = Image.new('RGB', (1, 1)) + im.info['transparency'] = (255, 0, 0) + self.assert_warning(UserWarning, im.save, out) + + reloaded = Image.open(out) + self.assertNotIn('transparency', reloaded.info) + + # Multiple frames + im = Image.new('RGB', (1, 1)) + im.info['transparency'] = b"" + ims = [Image.new('RGB', (1, 1))] + self.assert_warning(UserWarning, + im.save, out, save_all=True, append_images=ims) + + reloaded = Image.open(out) + self.assertNotIn('transparency', reloaded.info) + def test_bbox(self): out = self.tempfile('temp.gif') diff --git a/src/PIL/GifImagePlugin.py b/src/PIL/GifImagePlugin.py index fec2f7663..425cc1899 100644 --- a/src/PIL/GifImagePlugin.py +++ b/src/PIL/GifImagePlugin.py @@ -372,6 +372,8 @@ def _normalize_palette(im, palette, info): def _write_single_frame(im, fp, palette): im_out = _normalize_mode(im, True) + for k, v in im_out.info.items(): + im.encoderinfo.setdefault(k, v) im_out = _normalize_palette(im_out, palette, im.encoderinfo) for s in _get_global_header(im_out, im.encoderinfo): @@ -392,8 +394,8 @@ def _write_single_frame(im, fp, palette): def _write_multiple_frames(im, fp, palette): - duration = im.encoderinfo.get("duration", None) - disposal = im.encoderinfo.get('disposal', None) + duration = im.encoderinfo.get("duration", im.info.get("duration")) + disposal = im.encoderinfo.get("disposal", im.info.get("disposal")) im_frames = [] frame_count = 0 @@ -402,6 +404,9 @@ def _write_multiple_frames(im, fp, palette): for im_frame in ImageSequence.Iterator(imSequence): # a copy is required here since seek can still mutate the image im_frame = _normalize_mode(im_frame.copy()) + if frame_count == 0: + for k, v in im_frame.info.items(): + im.encoderinfo.setdefault(k, v) im_frame = _normalize_palette(im_frame, palette, im.encoderinfo) encoderinfo = im.encoderinfo.copy() @@ -460,12 +465,10 @@ def _save_all(im, fp, filename): def _save(im, fp, filename, save_all=False): - for k, v in im.info.items(): - im.encoderinfo.setdefault(k, v) # header - try: - palette = im.encoderinfo["palette"] - except KeyError: + if "palette" in im.encoderinfo or "palette" in im.info: + palette = im.encoderinfo.get("palette", im.info.get("palette")) + else: palette = None im.encoderinfo["optimize"] = im.encoderinfo.get("optimize", True) From 9bdb9d8bef23a1faa244eb0cdae26c169812d720 Mon Sep 17 00:00:00 2001 From: Yann Diorcet Date: Fri, 28 Sep 2018 10:40:31 +0200 Subject: [PATCH 003/164] Fix pkg-config call --- setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 761d552cc..a9e96b836 100755 --- a/setup.py +++ b/setup.py @@ -147,15 +147,20 @@ LCMS_ROOT = None def _pkg_config(name): try: - command = [ + command_libs = [ 'pkg-config', '--libs-only-L', name, + ] + command_cflags = [ + 'pkg-config', '--cflags-only-I', name, ] if not DEBUG: - command.append('--silence-errors') - libs = subprocess.check_output(command).decode('utf8').split(' ') - return libs[1][2:].strip(), libs[0][2:].strip() + command_libs.append('--silence-errors') + command_cflags.append('--silence-errors') + libs = subprocess.check_output(command_libs).decode('utf8').strip().replace('-L', '') + cflags = subprocess.check_output(command_cflags).decode('utf8').strip().replace('-I', '') + return (libs, cflags) except Exception: pass From 2e288e74ab79141765329ce7f473e8530139d235 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 11 Sep 2018 15:58:31 +0200 Subject: [PATCH 004/164] Fix potential un-terminated buffer problem (CWE-120) --- src/libImaging/Histo.c | 4 +++- src/libImaging/Palette.c | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libImaging/Histo.c b/src/libImaging/Histo.c index 0bfc8dfe9..2b35873e8 100644 --- a/src/libImaging/Histo.c +++ b/src/libImaging/Histo.c @@ -41,7 +41,9 @@ ImagingHistogramNew(Imaging im) /* Create histogram descriptor */ h = calloc(1, sizeof(struct ImagingHistogramInstance)); - strncpy(h->mode, im->mode, IMAGING_MODE_LENGTH); + strncpy(h->mode, im->mode, IMAGING_MODE_LENGTH-1); + h->mode[IMAGING_MODE_LENGTH-1] = 0; + h->bands = im->bands; h->histogram = calloc(im->pixelsize, 256 * sizeof(long)); diff --git a/src/libImaging/Palette.c b/src/libImaging/Palette.c index 31c2c0245..7aee6e8ee 100644 --- a/src/libImaging/Palette.c +++ b/src/libImaging/Palette.c @@ -37,7 +37,8 @@ ImagingPaletteNew(const char* mode) if (!palette) return (ImagingPalette) ImagingError_MemoryError(); - strncpy(palette->mode, mode, IMAGING_MODE_LENGTH); + strncpy(palette->mode, mode, IMAGING_MODE_LENGTH-1); + palette->mode[IMAGING_MODE_LENGTH-1] = 0; /* Initialize to ramp */ for (i = 0; i < 256; i++) { From 78bf8ea0413c80f23f016a16c274d31d8bd23f66 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 13 Sep 2018 12:22:11 +0200 Subject: [PATCH 005/164] Fix potential leaked storage issues (CWE-772) --- src/Tk/tkImaging.c | 2 ++ src/_imaging.c | 15 +++++++++++++-- src/encode.c | 12 +++++++++--- src/libImaging/Histo.c | 12 +++++++++--- src/libImaging/Quant.c | 2 ++ src/libImaging/QuantOctree.c | 1 + src/libImaging/Resample.c | 2 ++ src/path.c | 8 ++++++-- 8 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/Tk/tkImaging.c b/src/Tk/tkImaging.c index 66e093fae..991cf1c95 100644 --- a/src/Tk/tkImaging.c +++ b/src/Tk/tkImaging.c @@ -425,6 +425,7 @@ int load_tkinter_funcs(void) /* Try loading from the main program namespace first */ main_program = dlopen(NULL, RTLD_LAZY); if (_func_loader(main_program) == 0) { + dlclose(main_program); return 0; } /* Clear exception triggered when we didn't find symbols above */ @@ -453,6 +454,7 @@ int load_tkinter_funcs(void) /* dlclose probably safe because tkinter has been imported. */ dlclose(tkinter_lib); exit: + dlclose(main_program); Py_XDECREF(pModule); Py_XDECREF(pString); return ret; diff --git a/src/_imaging.c b/src/_imaging.c index e4b31f10b..5afef976b 100644 --- a/src/_imaging.c +++ b/src/_imaging.c @@ -1042,8 +1042,10 @@ _gaussian_blur(ImagingObject* self, PyObject* args) if (!imOut) return NULL; - if (!ImagingGaussianBlur(imOut, imIn, radius, passes)) + if (!ImagingGaussianBlur(imOut, imIn, radius, passes)) { + ImagingDelete(imOut); return NULL; + } return PyImagingNew(imOut); } @@ -1931,8 +1933,10 @@ _box_blur(ImagingObject* self, PyObject* args) if (!imOut) return NULL; - if (!ImagingBoxBlur(imOut, imIn, radius, n)) + if (!ImagingBoxBlur(imOut, imIn, radius, n)) { + ImagingDelete(imOut); return NULL; + } return PyImagingNew(imOut); } @@ -2578,6 +2582,7 @@ _draw_arc(ImagingDrawObject* self, PyObject* args) return NULL; if (n != 2) { PyErr_SetString(PyExc_TypeError, must_be_two_coordinates); + free(xy); return NULL; } @@ -2615,6 +2620,7 @@ _draw_bitmap(ImagingDrawObject* self, PyObject* args) PyErr_SetString(PyExc_TypeError, "coordinate list must contain exactly 1 coordinate" ); + free(xy); return NULL; } @@ -2651,6 +2657,7 @@ _draw_chord(ImagingDrawObject* self, PyObject* args) return NULL; if (n != 2) { PyErr_SetString(PyExc_TypeError, must_be_two_coordinates); + free(xy); return NULL; } @@ -2687,6 +2694,7 @@ _draw_ellipse(ImagingDrawObject* self, PyObject* args) return NULL; if (n != 2) { PyErr_SetString(PyExc_TypeError, must_be_two_coordinates); + free(xy); return NULL; } @@ -2838,6 +2846,7 @@ _draw_pieslice(ImagingDrawObject* self, PyObject* args) return NULL; if (n != 2) { PyErr_SetString(PyExc_TypeError, must_be_two_coordinates); + free(xy); return NULL; } @@ -2876,6 +2885,7 @@ _draw_polygon(ImagingDrawObject* self, PyObject* args) PyErr_SetString(PyExc_TypeError, "coordinate list must contain at least 2 coordinates" ); + free(xy); return NULL; } @@ -2919,6 +2929,7 @@ _draw_rectangle(ImagingDrawObject* self, PyObject* args) return NULL; if (n != 2) { PyErr_SetString(PyExc_TypeError, must_be_two_coordinates); + free(xy); return NULL; } diff --git a/src/encode.c b/src/encode.c index c60048c41..a6fc16e73 100644 --- a/src/encode.c +++ b/src/encode.c @@ -584,11 +584,15 @@ PyImaging_ZipEncoderNew(PyObject* self, PyObject* args) dictionary = NULL; encoder = PyImaging_EncoderNew(sizeof(ZIPSTATE)); - if (encoder == NULL) + if (encoder == NULL) { + free(dictionary); return NULL; + } - if (get_packer(encoder, mode, rawmode) < 0) + if (get_packer(encoder, mode, rawmode) < 0) { + free(dictionary); return NULL; + } encoder->encode = ImagingZipEncode; encoder->cleanup = ImagingZipEncodeCleanup; @@ -749,8 +753,10 @@ PyImaging_JpegEncoderNew(PyObject* self, PyObject* args) if (rawExif && rawExifLen > 0) { /* malloc check ok, length is from python parsearg */ char* pp = malloc(rawExifLen); // Freed in JpegEncode, Case 5 - if (!pp) + if (!pp) { + if (extra) free(extra); return PyErr_NoMemory(); + } memcpy(pp, rawExif, rawExifLen); rawExif = pp; } else diff --git a/src/libImaging/Histo.c b/src/libImaging/Histo.c index 2b35873e8..b7c1a9834 100644 --- a/src/libImaging/Histo.c +++ b/src/libImaging/Histo.c @@ -82,8 +82,10 @@ ImagingGetHistogram(Imaging im, Imaging imMask, void* minmax) h->histogram[im->image8[y][x]]++; ImagingSectionLeave(&cookie); } else { /* yes, we need the braces. C isn't Python! */ - if (im->type != IMAGING_TYPE_UINT8) + if (im->type != IMAGING_TYPE_UINT8) { + ImagingHistogramDelete(h); return ImagingError_ModeError(); + } ImagingSectionEnter(&cookie); for (y = 0; y < im->ysize; y++) { UINT8* in = (UINT8*) im->image32[y]; @@ -122,8 +124,10 @@ ImagingGetHistogram(Imaging im, Imaging imMask, void* minmax) ImagingSectionLeave(&cookie); break; case IMAGING_TYPE_INT32: - if (!minmax) + if (!minmax) { + ImagingHistogramDelete(h); return ImagingError_ValueError("min/max not given"); + } if (!im->xsize || !im->ysize) break; imin = ((INT32*) minmax)[0]; @@ -143,8 +147,10 @@ ImagingGetHistogram(Imaging im, Imaging imMask, void* minmax) ImagingSectionLeave(&cookie); break; case IMAGING_TYPE_FLOAT32: - if (!minmax) + if (!minmax) { + ImagingHistogramDelete(h); return ImagingError_ValueError("min/max not given"); + } if (!im->xsize || !im->ysize) break; fmin = ((FLOAT32*) minmax)[0]; diff --git a/src/libImaging/Quant.c b/src/libImaging/Quant.c index df3138162..b94dc6e1d 100644 --- a/src/libImaging/Quant.c +++ b/src/libImaging/Quant.c @@ -568,6 +568,8 @@ split(BoxNode *node) left=malloc(sizeof(BoxNode)); right=malloc(sizeof(BoxNode)); if (!left||!right) { + free(left); + free(right); return 0; } for(i=0;i<3;i++) { diff --git a/src/libImaging/QuantOctree.c b/src/libImaging/QuantOctree.c index 6e4143d9e..6c0f605c9 100644 --- a/src/libImaging/QuantOctree.c +++ b/src/libImaging/QuantOctree.c @@ -481,6 +481,7 @@ error: free(qp); free_color_cube(lookupCube); free_color_cube(coarseLookupCube); + free(paletteBuckets); free(paletteBucketsCoarse); free(paletteBucketsFine); free_color_cube(coarseCube); diff --git a/src/libImaging/Resample.c b/src/libImaging/Resample.c index 6fae2081d..90e2aa1d0 100644 --- a/src/libImaging/Resample.c +++ b/src/libImaging/Resample.c @@ -621,6 +621,8 @@ ImagingResampleInner(Imaging imIn, int xsize, int ysize, if ( ! ksize_vert) { free(bounds_horiz); free(kk_horiz); + free(bounds_vert); + free(kk_vert); return NULL; } diff --git a/src/path.c b/src/path.c index d1c18c8ed..eb1e065f9 100644 --- a/src/path.c +++ b/src/path.c @@ -82,12 +82,16 @@ path_new(Py_ssize_t count, double* xy, int duplicate) xy = p; } - if (PyType_Ready(&PyPathType) < 0) + if (PyType_Ready(&PyPathType) < 0) { + free(xy); return NULL; + } path = PyObject_New(PyPathObject, &PyPathType); - if (path == NULL) + if (path == NULL) { + free(xy); return NULL; + } path->count = count; path->xy = xy; From ddb01ac67e7052ccdf5f66df6858ba38eff4cf8e Mon Sep 17 00:00:00 2001 From: Yann Diorcet Date: Thu, 4 Oct 2018 10:32:10 +0200 Subject: [PATCH 006/164] Add environment variables for settings component prefixes --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index a9e96b836..bceb9169f 100755 --- a/setup.py +++ b/setup.py @@ -245,6 +245,11 @@ class pil_build_ext(build_ext): IMAGEQUANT_ROOT="libimagequant" ).items(): root = globals()[root_name] + + if root is None and root_name in os.environ: + prefix = os.environ[root_name] + root = (os.path.join(prefix, 'lib'), os.path.join(prefix, 'include')) + if root is None and pkg_config: if isinstance(lib_name, tuple): for lib_name2 in lib_name: From 990fb03218440e9aa2f4dd18c4ea7806a96cc8ed Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 26 Aug 2018 13:52:02 +1000 Subject: [PATCH 007/164] Changed if to elif --- Tests/test_file_tiff_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_file_tiff_metadata.py b/Tests/test_file_tiff_metadata.py index 9c615354c..7407cc456 100644 --- a/Tests/test_file_tiff_metadata.py +++ b/Tests/test_file_tiff_metadata.py @@ -135,7 +135,7 @@ class TestFileTiffMetadata(PillowTestCase): for k, v in original.items(): if isinstance(v, IFDRational): original[k] = IFDRational(*_limit_rational(v, 2**31)) - if isinstance(v, tuple) and isinstance(v[0], IFDRational): + elif isinstance(v, tuple) and isinstance(v[0], IFDRational): original[k] = tuple(IFDRational(*_limit_rational(elt, 2**31)) for elt in v) From a3d45e9cefd05105a360588cd3087ccd5ae63b35 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 25 Oct 2018 19:45:13 +1100 Subject: [PATCH 008/164] Added custom int and float TIFF tags --- Tests/test_file_libtiff.py | 18 +++++++++++++++ src/PIL/TiffImagePlugin.py | 9 ++++---- src/PIL/TiffTags.py | 1 + src/encode.c | 46 ++++++++++++++++++++++++++----------- src/libImaging/TiffDecode.c | 13 +++++++++++ src/libImaging/TiffDecode.h | 1 + 6 files changed, 69 insertions(+), 19 deletions(-) diff --git a/Tests/test_file_libtiff.py b/Tests/test_file_libtiff.py index 58a3d38c5..19c952ae4 100644 --- a/Tests/test_file_libtiff.py +++ b/Tests/test_file_libtiff.py @@ -231,6 +231,24 @@ class TestFileLibTiff(LibTiffTestCase): TiffImagePlugin.WRITE_LIBTIFF = False + def test_custom_metadata(self): + custom = { + 37000: 4, + 37001: 4.2 + } + for libtiff in [False, True]: + TiffImagePlugin.WRITE_LIBTIFF = libtiff + + im = hopper() + + out = self.tempfile("temp.tif") + im.save(out, tiffinfo=custom) + TiffImagePlugin.WRITE_LIBTIFF = False + + reloaded = Image.open(out) + for tag, value in custom.items(): + self.assertEqual(reloaded.tag_v2[tag], value) + def test_int_dpi(self): # issue #1765 im = hopper('RGB') diff --git a/src/PIL/TiffImagePlugin.py b/src/PIL/TiffImagePlugin.py index 5059a1324..c2b09e294 100644 --- a/src/PIL/TiffImagePlugin.py +++ b/src/PIL/TiffImagePlugin.py @@ -1499,11 +1499,10 @@ def _save(im, fp, filename): getattr(im, 'tag_v2', {}).items(), legacy_ifd.items()): # Libtiff can only process certain core items without adding - # them to the custom dictionary. It will segfault if it attempts - # to add a custom tag without the dictionary entry - # - # UNDONE -- add code for the custom dictionary - if tag not in TiffTags.LIBTIFF_CORE: + # them to the custom dictionary. Support has only been been added + # for int and float values + if tag not in TiffTags.LIBTIFF_CORE and not \ + (isinstance(value, int) or isinstance(value, float)): continue if tag not in atts and tag not in blocklist: if isinstance(value, str if py3 else unicode): diff --git a/src/PIL/TiffTags.py b/src/PIL/TiffTags.py index c1e14af4a..d810f2f35 100644 --- a/src/PIL/TiffTags.py +++ b/src/PIL/TiffTags.py @@ -425,6 +425,7 @@ TYPES = {} # some of these are not in our TAGS_V2 dict and were included from tiff.h +# This list also exists in encode.c LIBTIFF_CORE = {255, 256, 257, 258, 259, 262, 263, 266, 274, 277, 278, 280, 281, 340, 341, 282, 283, 284, 286, 287, 296, 297, 321, 320, 338, 32995, 322, 323, 32998, diff --git a/src/encode.c b/src/encode.c index c60048c41..385d576dd 100644 --- a/src/encode.c +++ b/src/encode.c @@ -804,7 +804,13 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) PyObject *dir; PyObject *key, *value; Py_ssize_t pos = 0; - int status; + int key_int, status, is_core_tag, i; + // This list also exists in TiffTags.py + const int tags[32] = { + 256, 257, 258, 259, 262, 263, 266, 269, 274, 277, 278, 280, 281, 340, + 341, 282, 283, 284, 286, 287, 296, 297, 321, 338, 32995, 32998, 32996, + 339, 32997, 330, 531, 530 + }; Py_ssize_t d_size; PyObject *keys, *values; @@ -845,21 +851,33 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) for (pos = 0; pos < d_size; pos++) { key = PyList_GetItem(keys, pos); + key_int = (int)PyInt_AsLong(key); value = PyList_GetItem(values, pos); status = 0; - TRACE(("Attempting to set key: %d\n", (int)PyInt_AsLong(key))); + is_core_tag = 0; + for (i=0; i<32; i++) { + if (tags[i] == key_int) { + is_core_tag = 1; + break; + } + } + TRACE(("Attempting to set key: %d\n", key_int)); if (PyInt_Check(value)) { - TRACE(("Setting from Int: %d %ld \n", (int)PyInt_AsLong(key),PyInt_AsLong(value))); - status = ImagingLibTiffSetField(&encoder->state, - (ttag_t) PyInt_AsLong(key), - PyInt_AsLong(value)); + TRACE(("Setting from Int: %d %ld \n", key_int, PyInt_AsLong(value))); + if (is_core_tag || !ImagingLibTiffMergeFieldInfo(&encoder->state, TIFF_LONG, key_int)) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) PyInt_AsLong(key), + PyInt_AsLong(value)); + } } else if (PyFloat_Check(value)) { - TRACE(("Setting from Float: %d, %f \n", (int)PyInt_AsLong(key),PyFloat_AsDouble(value))); - status = ImagingLibTiffSetField(&encoder->state, - (ttag_t) PyInt_AsLong(key), - (float)PyFloat_AsDouble(value)); + TRACE(("Setting from Float: %d, %f \n", key_int, PyFloat_AsDouble(value))); + if (is_core_tag || !ImagingLibTiffMergeFieldInfo(&encoder->state, TIFF_DOUBLE, key_int)) { + status = ImagingLibTiffSetField(&encoder->state, + (ttag_t) PyInt_AsLong(key), + (double)PyFloat_AsDouble(value)); + } } else if (PyBytes_Check(value)) { - TRACE(("Setting from Bytes: %d, %s \n", (int)PyInt_AsLong(key),PyBytes_AsString(value))); + TRACE(("Setting from Bytes: %d, %s \n", key_int, PyBytes_AsString(value))); status = ImagingLibTiffSetField(&encoder->state, (ttag_t) PyInt_AsLong(key), PyBytes_AsString(value)); @@ -867,7 +885,7 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) Py_ssize_t len,i; float *floatav; int *intav; - TRACE(("Setting from Tuple: %d \n", (int)PyInt_AsLong(key))); + TRACE(("Setting from Tuple: %d \n", key_int)); len = PyTuple_Size(value); if (len) { if (PyInt_Check(PyTuple_GetItem(value,0))) { @@ -898,13 +916,13 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) } } else { TRACE(("Unhandled type in tuple for key %d : %s \n", - (int)PyInt_AsLong(key), + key_int, PyBytes_AsString(PyObject_Str(value)))); } } } else { TRACE(("Unhandled type for key %d : %s \n", - (int)PyInt_AsLong(key), + key_int, PyBytes_AsString(PyObject_Str(value)))); } if (!status) { diff --git a/src/libImaging/TiffDecode.c b/src/libImaging/TiffDecode.c index e77afdf61..bfab4b79e 100644 --- a/src/libImaging/TiffDecode.c +++ b/src/libImaging/TiffDecode.c @@ -402,6 +402,19 @@ int ImagingLibTiffEncodeInit(ImagingCodecState state, char *filename, int fp) { } +int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_type, int key){ + TIFFSTATE *clientstate = (TIFFSTATE *)state->context; + char field_name[10]; + uint32 n; + + const TIFFFieldInfo info[] = { + { key, 0, 1, field_type, FIELD_CUSTOM, 1, 0, field_name } + }; + n = sizeof(info) / sizeof(info[0]); + + return TIFFMergeFieldInfo(clientstate->tiff, info, n); +} + int ImagingLibTiffSetField(ImagingCodecState state, ttag_t tag, ...){ // after tif_dir.c->TIFFSetField. TIFFSTATE *clientstate = (TIFFSTATE *)state->context; diff --git a/src/libImaging/TiffDecode.h b/src/libImaging/TiffDecode.h index e14a09329..e29a6c88f 100644 --- a/src/libImaging/TiffDecode.h +++ b/src/libImaging/TiffDecode.h @@ -45,6 +45,7 @@ typedef struct { extern int ImagingLibTiffInit(ImagingCodecState state, int fp, int offset); extern int ImagingLibTiffEncodeInit(ImagingCodecState state, char *filename, int fp); +extern int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_type, int key); extern int ImagingLibTiffSetField(ImagingCodecState state, ttag_t tag, ...); From ddf8593e7bb4f5edc64c96ea730076920ea8cf33 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 25 Oct 2018 19:36:49 +1100 Subject: [PATCH 009/164] Do not write custom tags with libtiff < 4 --- Tests/test_file_libtiff.py | 11 ++++++++++- Tests/versions.py | 1 + src/PIL/TiffImagePlugin.py | 13 ++++++++++--- src/_imaging.c | 7 +++++++ src/libImaging/TiffDecode.c | 16 +++++++++++++++- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/Tests/test_file_libtiff.py b/Tests/test_file_libtiff.py index 19c952ae4..2ddb2263a 100644 --- a/Tests/test_file_libtiff.py +++ b/Tests/test_file_libtiff.py @@ -8,6 +8,7 @@ import io import logging import itertools import os +import distutils.version from PIL import Image, TiffImagePlugin, TiffTags @@ -236,7 +237,15 @@ class TestFileLibTiff(LibTiffTestCase): 37000: 4, 37001: 4.2 } - for libtiff in [False, True]: + + libtiff_version = TiffImagePlugin._libtiff_version() + + libtiffs = [False] + if distutils.version.StrictVersion(libtiff_version) >= \ + distutils.version.StrictVersion("4.0"): + libtiffs.append(True) + + for libtiff in libtiffs: TiffImagePlugin.WRITE_LIBTIFF = libtiff im = hopper() diff --git a/Tests/versions.py b/Tests/versions.py index abc1a3be9..835865b37 100644 --- a/Tests/versions.py +++ b/Tests/versions.py @@ -10,6 +10,7 @@ def version(module, version): version(Image, "jpeglib") version(Image, "zlib") +version(Image, "libtiff") try: from PIL import ImageFont diff --git a/src/PIL/TiffImagePlugin.py b/src/PIL/TiffImagePlugin.py index c2b09e294..2b4ddb038 100644 --- a/src/PIL/TiffImagePlugin.py +++ b/src/PIL/TiffImagePlugin.py @@ -54,6 +54,7 @@ import os import struct import sys import warnings +import distutils.version from .TiffTags import TYPES @@ -284,6 +285,10 @@ def _limit_rational(val, max_val): return n_d[::-1] if inv else n_d +def _libtiff_version(): + return Image.core.libtiff_version.split("\n")[0].split("Version ")[1] + + ## # Wrapper for TIFF IFDs. @@ -1501,9 +1506,11 @@ def _save(im, fp, filename): # Libtiff can only process certain core items without adding # them to the custom dictionary. Support has only been been added # for int and float values - if tag not in TiffTags.LIBTIFF_CORE and not \ - (isinstance(value, int) or isinstance(value, float)): - continue + if tag not in TiffTags.LIBTIFF_CORE: + if (distutils.version.StrictVersion(_libtiff_version()) < + distutils.version.StrictVersion("4.0")) \ + or not (isinstance(value, int) or isinstance(value, float)): + continue if tag not in atts and tag not in blocklist: if isinstance(value, str if py3 else unicode): atts[tag] = value.encode('ascii', 'replace') + b"\0" diff --git a/src/_imaging.c b/src/_imaging.c index 474444348..5b8ccb8ad 100644 --- a/src/_imaging.c +++ b/src/_imaging.c @@ -3849,6 +3849,13 @@ setup_module(PyObject* m) { } #endif +#ifdef HAVE_LIBTIFF + { + extern const char * ImagingTiffVersion(void); + PyDict_SetItemString(d, "libtiff_version", PyUnicode_FromString(ImagingTiffVersion())); + } +#endif + PyDict_SetItemString(d, "PILLOW_VERSION", PyUnicode_FromString(version)); return 0; diff --git a/src/libImaging/TiffDecode.c b/src/libImaging/TiffDecode.c index bfab4b79e..64ac86e6a 100644 --- a/src/libImaging/TiffDecode.c +++ b/src/libImaging/TiffDecode.c @@ -406,13 +406,20 @@ int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_typ TIFFSTATE *clientstate = (TIFFSTATE *)state->context; char field_name[10]; uint32 n; + int status = 0; const TIFFFieldInfo info[] = { { key, 0, 1, field_type, FIELD_CUSTOM, 1, 0, field_name } }; n = sizeof(info) / sizeof(info[0]); - return TIFFMergeFieldInfo(clientstate->tiff, info, n); + // Test for libtiff 4.0 or later, excluding libtiff 3.9.6 and 3.9.7 +#if TIFFLIB_VERSION >= 20111221 && TIFFLIB_VERSION != 20120218 && TIFFLIB_VERSION != 20120922 + status = TIFFMergeFieldInfo(clientstate->tiff, info, n); +#else + TIFFMergeFieldInfo(clientstate->tiff, info, n); +#endif + return status; } int ImagingLibTiffSetField(ImagingCodecState state, ttag_t tag, ...){ @@ -514,4 +521,11 @@ int ImagingLibTiffEncode(Imaging im, ImagingCodecState state, UINT8* buffer, int state->errcode = IMAGING_CODEC_END; return 0; } + +const char* +ImagingTiffVersion(void) +{ + return TIFFGetVersion(); +} + #endif From 207ed04d87cd29f0117566d1191bdcc6e5716acb Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 25 Oct 2018 20:17:20 +1100 Subject: [PATCH 010/164] Removed hardcoded array size Co-Authored-By: radarhere --- src/encode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/encode.c b/src/encode.c index 385d576dd..238eabd1e 100644 --- a/src/encode.c +++ b/src/encode.c @@ -855,7 +855,8 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) value = PyList_GetItem(values, pos); status = 0; is_core_tag = 0; - for (i=0; i<32; i++) { + int number_of_tags = sizeof(tags) / sizeof(int); + for (i=0; i Date: Thu, 25 Oct 2018 20:18:49 +1100 Subject: [PATCH 011/164] Changed array size to be arbitrary Co-Authored-By: radarhere --- src/encode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encode.c b/src/encode.c index 238eabd1e..2626e0eb8 100644 --- a/src/encode.c +++ b/src/encode.c @@ -806,7 +806,7 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) Py_ssize_t pos = 0; int key_int, status, is_core_tag, i; // This list also exists in TiffTags.py - const int tags[32] = { + const int tags[] = { 256, 257, 258, 259, 262, 263, 266, 269, 274, 277, 278, 280, 281, 340, 341, 282, 283, 284, 286, 287, 296, 297, 321, 338, 32995, 32998, 32996, 339, 32997, 330, 531, 530 From d8e66c1882a11a0f81c1f3e3308a64134779ab55 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 25 Oct 2018 21:00:38 +1100 Subject: [PATCH 012/164] Moved variable declaration and execution --- src/encode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/encode.c b/src/encode.c index 2626e0eb8..bce1c2988 100644 --- a/src/encode.c +++ b/src/encode.c @@ -804,7 +804,7 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) PyObject *dir; PyObject *key, *value; Py_ssize_t pos = 0; - int key_int, status, is_core_tag, i; + int key_int, status, is_core_tag, number_of_tags, i; // This list also exists in TiffTags.py const int tags[] = { 256, 257, 258, 259, 262, 263, 266, 269, 274, 277, 278, 280, 281, 340, @@ -849,13 +849,13 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) return NULL; } + number_of_tags = sizeof(tags) / sizeof(int); for (pos = 0; pos < d_size; pos++) { key = PyList_GetItem(keys, pos); key_int = (int)PyInt_AsLong(key); value = PyList_GetItem(values, pos); status = 0; is_core_tag = 0; - int number_of_tags = sizeof(tags) / sizeof(int); for (i=0; i Date: Sat, 3 Nov 2018 13:13:04 +1100 Subject: [PATCH 013/164] Added UnixViewer get_command --- Tests/test_imageshow.py | 10 ++++++++++ src/PIL/ImageShow.py | 9 +++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index da91e35c7..244c18d91 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -14,6 +14,9 @@ class TestImageShow(PillowTestCase): # Test registering a viewer that is not a class ImageShow.register("not a class") + # Restore original state + ImageShow._viewers.pop() + def test_show(self): class TestViewer: methodCalled = False @@ -28,6 +31,9 @@ class TestImageShow(PillowTestCase): self.assertTrue(ImageShow.show(im)) self.assertTrue(viewer.methodCalled) + # Restore original state + ImageShow._viewers.pop(0) + def test_viewer(self): viewer = ImageShow.Viewer() @@ -35,6 +41,10 @@ class TestImageShow(PillowTestCase): self.assertRaises(NotImplementedError, viewer.get_command, None) + def test_viewers(self): + for viewer in ImageShow._viewers: + viewer.get_command('test.jpg') + if __name__ == '__main__': unittest.main() diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index b50d61358..f1819cc5c 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -148,12 +148,9 @@ else: format = "PNG" options = {'compress_level': 1} - def show_file(self, file, **options): - command, executable = self.get_command_ex(file, **options) - command = "(%s %s; rm -f %s)&" % (command, quote(file), - quote(file)) - os.system(command) - return 1 + def get_command(self, file, **options): + command = self.get_command_ex(file, **options)[0] + return "(%s %s; rm -f %s)&" % (command, quote(file), quote(file)) # implementations From fea3dafd05c8ca44c30167c34ce4d82964421efb Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 3 Nov 2018 14:24:10 +1100 Subject: [PATCH 014/164] Supply filename through stdin instead of inline in Mac and Unix ImageShow viewers --- src/PIL/ImageShow.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index f1819cc5c..7d758c737 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -17,6 +17,8 @@ from __future__ import print_function from PIL import Image import os import sys +import subprocess +import tempfile if sys.version_info.major >= 3: from shlex import quote @@ -128,6 +130,21 @@ elif sys.platform == "darwin": quote(file)) return command + def show_file(self, file, **options): + """Display given file""" + f, path = tempfile.mkstemp() + f.write(file) + f.close() + with open(path, "r") as f: + subprocess.Popen([ + 'im=$(cat);' + 'open -a /Applications/Preview.app $im;' + 'sleep 20;' + 'rm -f $im' + ], shell=True, stdin=f) + os.remove(path) + return 1 + register(MacViewer) else: @@ -152,6 +169,21 @@ else: command = self.get_command_ex(file, **options)[0] return "(%s %s; rm -f %s)&" % (command, quote(file), quote(file)) + def show_file(self, file, **options): + """Display given file""" + f, path = tempfile.mkstemp() + f.write(file) + f.close() + with open(path, "r") as f: + command = self.get_command_ex(file, **options)[0] + subprocess.Popen([ + 'im=$(cat);' + + command+' $im;' + 'rm -f $im' + ], shell=True, stdin=f) + os.remove(path) + return 1 + # implementations class DisplayViewer(UnixViewer): From b637b5f7a2faefcc5db45262f409d143c62bc74f Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 13 Nov 2018 22:13:55 +1100 Subject: [PATCH 015/164] Close exclusive fp before discarding --- Tests/test_file_icns.py | 5 ++++- Tests/test_file_png.py | 4 +++- src/PIL/IcnsImagePlugin.py | 2 ++ src/PIL/PngImagePlugin.py | 2 ++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Tests/test_file_icns.py b/Tests/test_file_icns.py index b19cf2e4c..5e0f377c4 100644 --- a/Tests/test_file_icns.py +++ b/Tests/test_file_icns.py @@ -17,7 +17,10 @@ class TestFileIcns(PillowTestCase): # Loading this icon by default should result in the largest size # (512x512@2x) being loaded im = Image.open(TEST_FILE) - im.load() + + # Assert that there is no unclosed file warning + self.assert_warning(None, im.load) + self.assertEqual(im.mode, "RGBA") self.assertEqual(im.size, (1024, 1024)) self.assertEqual(im.format, "ICNS") diff --git a/Tests/test_file_png.py b/Tests/test_file_png.py index c958c0b39..ca7d3c039 100644 --- a/Tests/test_file_png.py +++ b/Tests/test_file_png.py @@ -327,7 +327,9 @@ class TestFilePng(PillowTestCase): # Check open/load/verify exception (@PIL150) im = Image.open(TEST_PNG_FILE) - im.verify() + + # Assert that there is no unclosed file warning + self.assert_warning(None, im.verify) im = Image.open(TEST_PNG_FILE) im.load() diff --git a/src/PIL/IcnsImagePlugin.py b/src/PIL/IcnsImagePlugin.py index d156b240f..2ea66675f 100644 --- a/src/PIL/IcnsImagePlugin.py +++ b/src/PIL/IcnsImagePlugin.py @@ -311,6 +311,8 @@ class IcnsImageFile(ImageFile.ImageFile): self.im = im.im self.mode = im.mode self.size = im.size + if self._exclusive_fp: + self.fp.close() self.fp = None self.icns = None self.tile = () diff --git a/src/PIL/PngImagePlugin.py b/src/PIL/PngImagePlugin.py index 15077fceb..b89bd0efc 100644 --- a/src/PIL/PngImagePlugin.py +++ b/src/PIL/PngImagePlugin.py @@ -600,6 +600,8 @@ class PngImageFile(ImageFile.ImageFile): self.png.verify() self.png.close() + if self._exclusive_fp: + self.fp.close() self.fp = None def load_prepare(self): From c0ee319af13a9ff9a93561e7c30bbdcfdef35a17 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 13 Nov 2018 22:24:59 +1100 Subject: [PATCH 016/164] Close fp before replacing --- Tests/test_file_cur.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/test_file_cur.py b/Tests/test_file_cur.py index 23055a0ad..1628007ee 100644 --- a/Tests/test_file_cur.py +++ b/Tests/test_file_cur.py @@ -26,6 +26,7 @@ class TestFileCur(PillowTestCase): no_cursors_file = "Tests/images/no_cursors.cur" cur = CurImagePlugin.CurImageFile(TEST_FILE) + cur.fp.close() with open(no_cursors_file, "rb") as cur.fp: self.assertRaises(TypeError, cur._open) From 47775f3d50b634d1a293370e55fd72fa62640ded Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 17 Nov 2018 21:56:06 +1100 Subject: [PATCH 017/164] Resolve __fp when closing and deleting --- Tests/test_file_dcx.py | 6 ++++++ Tests/test_file_fli.py | 6 ++++++ Tests/test_file_gif.py | 6 ++++++ Tests/test_file_im.py | 6 ++++++ Tests/test_file_mpo.py | 6 ++++++ Tests/test_file_spider.py | 6 ++++++ Tests/test_file_tiff.py | 6 ++++++ src/PIL/DcxImagePlugin.py | 8 ++++++++ src/PIL/FliImagePlugin.py | 8 ++++++++ src/PIL/GifImagePlugin.py | 8 ++++++++ src/PIL/ImImagePlugin.py | 8 ++++++++ src/PIL/Image.py | 4 ++++ src/PIL/MicImagePlugin.py | 9 ++++++++- src/PIL/MpoImagePlugin.py | 8 ++++++++ src/PIL/SpiderImagePlugin.py | 8 ++++++++ src/PIL/TiffImagePlugin.py | 8 ++++++++ 16 files changed, 110 insertions(+), 1 deletion(-) diff --git a/Tests/test_file_dcx.py b/Tests/test_file_dcx.py index 28ebb91dc..52c0e13f4 100644 --- a/Tests/test_file_dcx.py +++ b/Tests/test_file_dcx.py @@ -20,6 +20,12 @@ class TestFileDcx(PillowTestCase): orig = hopper() self.assert_image_equal(im, orig) + def test_unclosed_file(self): + def open(): + im = Image.open(TEST_FILE) + im.load() + self.assert_warning(None, open) + def test_invalid_file(self): with open("Tests/images/flower.jpg", "rb") as fp: self.assertRaises(SyntaxError, diff --git a/Tests/test_file_fli.py b/Tests/test_file_fli.py index 142af3cec..2b2f2da6d 100644 --- a/Tests/test_file_fli.py +++ b/Tests/test_file_fli.py @@ -27,6 +27,12 @@ class TestFileFli(PillowTestCase): self.assertEqual(im.info["duration"], 71) self.assertTrue(im.is_animated) + def test_unclosed_file(self): + def open(): + im = Image.open(static_test_file) + im.load() + self.assert_warning(None, open) + def test_tell(self): # Arrange im = Image.open(static_test_file) diff --git a/Tests/test_file_gif.py b/Tests/test_file_gif.py index cfa1bb1e7..16803dc11 100644 --- a/Tests/test_file_gif.py +++ b/Tests/test_file_gif.py @@ -33,6 +33,12 @@ class TestFileGif(PillowTestCase): self.assertEqual(im.format, "GIF") self.assertEqual(im.info["version"], b"GIF89a") + def test_unclosed_file(self): + def open(): + im = Image.open(TEST_GIF) + im.load() + self.assert_warning(None, open) + def test_invalid_file(self): invalid_file = "Tests/images/flower.jpg" diff --git a/Tests/test_file_im.py b/Tests/test_file_im.py index c99924767..f3349b736 100644 --- a/Tests/test_file_im.py +++ b/Tests/test_file_im.py @@ -15,6 +15,12 @@ class TestFileIm(PillowTestCase): self.assertEqual(im.size, (128, 128)) self.assertEqual(im.format, "IM") + def test_unclosed_file(self): + def open(): + im = Image.open(TEST_IM) + im.load() + self.assert_warning(None, open) + def test_tell(self): # Arrange im = Image.open(TEST_IM) diff --git a/Tests/test_file_mpo.py b/Tests/test_file_mpo.py index 70bb9b105..9f79d8cfa 100644 --- a/Tests/test_file_mpo.py +++ b/Tests/test_file_mpo.py @@ -31,6 +31,12 @@ class TestFileMpo(PillowTestCase): self.assertEqual(im.size, (640, 480)) self.assertEqual(im.format, "MPO") + def test_unclosed_file(self): + def open(): + im = Image.open(test_files[0]) + im.load() + self.assert_warning(None, open) + def test_app(self): for test_file in test_files: # Test APP/COM reader (@PIL135) diff --git a/Tests/test_file_spider.py b/Tests/test_file_spider.py index b54b92e04..4612b36a1 100644 --- a/Tests/test_file_spider.py +++ b/Tests/test_file_spider.py @@ -18,6 +18,12 @@ class TestImageSpider(PillowTestCase): self.assertEqual(im.size, (128, 128)) self.assertEqual(im.format, "SPIDER") + def test_unclosed_file(self): + def open(): + im = Image.open(TEST_FILE) + im.load() + self.assert_warning(None, open) + def test_save(self): # Arrange temp = self.tempfile('temp.spider') diff --git a/Tests/test_file_tiff.py b/Tests/test_file_tiff.py index 56ad69473..95a181363 100644 --- a/Tests/test_file_tiff.py +++ b/Tests/test_file_tiff.py @@ -40,6 +40,12 @@ class TestFileTiff(PillowTestCase): hopper("I").save(filename) Image.open(filename) + def test_unclosed_file(self): + def open(): + im = Image.open("Tests/images/multipage.tiff") + im.load() + self.assert_warning(None, open) + def test_mac_tiff(self): # Read RGBa images from macOS [@PIL136] diff --git a/src/PIL/DcxImagePlugin.py b/src/PIL/DcxImagePlugin.py index 204592759..1b72cbc99 100644 --- a/src/PIL/DcxImagePlugin.py +++ b/src/PIL/DcxImagePlugin.py @@ -81,6 +81,14 @@ class DcxImageFile(PcxImageFile): def tell(self): return self.frame + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + Image.register_open(DcxImageFile.format, DcxImageFile, _accept) diff --git a/src/PIL/FliImagePlugin.py b/src/PIL/FliImagePlugin.py index c78c8c622..d093ed705 100644 --- a/src/PIL/FliImagePlugin.py +++ b/src/PIL/FliImagePlugin.py @@ -154,6 +154,14 @@ class FliImageFile(ImageFile.ImageFile): def tell(self): return self.__frame + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # # registry diff --git a/src/PIL/GifImagePlugin.py b/src/PIL/GifImagePlugin.py index ecd993cd5..7a68064fd 100644 --- a/src/PIL/GifImagePlugin.py +++ b/src/PIL/GifImagePlugin.py @@ -296,6 +296,14 @@ class GifImageFile(ImageFile.ImageFile): self.im = self._prev_im self._prev_im = self.im.copy() + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # -------------------------------------------------------------------- # Write GIF files diff --git a/src/PIL/ImImagePlugin.py b/src/PIL/ImImagePlugin.py index 38b3f46f4..1a4fdb65a 100644 --- a/src/PIL/ImImagePlugin.py +++ b/src/PIL/ImImagePlugin.py @@ -290,6 +290,14 @@ class ImImageFile(ImageFile.ImageFile): def tell(self): return self.frame + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # # -------------------------------------------------------------------- # Save IM files diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 541c62c49..ad30de478 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -596,6 +596,8 @@ class Image(object): try: self.fp.close() self.fp = None + if hasattr(self, "_close__fp"): + self._close__fp() except Exception as msg: logger.debug("Error closing: %s", msg) @@ -613,6 +615,8 @@ class Image(object): and self.fp and self._exclusive_fp): self.fp.close() self.fp = None + if hasattr(self, "_close__fp"): + self._close__fp() def _copy(self): self.load() diff --git a/src/PIL/MicImagePlugin.py b/src/PIL/MicImagePlugin.py index 1dbb6a588..6e29e456d 100644 --- a/src/PIL/MicImagePlugin.py +++ b/src/PIL/MicImagePlugin.py @@ -95,9 +95,16 @@ class MicImageFile(TiffImagePlugin.TiffImageFile): self.frame = frame def tell(self): - return self.frame + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # # -------------------------------------------------------------------- diff --git a/src/PIL/MpoImagePlugin.py b/src/PIL/MpoImagePlugin.py index a1a8d655a..06b93d921 100644 --- a/src/PIL/MpoImagePlugin.py +++ b/src/PIL/MpoImagePlugin.py @@ -84,6 +84,14 @@ class MpoImageFile(JpegImagePlugin.JpegImageFile): def tell(self): return self.__frame + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # --------------------------------------------------------------------- # Registry stuff diff --git a/src/PIL/SpiderImagePlugin.py b/src/PIL/SpiderImagePlugin.py index 3f5795210..9dd7a7c03 100644 --- a/src/PIL/SpiderImagePlugin.py +++ b/src/PIL/SpiderImagePlugin.py @@ -193,6 +193,14 @@ class SpiderImageFile(ImageFile.ImageFile): from PIL import ImageTk return ImageTk.PhotoImage(self.convert2byte(), palette=256) + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # -------------------------------------------------------------------- # Image series diff --git a/src/PIL/TiffImagePlugin.py b/src/PIL/TiffImagePlugin.py index 5059a1324..5ae8211a4 100644 --- a/src/PIL/TiffImagePlugin.py +++ b/src/PIL/TiffImagePlugin.py @@ -1347,6 +1347,14 @@ class TiffImageFile(ImageFile.ImageFile): palette = [o8(b // 256) for b in self.tag_v2[COLORMAP]] self.palette = ImagePalette.raw("RGB;L", b"".join(palette)) + def _close__fp(self): + try: + self.__fp.close() + except AttributeError: + pass + finally: + self.__fp = None + # # -------------------------------------------------------------------- From 84df069169b4ad429f34a6b1bc318ba9aae73129 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 20 Nov 2018 18:50:00 +1100 Subject: [PATCH 018/164] Use im.info background when saving --- src/PIL/WebPImagePlugin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/PIL/WebPImagePlugin.py b/src/PIL/WebPImagePlugin.py index 0e9de6e48..084eedf53 100644 --- a/src/PIL/WebPImagePlugin.py +++ b/src/PIL/WebPImagePlugin.py @@ -191,7 +191,11 @@ def _save_all(im, fp, filename): _save(im, fp, filename) return - background = encoderinfo.get("background", (0, 0, 0, 0)) + background = (0, 0, 0, 0) + if "background" in encoderinfo: + background = encoderinfo["background"] + elif "background" in im.info: + background = im.info["background"] duration = im.encoderinfo.get("duration", 0) loop = im.encoderinfo.get("loop", 0) minimize_size = im.encoderinfo.get("minimize_size", False) From 6c126ca2821d0fba00fffc290aa3226692e15ff8 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 20 Nov 2018 19:50:14 +1100 Subject: [PATCH 019/164] Change color table index background to tuple when saving --- Tests/test_file_webp.py | 20 ++++++++++++++++++++ src/PIL/WebPImagePlugin.py | 8 ++++++++ 2 files changed, 28 insertions(+) diff --git a/Tests/test_file_webp.py b/Tests/test_file_webp.py index bdec8273d..a79ae8d9f 100644 --- a/Tests/test_file_webp.py +++ b/Tests/test_file_webp.py @@ -153,6 +153,26 @@ class TestFileWebp(PillowTestCase): Image.open(blob).load() Image.open(blob).load() + @unittest.skipUnless(HAVE_WEBP and _webp.HAVE_WEBPANIM, + "WebP save all not available") + def test_background_from_gif(self): + im = Image.open("Tests/images/chi.gif") + original_value = im.convert("RGB").getpixel((1, 1)) + + # Save as WEBP + out_webp = self.tempfile("temp.webp") + im.save(out_webp, save_all=True) + + # Save as GIF + out_gif = self.tempfile("temp.gif") + Image.open(out_webp).save(out_gif) + + reread = Image.open(out_gif) + reread_value = reread.convert("RGB").getpixel((1, 1)) + difference = sum([abs(original_value[i] - reread_value[i]) + for i in range(0, 3)]) + self.assertLess(difference, 5) + if __name__ == '__main__': unittest.main() diff --git a/src/PIL/WebPImagePlugin.py b/src/PIL/WebPImagePlugin.py index 084eedf53..212e6b4c7 100644 --- a/src/PIL/WebPImagePlugin.py +++ b/src/PIL/WebPImagePlugin.py @@ -196,6 +196,14 @@ def _save_all(im, fp, filename): background = encoderinfo["background"] elif "background" in im.info: background = im.info["background"] + if isinstance(background, int): + # GifImagePlugin stores a global color table index in + # info["background"]. So it must be converted to an RGBA value + palette = im.getpalette() + if palette: + r, g, b = palette[background*3:(background+1)*3] + background = (r, g, b, 0) + duration = im.encoderinfo.get("duration", 0) loop = im.encoderinfo.get("loop", 0) minimize_size = im.encoderinfo.get("minimize_size", False) From 0f42cdd63936fcc4c805c55e3b9c666d2bd64439 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Fri, 23 Nov 2018 18:57:17 +1100 Subject: [PATCH 020/164] Added functions to Read The Docs sidebar --- docs/_static/js/script.js | 60 +++++++++++++++++++++++++++++++++++++++ docs/conf.py | 3 ++ 2 files changed, 63 insertions(+) create mode 100644 docs/_static/js/script.js diff --git a/docs/_static/js/script.js b/docs/_static/js/script.js new file mode 100644 index 000000000..2cd7335ec --- /dev/null +++ b/docs/_static/js/script.js @@ -0,0 +1,60 @@ +jQuery(document).ready(function ($) { + setTimeout(function () { + var sectionID = 'base'; + var search = function ($section, $sidebarItem) { + $section.children('.section, .function, .method').each(function () { + if ($(this).hasClass('section')) { + sectionID = $(this).attr('id'); + search($(this), $sidebarItem.parent().find('[href=#'+sectionID+']')); + } else { + var $dt = $(this).children('dt'); + var id = $dt.attr('id'); + if (id === undefined) { + return; + } + + var $functionsUL = $sidebarItem.siblings('[data-sectionID='+sectionID+']'); + if (!$functionsUL.length) { + $functionsUL = $('