Merge remote-tracking branch 'upstream/master' into ResourceWarning

CHANGES.rst

@@ -1,9 +1,15 @@
 Changelog (Pillow)
 ==================
 
-2.7.0 (unreleased)
+2.7.0 (2015-01-01)
 ------------------
 
+- Look for OSX and Linux fonts in common places. #1054
+  [charleslaw]
+
+- Fix potential PNG decompression DOS #1060
+  [wiredfool]
+
 - Use underscores, not spaces, in TIFF tag kwargs. #1044, #1058
   [anntzer, hugovk]
   
@@ -73,6 +79,15 @@ Changelog (Pillow)
 - Fixes for things rpmlint complains about #942
   [manisandro]
 
+2.6.2 (2015-01-01)
+------------------
+
+- Fix potential PNG decompression DOS #1060 
+  [wiredfool]
+
+- Fix Regression in PyPy 2.4 in streamio  #958
+  [wiredfool]
+
 2.6.1 (2014-10-11)
 ------------------
 

PIL/ImageFilter.py

@@ -162,8 +162,13 @@ class UnsharpMask(Filter):
     See Wikipedia's entry on `digital unsharp masking`_ for an explanation of
     the parameters.
 
-    .. _digital unsharp masking:
-    https://en.wikipedia.org/wiki/Unsharp_masking#Digital_unsharp_masking
+    :param radius: Blur Radius
+    :param percent: Unsharp strength, in percent
+    :param threshold: Threshold controls the minimum brightness change that
+      will be sharpened
+
+    .. _digital unsharp masking: https://en.wikipedia.org/wiki/Unsharp_masking#Digital_unsharp_masking
+
     """
     name = "UnsharpMask"
 

PIL/ImageFont.py

@@ -239,6 +239,10 @@ def truetype(font=None, size=10, index=0, encoding="", filename=None):
     try:
         return FreeTypeFont(font, size, index, encoding)
     except IOError:
+        if font.endswith(".ttf"):
+            ttf_filename = font
+        else:
+            ttf_filename = "%s.ttf" % font
         if sys.platform == "win32":
             # check the windows font repository
             # NOTE: must use uppercase WINDIR, to work around bugs in
@@ -247,6 +251,25 @@ def truetype(font=None, size=10, index=0, encoding="", filename=None):
             if windir:
                 filename = os.path.join(windir, "fonts", font)
                 return FreeTypeFont(filename, size, index, encoding)
+        elif sys.platform in ('linux', 'linux2'):
+            lindirs = os.environ.get("XDG_DATA_DIRS", "")
+            if not lindirs:
+                #According to the freedesktop spec, XDG_DATA_DIRS should
+                #default to /usr/share
+                lindirs = '/usr/share'
+            lindirs = lindirs.split(":")
+            for lindir in lindirs:
+                parentpath = os.path.join(lindir, "fonts")
+                for walkroot, walkdir, walkfilenames in os.walk(parentpath):
+                    if ttf_filename in walkfilenames:
+                        filepath = os.path.join(walkroot, ttf_filename)
+                        return FreeTypeFont(filepath, size, index, encoding)
+        elif sys.platform == 'darwin':
+            macdirs = ['/Library/Fonts/', '/System/Library/Fonts/', os.path.expanduser('~/Library/Fonts/')]
+            for macdir in macdirs:
+                filepath = os.path.join(macdir, ttf_filename)
+                if os.path.exists(filepath):
+                    return FreeTypeFont(filepath, size, index, encoding)
         raise
 
 

PIL/PngImagePlugin.py

@@ -72,6 +72,19 @@ _MODES = {
 
 _simple_palette = re.compile(b'^\xff+\x00\xff*$')
 
+# Maximum decompressed size for a iTXt or zTXt chunk.
+# Eliminates decompression bombs where compressed chunks can expand 1000x
+MAX_TEXT_CHUNK = ImageFile.SAFEBLOCK
+# Set the maximum total text chunk size.
+MAX_TEXT_MEMORY = 64 * MAX_TEXT_CHUNK
+
+def _safe_zlib_decompress(s):
+    dobj = zlib.decompressobj()
+    plaintext = dobj.decompress(s, MAX_TEXT_CHUNK)
+    if dobj.unconsumed_tail:
+        raise ValueError("Decompressed Data Too Large")
+    return plaintext
+
 
 # --------------------------------------------------------------------
 # Support classes.  Suitable for PNG and related formats like MNG etc.
@@ -260,6 +273,14 @@ class PngStream(ChunkStream):
         self.im_tile = None
         self.im_palette = None
 
+        self.text_memory = 0
+
+    def check_text_memory(self, chunklen):
+        self.text_memory += chunklen
+        if self.text_memory > MAX_TEXT_MEMORY:
+            raise ValueError("Too much memory used in text chunks: %s>MAX_TEXT_MEMORY" %
+                             self.text_memory)
+
     def chunk_iCCP(self, pos, length):
 
         # ICC profile
@@ -278,7 +299,7 @@ class PngStream(ChunkStream):
             raise SyntaxError("Unknown compression method %s in iCCP chunk" %
                               comp_method)
         try:
-            icc_profile = zlib.decompress(s[i+2:])
+            icc_profile = _safe_zlib_decompress(s[i+2:])
         except zlib.error:
             icc_profile = None  # FIXME
         self.im_info["icc_profile"] = icc_profile
@@ -372,6 +393,8 @@ class PngStream(ChunkStream):
                 v = v.decode('latin-1', 'replace')
 
             self.im_info[k] = self.im_text[k] = v
+            self.check_text_memory(len(v))
+
         return s
 
     def chunk_zTXt(self, pos, length):
@@ -391,7 +414,7 @@ class PngStream(ChunkStream):
             raise SyntaxError("Unknown compression method %s in zTXt chunk" %
                               comp_method)
         try:
-            v = zlib.decompress(v[1:])
+            v = _safe_zlib_decompress(v[1:])
         except zlib.error:
             v = b""
 
@@ -401,6 +424,8 @@ class PngStream(ChunkStream):
                 v = v.decode('latin-1', 'replace')
 
             self.im_info[k] = self.im_text[k] = v
+            self.check_text_memory(len(v))
+
         return s
 
     def chunk_iTXt(self, pos, length):
@@ -421,7 +446,7 @@ class PngStream(ChunkStream):
         if cf != 0:
             if cm == 0:
                 try:
-                    v = zlib.decompress(v)
+                    v = _safe_zlib_decompress(v)
                 except zlib.error:
                     return s
             else:
@@ -436,7 +461,8 @@ class PngStream(ChunkStream):
                 return s
 
         self.im_info[k] = self.im_text[k] = iTXt(v, lang, tk)
-
+        self.check_text_memory(len(v))
+            
         return s
 
 

PIL/__init__.py

@@ -12,7 +12,7 @@
 # ;-)
 
 VERSION = '1.1.7'  # PIL version
-PILLOW_VERSION = '2.6.0'  # Pillow
+PILLOW_VERSION = '2.7.0'  # Pillow
 
 _plugins = ['BmpImagePlugin',
             'BufrStubImagePlugin',

Tests/check_png_dos.py

@@ -0,0 +1,47 @@
+from helper import unittest, PillowTestCase
+from PIL import Image, PngImagePlugin
+from io import BytesIO
+import zlib
+
+TEST_FILE = "Tests/images/png_decompression_dos.png"
+
+class TestPngDos(PillowTestCase):
+    def test_dos_text(self):
+
+        try:
+            im = Image.open(TEST_FILE)
+            im.load()
+        except ValueError as msg:
+            self.assertTrue(msg, "Decompressed Data Too Large")
+            return
+
+        for s in im.text.values():
+            self.assertLess(len(s), 1024*1024, "Text chunk larger than 1M")
+
+    def test_dos_total_memory(self):
+        im = Image.new('L',(1,1))
+        compressed_data = zlib.compress('a'*1024*1023)
+
+        info = PngImagePlugin.PngInfo()
+
+        for x in range(64):
+            info.add_text('t%s'%x, compressed_data, 1)
+            info.add_itxt('i%s'%x, compressed_data, zip=True)
+
+        b = BytesIO()
+        im.save(b, 'PNG', pnginfo=info)
+        b.seek(0)
+        
+        try:
+            im2 = Image.open(b)
+        except ValueError as msg:
+            self.assertIn("Too much memory", msg)
+            return
+
+        total_len = 0
+        for txt in im2.text.values():
+            total_len += len(txt)
+        self.assertLess(total_len, 64*1024*1024, "Total text chunks greater than 64M")
+        
+if __name__ == '__main__':
+    unittest.main()

Tests/test_file_png.py

@@ -153,7 +153,7 @@ class TestFilePng(PillowTestCase):
 
         im = load(HEAD + chunk(b'iTXt', b'spam\0\1\0en\0Spam\0' +
                                zlib.compress(b"egg")[:1]) + TAIL)
-        self.assertEqual(im.info, {})
+        self.assertEqual(im.info, {'spam':''})
 
         im = load(HEAD + chunk(b'iTXt', b'spam\0\1\1en\0Spam\0' +
                                zlib.compress(b"egg")) + TAIL)

Tests/test_imagefont.py

@@ -4,6 +4,8 @@ from PIL import Image
 from PIL import ImageDraw
 from io import BytesIO
 import os
+import sys
+import copy
 
 FONT_PATH = "Tests/fonts/FreeMono.ttf"
 FONT_SIZE = 20
@@ -13,6 +15,29 @@ try:
     from PIL import ImageFont
     ImageFont.core.getfont  # check if freetype is available
 
+    class SimplePatcher():
+        def __init__(self, parent_obj, attr_name, value):
+            self._parent_obj = parent_obj
+            self._attr_name = attr_name
+            self._saved = None
+            self._is_saved = False
+            self._value = value
+        def __enter__(self):
+            # Patch the attr on the object
+            if hasattr(self._parent_obj, self._attr_name):
+                self._saved = getattr(self._parent_obj, self._attr_name)
+                setattr(self._parent_obj, self._attr_name, self._value)
+                self._is_saved = True
+            else:
+                setattr(self._parent_obj, self._attr_name, self._value)
+                self._is_saved = False
+        def __exit__(self, type, value, traceback):
+            # Restore the original value
+            if self._is_saved:
+                setattr(self._parent_obj, self._attr_name, self._saved)
+            else:
+                delattr(self._parent_obj, self._attr_name)
+
     class TestImageFont(PillowTestCase):
 
         def test_sanity(self):
@@ -192,6 +217,45 @@ try:
             # Assert
             self.assert_image_equal(im, target_img)
 
+        def _test_fake_loading_font(self, path_to_fake):
+            #Make a copy of FreeTypeFont so we can patch the original
+            free_type_font = copy.deepcopy(ImageFont.FreeTypeFont)
+            with SimplePatcher(ImageFont, '_FreeTypeFont', free_type_font):
+                def loadable_font(filepath, size, index, encoding):
+                    if filepath == path_to_fake:
+                        return ImageFont._FreeTypeFont(FONT_PATH, size, index, encoding)
+                    return ImageFont._FreeTypeFont(filepath, size, index, encoding)
+                with SimplePatcher(ImageFont, 'FreeTypeFont', loadable_font):
+                    font = ImageFont.truetype('Arial')
+                    #Make sure it's loaded
+                    name = font.getname()
+                    self.assertEqual(('FreeMono', 'Regular'), name)
+
+
+        @unittest.skipIf(sys.platform.startswith('win32'), "requires Unix or MacOS")
+        def test_find_linux_font(self):
+            #A lot of mocking here - this is more for hitting code and catching
+            #syntax like errors
+            with SimplePatcher(sys, 'platform', 'linux'):
+                patched_env = copy.deepcopy(os.environ)
+                patched_env['XDG_DATA_DIRS'] = '/usr/share/:/usr/local/share/'
+                with SimplePatcher(os, 'environ', patched_env):
+                    def fake_walker(path):
+                        if path == '/usr/local/share/fonts':
+                            return [(path, [], ['Arial.ttf'], )]
+                        return [(path, [], ['some_random_font.ttf'], )]
+                    with SimplePatcher(os, 'walk', fake_walker):
+                        self._test_fake_loading_font('/usr/local/share/fonts/Arial.ttf')
+
+        @unittest.skipIf(sys.platform.startswith('win32'), "requires Unix or MacOS")
+        def test_find_osx_font(self):
+            #Like the linux test, more cover hitting code rather than testing
+            #correctness.
+            with SimplePatcher(sys, 'platform', 'darwin'):
+                fake_font_path = '/System/Library/Fonts/Arial.ttf'
+                with SimplePatcher(os.path, 'exists', lambda x: x == fake_font_path):
+                    self._test_fake_loading_font(fake_font_path)
+
 
 except ImportError:
     class TestImageFont(PillowTestCase):

Tests/test_scipy.py

@@ -11,7 +11,10 @@ except:
     
 
 class Test_scipy_resize(PillowTestCase):
-    """ Tests for scipy regression in 2.6.0 """
+    """ Tests for scipy regression in 2.6.0
+
+    Tests from https://github.com/scipy/scipy/blob/master/scipy/misc/pilutil.py 
+    """
 
     def setUp(self):
         if not HAS_SCIPY:
@@ -27,10 +30,10 @@ class Test_scipy_resize(PillowTestCase):
     def test_imresize4(self):
         im = np.array([[1,2],
                        [3,4]])
-        res = np.array([[ 1. ,  1. ,  1.5,  2. ],
-                        [ 1. ,  1. ,  1.5,  2. ],
-                        [ 2. ,  2. ,  2.5,  3. ],
-                        [ 3. ,  3. ,  3.5,  4. ]], dtype=np.float32)
+        res = np.array([[ 1.  ,  1.25,  1.75,  2.  ],
+                        [ 1.5 ,  1.75,  2.25,  2.5 ],
+                        [ 2.5 ,  2.75,  3.25,  3.5 ],
+                        [ 3.  ,  3.25,  3.75,  4.  ]], dtype=np.float32)
         # Check that resizing by target size, float and int are the same
         im2 = misc.imresize(im, (4,4), mode='F')  # output size
         im3 = misc.imresize(im, 2., mode='F')  # fraction

_imaging.c

@@ -71,7 +71,7 @@
  * See the README file for information on usage and redistribution.
  */
 
-#define PILLOW_VERSION "2.6.0"
+#define PILLOW_VERSION "2.7.0"
 
 #include "Python.h"
 

docs/handbook/image-file-formats.rst

@@ -333,7 +333,12 @@ The :py:meth:`~PIL.Image.Image.open` method sets the following
     transparent palette image.
 
 ``Open`` also sets ``Image.text`` to a list of the values of the
-``tEXt``, ``zTXt``, and ``iTXt`` chunks of the PNG image.
+``tEXt``, ``zTXt``, and ``iTXt`` chunks of the PNG image. Individual
+compressed chunks are limited to a decompressed size of
+``PngImagePlugin.MAX_TEXT_CHUNK``, by default 1MB, to prevent
+decompression bombs. Additionally, the total size of all of the text
+chunks is limited to ``PngImagePlugin.MAX_TEXT_MEMORY``, defaulting to
+64MB.
 
 The :py:meth:`~PIL.Image.Image.save` method supports the following options:
 

docs/releasenotes/2.7.0.rst

@@ -1,6 +1,21 @@
 Pillow 2.7.0
 ============
 
+Png text chunk size limits
+--------------------------
+
+To prevent potential denial of service attacks using compressed text
+chunks, there are now limits to the decompressed size of text chunks
+decoded from PNG images. If the limits are exceeded when opening a PNG
+image a ``ValueError`` will be raised. 
+
+Individual text chunks are limited to
+:py:attr:`PIL.PngImagePlugin.MAX_TEXT_CHUNK`, set to 1MB by
+default. The total decompressed size of all text chunks is limited to
+:py:attr:`PIL.PngImagePlugin.MAX_TEXT_MEMORY`, which defaults to
+64MB. These values can be changed prior to opening PNG images if you
+know that there are large text blocks that are desired.
+
 Image resizing filters
 ----------------------
 
@@ -141,3 +156,13 @@ The previous implementation takes into account only source pixels within
 so the quality was worse compared to other Gaussian blur software.
 
 The new implementation does not have this drawback.
+
+TFF Parameter Changes
+----------------------
+
+Several kwarg parameters for saving TIFF images were previously
+specified as strings with included spaces (e.g. 'x resolution'). This
+was difficult to use as kwargs without constructing and passing a
+dictionary. These parameters now use the underscore character instead
+of space. (e.g. 'x_resolution')
+

setup.py

@@ -90,7 +90,7 @@ except (ImportError, OSError):
 
 
 NAME = 'Pillow'
-PILLOW_VERSION = '2.6.0'
+PILLOW_VERSION = '2.7.0'
 TCL_ROOT = None
 JPEG_ROOT = None
 JPEG2K_ROOT = None