From 8193566ff1a2eba3a2f0f4cddf17833346c18b79 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Mon, 4 Aug 2014 14:48:42 -0700 Subject: [PATCH 01/59] Try to open the file in universal line ending mode, fallback to prev behavior --- PIL/EpsImagePlugin.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/PIL/EpsImagePlugin.py b/PIL/EpsImagePlugin.py index 9f963f7e6..197fa8753 100644 --- a/PIL/EpsImagePlugin.py +++ b/PIL/EpsImagePlugin.py @@ -179,6 +179,20 @@ class PSFile: self.char = None return s.decode('latin-1') + "\n" +class PSFpWrapper: + """ Wrapper for a filepointer that has been opened in universal mode """ + def __init__(self,fp): + self.fp = fp + + def __getattr__(self, attr): + """ Delegate everything that we're not wrapping """ + return getattr(self.fp, attr) + + def read(self, count): + return self.fp.read(count) + + readbinary = read + def _accept(prefix): return prefix[:4] == b"%!PS" or i32(prefix) == 0xC6D3D0C5 @@ -195,7 +209,11 @@ class EpsImageFile(ImageFile.ImageFile): def _open(self): - fp = PSFile(self.fp) + try: + fp = PSFpWrapper(open(self.fp.name, "Ur", 'latin-1')) + except: + print ("fallback to psfile") + fp = PSFile(self.fp) # FIX for: Some EPS file not handled correctly / issue #302 # EPS can contain binary data From e52152baaddf6bb6a74bb2ed1f24bc1545c59d75 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Mon, 4 Aug 2014 15:27:08 -0700 Subject: [PATCH 02/59] Test bytesio object --- Tests/test_file_eps.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 0ca4249a3..5b369552a 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -63,6 +63,17 @@ class TestFileEps(PillowTestCase): with io.open(self.tempfile('temp_iobase.eps'), 'wb') as fh: image1.save(fh, 'EPS') + def test_bytesio_object(self): + with open(file1, 'rb') as f: + img_bytes = io.BytesIO(f.read()) + + img = Image.open(img_bytes) + img.load() + + image1_scale1_compare = Image.open(file1_compare).convert("RGB") + image1_scale1_compare.load() + self.assert_image_similar(img, image1_scale1_compare, 5) + def test_render_scale1(self): # We need png support for these render test codecs = dir(Image.core) From 9b35a4538d4cb121806f41c4338ac50a83107e5c Mon Sep 17 00:00:00 2001 From: wiredfool Date: Mon, 4 Aug 2014 15:28:57 -0700 Subject: [PATCH 03/59] Read the offset using original fp Read the rest of the text data using one of the line ending wrappers. --- PIL/EpsImagePlugin.py | 66 ++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/PIL/EpsImagePlugin.py b/PIL/EpsImagePlugin.py index 197fa8753..88ce23b22 100644 --- a/PIL/EpsImagePlugin.py +++ b/PIL/EpsImagePlugin.py @@ -189,9 +189,11 @@ class PSFpWrapper: return getattr(self.fp, attr) def read(self, count): + return self.fp.read(count).decode('latin-1') + + def readbinary(self, count): return self.fp.read(count) - readbinary = read def _accept(prefix): @@ -208,39 +210,24 @@ class EpsImageFile(ImageFile.ImageFile): format_description = "Encapsulated Postscript" def _open(self): + (length, offset) = self._find_offset(self.fp) + # Rewrap the open file pointer in something that will + # convert line endings and decode to latin-1. try: - fp = PSFpWrapper(open(self.fp.name, "Ur", 'latin-1')) - except: - print ("fallback to psfile") + if bytes is str: + # Python2, need the decode to latin-1 on read. + fp = PSFpWrapper(open(self.fp.name, "Ur")) + else: + # Python3, can use bare open command. + fp = open(self.fp.name, "Ur", encoding='latin-1') + except Exception as msg: + # Expect this for bytesio/stringio fp = PSFile(self.fp) - # FIX for: Some EPS file not handled correctly / issue #302 - # EPS can contain binary data - # or start directly with latin coding - # read header in both ways to handle both - # file types - # more info see http://partners.adobe.com/public/developer/en/ps/5002.EPSF_Spec.pdf - - # for HEAD without binary preview - s = fp.read(4) - # for HEAD with binary preview - fp.seek(0) - sb = fp.readbinary(160) - - if s[:4] == "%!PS": - fp.seek(0, 2) - length = fp.tell() - offset = 0 - elif i32(sb[0:4]) == 0xC6D3D0C5: - offset = i32(sb[4:8]) - length = i32(sb[8:12]) - else: - raise SyntaxError("not an EPS file") - - # go to offset - start of "%!PS" + # go to offset - start of "%!PS" fp.seek(offset) - + box = None self.mode = "RGB" @@ -372,6 +359,27 @@ class EpsImageFile(ImageFile.ImageFile): if not box: raise IOError("cannot determine EPS bounding box") + def _find_offset(self, fp): + + s = fp.read(160) + + if s[:4] == b"%!PS": + # for HEAD without binary preview + fp.seek(0, 2) + length = fp.tell() + offset = 0 + elif i32(s[0:4]) == 0xC6D3D0C5: + # FIX for: Some EPS file not handled correctly / issue #302 + # EPS can contain binary data + # or start directly with latin coding + # more info see http://partners.adobe.com/public/developer/en/ps/5002.EPSF_Spec.pdf + offset = i32(s[4:8]) + length = i32(s[8:12]) + else: + raise SyntaxError("not an EPS file") + + return (length, offset) + def load(self, scale=1): # Load EPS via Ghostscript if not self.tile: From 8f75cc2bbf9cb274a7167f895ab0d437ddca1c07 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 7 Aug 2014 16:41:10 -0700 Subject: [PATCH 04/59] Don't search for the imagedata if we're not going to do anything with it --- PIL/EpsImagePlugin.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/PIL/EpsImagePlugin.py b/PIL/EpsImagePlugin.py index 88ce23b22..745e01b02 100644 --- a/PIL/EpsImagePlugin.py +++ b/PIL/EpsImagePlugin.py @@ -308,7 +308,7 @@ class EpsImageFile(ImageFile.ImageFile): s = s[:-1] if s[:11] == "%ImageData:": - + # Encoded bitmapped image. [x, y, bi, mo, z3, z4, en, id] =\ s[11:].split(None, 7) @@ -335,23 +335,10 @@ class EpsImageFile(ImageFile.ImageFile): self.mode = "RGB" else: break - - if id[:1] == id[-1:] == '"': - id = id[1:-1] - - # Scan forward to the actual image data - while True: - s = fp.readline() - if not s: - break - if s[:len(id)] == id: - self.size = x, y - self.tile2 = [(decoder, - (0, 0, x, y), - fp.tell(), - 0)] - return - + + self.size = (x,y) + return + s = fp.readline() if not s: break From ee46f45b96075593a5d4d39d9fa2b609e104fec4 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 8 Aug 2014 15:47:39 -0700 Subject: [PATCH 05/59] Remove unused code, tighten up readline for all the line endings --- PIL/EpsImagePlugin.py | 111 ++++++++++-------------------------------- 1 file changed, 26 insertions(+), 85 deletions(-) diff --git a/PIL/EpsImagePlugin.py b/PIL/EpsImagePlugin.py index 745e01b02..97680fec8 100644 --- a/PIL/EpsImagePlugin.py +++ b/PIL/EpsImagePlugin.py @@ -143,58 +143,28 @@ def Ghostscript(tile, size, fp, scale=1): class PSFile: - """Wrapper that treats either CR or LF as end of line.""" + """Wrapper for bytesio object that treats either CR or LF as end of line.""" def __init__(self, fp): self.fp = fp self.char = None - def __getattr__(self, id): - v = getattr(self.fp, id) - setattr(self, id, v) - return v def seek(self, offset, whence=0): self.char = None self.fp.seek(offset, whence) - def read(self, count): - return self.fp.read(count).decode('latin-1') - def readbinary(self, count): - return self.fp.read(count) - def tell(self): - pos = self.fp.tell() - if self.char: - pos -= 1 - return pos def readline(self): - s = b"" - if self.char: - c = self.char - self.char = None - else: - c = self.fp.read(1) + s = self.char or b"" + self.char = None + + c = self.fp.read(1) while c not in b"\r\n": s = s + c c = self.fp.read(1) - if c == b"\r": - self.char = self.fp.read(1) - if self.char == b"\n": - self.char = None - return s.decode('latin-1') + "\n" - -class PSFpWrapper: - """ Wrapper for a filepointer that has been opened in universal mode """ - def __init__(self,fp): - self.fp = fp - - def __getattr__(self, attr): - """ Delegate everything that we're not wrapping """ - return getattr(self.fp, attr) - - def read(self, count): - return self.fp.read(count).decode('latin-1') - - def readbinary(self, count): - return self.fp.read(count) - + self.char = self.fp.read(1) + # line endings can be 1 or 2 of \r \n, in either order + if self.char in b"\r\n": + self.char = None + + return s.decode('latin-1') def _accept(prefix): return prefix[:4] == b"%!PS" or i32(prefix) == 0xC6D3D0C5 @@ -209,6 +179,8 @@ class EpsImageFile(ImageFile.ImageFile): format = "EPS" format_description = "Encapsulated Postscript" + mode_map = { 1:"L", 2:"LAB", 3:"RGB" } + def _open(self): (length, offset) = self._find_offset(self.fp) @@ -216,8 +188,8 @@ class EpsImageFile(ImageFile.ImageFile): # convert line endings and decode to latin-1. try: if bytes is str: - # Python2, need the decode to latin-1 on read. - fp = PSFpWrapper(open(self.fp.name, "Ur")) + # Python2, no encoding conversion necessary + fp = open(self.fp.name, "Ur") else: # Python3, can use bare open command. fp = open(self.fp.name, "Ur", encoding='latin-1') @@ -236,18 +208,12 @@ class EpsImageFile(ImageFile.ImageFile): # # Load EPS header - s = fp.readline() + s = fp.readline().strip('\r\n') while s: - if len(s) > 255: raise SyntaxError("not an EPS file") - if s[-2:] == '\r\n': - s = s[:-2] - elif s[-1:] == '\n': - s = s[:-1] - try: m = split.match(s) except re.error as v: @@ -269,9 +235,7 @@ class EpsImageFile(ImageFile.ImageFile): pass else: - m = field.match(s) - if m: k = m.group(1) @@ -281,16 +245,16 @@ class EpsImageFile(ImageFile.ImageFile): self.info[k[:8]] = k[9:] else: self.info[k] = "" - elif s[0:1] == '%': + elif s[0] == '%': # handle non-DSC Postscript comments that some # tools mistakenly put in the Comments section pass else: raise IOError("bad EPS header") - s = fp.readline() + s = fp.readline().strip('\r\n') - if s[:1] != "%": + if s[0] != "%": break @@ -302,44 +266,21 @@ class EpsImageFile(ImageFile.ImageFile): if len(s) > 255: raise SyntaxError("not an EPS file") - if s[-2:] == '\r\n': - s = s[:-2] - elif s[-1:] == '\n': - s = s[:-1] - if s[:11] == "%ImageData:": # Encoded bitmapped image. - [x, y, bi, mo, z3, z4, en, id] =\ - s[11:].split(None, 7) + [x, y, bi, mo, z3, z4, en, id] = s[11:].split(None, 7) - x = int(x); y = int(y) - - bi = int(bi) - mo = int(mo) - - en = int(en) - - if en == 1: - decoder = "eps_binary" - elif en == 2: - decoder = "eps_hex" - else: + if int(bi) != 8: break - if bi != 8: - break - if mo == 1: - self.mode = "L" - elif mo == 2: - self.mode = "LAB" - elif mo == 3: - self.mode = "RGB" - else: + try: + self.mode = self.mode_map[int(mo)] + except: break - self.size = (x,y) + self.size = int(x), int(y) return - s = fp.readline() + s = fp.readline().strip('\r\n') if not s: break From 6dc276599eb15647020e40b623f64fb7e78baf3a Mon Sep 17 00:00:00 2001 From: wiredfool Date: Tue, 2 Sep 2014 22:47:05 -0700 Subject: [PATCH 06/59] test for all the different line ending characters --- Tests/test_file_eps.py | 65 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 5b369552a..4c8d1c014 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -148,6 +148,71 @@ class TestFileEps(PillowTestCase): # open image with binary preview Image.open(file3) + def _test_readline(self,t, ending): + ending = "Failure with line ending: %s" %("".join("%s" %ord(s) for s in ending)) + self.assertEqual(t.readline().strip('\r\n'), 'something', ending) + self.assertEqual(t.readline().strip('\r\n'), 'else', ending) + self.assertEqual(t.readline().strip('\r\n'), 'baz', ending) + self.assertEqual(t.readline().strip('\r\n'), 'bif', ending) + + def _test_readline_stringio(self, test_string, ending): + # check all the freaking line endings possible + try: + import StringIO + except: + # don't skip, it skips everything in the parent test + return + t = StringIO.StringIO(test_string) + self._test_readline(t, ending) + + def _test_readline_io(self, test_string, ending): + import io + t = io.StringIO(test_string) + self._test_readline(t, ending) + + def _test_readline_file_universal(self, test_string, ending): + f = self.tempfile('temp.txt') + with open(f,'wb') as w: + if str is bytes: + w.write(test_string) + else: + w.write(test_string.encode('UTF-8')) + + with open(f,'rU') as t: + self._test_readline(t, ending) + + def _test_readline_file_psfile(self, test_string, ending): + f = self.tempfile('temp.txt') + with open(f,'wb') as w: + if str is bytes: + w.write(test_string) + else: + w.write(test_string.encode('UTF-8')) + + with open(f,'rb') as r: + t = EpsImagePlugin.PSFile(r) + self._test_readline(t, ending) + + def test_readline(self): + # check all the freaking line endings possible from the spec + #test_string = u'something\r\nelse\n\rbaz\rbif\n' + line_endings = [u'\r\n', u'\n'] + not_working_endings = [u'\n\r', u'\r'] + strings = [u'something', u'else', u'baz', u'bif'] + + for ending in line_endings: + s = ending.join(strings) + self._test_readline_stringio(s, ending) + self._test_readline_io(s, ending) + self._test_readline_file_universal(s, ending) + + for ending in not_working_endings: + # these only work with the PSFile, while they're in spec, + # they're not likely to be used + s = ending.join(strings) + self._test_readline_file_psfile(s, ending) + + if __name__ == '__main__': unittest.main() From 81076d5f29d207e219224c29310677151f7c1b2b Mon Sep 17 00:00:00 2001 From: wiredfool Date: Tue, 2 Sep 2014 23:09:04 -0700 Subject: [PATCH 07/59] Don't copy input file for GS if not necessary, length variable name fix --- PIL/EpsImagePlugin.py | 49 ++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/PIL/EpsImagePlugin.py b/PIL/EpsImagePlugin.py index 97680fec8..58a4b7220 100644 --- a/PIL/EpsImagePlugin.py +++ b/PIL/EpsImagePlugin.py @@ -86,26 +86,32 @@ def Ghostscript(tile, size, fp, scale=1): out_fd, outfile = tempfile.mkstemp() os.close(out_fd) - in_fd, infile = tempfile.mkstemp() - os.close(in_fd) - - # ignore length and offset! - # ghostscript can read it - # copy whole file to read in ghostscript - with open(infile, 'wb') as f: - # fetch length of fp - fp.seek(0, 2) - fsize = fp.tell() - # ensure start position - # go back - fp.seek(0) - lengthfile = fsize - while lengthfile > 0: - s = fp.read(min(lengthfile, 100*1024)) - if not s: - break - length -= len(s) - f.write(s) + + infile_temp = None + if hasattr(fp, 'name') and os.path.exists(fp.name): + infile = fp.name + else: + in_fd, infile_temp = tempfile.mkstemp() + os.close(in_fd) + infile = infile_temp + + # ignore length and offset! + # ghostscript can read it + # copy whole file to read in ghostscript + with open(infile_temp, 'wb') as f: + # fetch length of fp + fp.seek(0, 2) + fsize = fp.tell() + # ensure start position + # go back + fp.seek(0) + lengthfile = fsize + while lengthfile > 0: + s = fp.read(min(lengthfile, 100*1024)) + if not s: + break + lengthfile -= len(s) + f.write(s) # Build ghostscript command command = ["gs", @@ -136,7 +142,8 @@ def Ghostscript(tile, size, fp, scale=1): finally: try: os.unlink(outfile) - os.unlink(infile) + if infile_temo: + os.unlink(infile_temp) except: pass return im From 918c77e98ac0a55586b322b5b05b46ea46ce37c6 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Tue, 2 Sep 2014 23:33:10 -0700 Subject: [PATCH 08/59] Py3.2 fix --- Tests/test_file_eps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 4c8d1c014..96b39b286 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -196,9 +196,9 @@ class TestFileEps(PillowTestCase): def test_readline(self): # check all the freaking line endings possible from the spec #test_string = u'something\r\nelse\n\rbaz\rbif\n' - line_endings = [u'\r\n', u'\n'] - not_working_endings = [u'\n\r', u'\r'] - strings = [u'something', u'else', u'baz', u'bif'] + line_endings = ['\r\n', '\n'] + not_working_endings = ['\n\r', '\r'] + strings = ['something', 'else', 'baz', 'bif'] for ending in line_endings: s = ending.join(strings) From 9cde0aa7d693de7ca268a8f594ca442fa6c85dd3 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 4 Sep 2014 10:09:05 -0700 Subject: [PATCH 09/59] Updated Changes.rst (formatting) [ci skip] --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index b8dbab630..8ce1a52ad 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,7 @@ Changelog (Pillow) ------------------ - Fix for reading multipage TIFFs #885 - [kostrom, wiredfool] + [kostrom, wiredfool] - Correctly handle saving gray and CMYK JPEGs with quality-keep #857 [etienned] From 255a090e9777bf6323d216734c0e8ec08526d022 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 10 Sep 2014 10:41:46 +1000 Subject: [PATCH 10/59] Avoid resize operation if image is already the correct size --- PIL/Image.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/PIL/Image.py b/PIL/Image.py index 5e1416a33..cd1e450ee 100644 --- a/PIL/Image.py +++ b/PIL/Image.py @@ -1514,6 +1514,9 @@ class Image: self.load() + if self.size == size: + return self._new(self.im) + if self.mode in ("1", "P"): resample = NEAREST From dd2608e1260a4933c0b9dfd78f0dcf4bdeb00647 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Wed, 10 Sep 2014 10:25:17 +0300 Subject: [PATCH 11/59] All network operations should be retried in order too prevent build failures due to network errors and large matrixes should fast finish. --- .travis.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 704c0d949..6226e5470 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,14 +16,14 @@ python: - 3.4 install: - - "sudo apt-get -qq install libfreetype6-dev liblcms2-dev python-qt4 ghostscript libffi-dev libjpeg-turbo-progs cmake imagemagick" - - "pip install cffi" - - "pip install coverage nose" + - "travis_retry sudo apt-get -qq install libfreetype6-dev liblcms2-dev python-qt4 ghostscript libffi-dev libjpeg-turbo-progs cmake imagemagick" + - "travis_retry pip install cffi" + - "travis_retry pip install coverage nose" # Pyroma installation is slow on Py3, so just do it for Py2. - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then travis_retry pip install pyroma; fi - - if [ "$TRAVIS_PYTHON_VERSION" == "2.6" ]; then pip install unittest2; fi + - if [ "$TRAVIS_PYTHON_VERSION" == "2.6" ]; then travis_retry pip install unittest2; fi # webp - pushd depends && ./install_webp.sh && popd @@ -41,20 +41,20 @@ script: after_success: # gather the coverage data - - sudo apt-get -qq install lcov + - travis_retry sudo apt-get -qq install lcov - lcov --capture --directory . -b . --output-file coverage.info # filter to remove system headers - lcov --remove coverage.info '/usr/*' -o coverage.filtered.info # convert to json - - gem install coveralls-lcov + - travis_retry gem install coveralls-lcov - coveralls-lcov -v -n coverage.filtered.info > coverage.c.json - coverage report - - pip install coveralls-merge + - travis_retry pip install coveralls-merge - coveralls-merge coverage.c.json - - pip install pep8 pyflakes + - travis_retry pip install pep8 pyflakes - pep8 --statistics --count PIL/*.py - pep8 --statistics --count Tests/*.py - pyflakes PIL/*.py | tee >(wc -l) @@ -65,3 +65,5 @@ after_success: # (Installation is very slow on Py3, so just do it for Py2.) - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then Scripts/diffcover-install.sh; fi - if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then Scripts/diffcover-run.sh; fi +matrix: + fast_finish: true From a2c2388dc81d34ba89b33148169da1e1abdbcfd6 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 10 Sep 2014 16:43:23 +0300 Subject: [PATCH 12/59] Add Landscape health badge [CI skip] See https://github.com/python-pillow/Pillow/issues/895 --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 482a1e2ec..d58666bc6 100644 --- a/README.rst +++ b/README.rst @@ -20,3 +20,6 @@ Pillow is the "friendly" PIL fork by `Alex Clark and Contributors Date: Wed, 10 Sep 2014 10:41:26 -0700 Subject: [PATCH 13/59] Update CHANGES.rst --- CHANGES.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8ce1a52ad..266da354d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,10 +4,13 @@ Changelog (Pillow) 2.6.0 (unreleased) ------------------ +- Don't resize if already the right size. + [radarhere] + - Fix for reading multipage TIFFs #885 [kostrom, wiredfool] -- Correctly handle saving gray and CMYK JPEGs with quality-keep #857 +- Correctly handle saving gray and CMYK JPEGs with quality=keep #857 [etienned] - Correct duplicate Tiff Metadata and Exif tag values From ce0fcef5804c1e6e0803092d881b48ceb274fd19 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 12 Sep 2014 21:41:12 -0700 Subject: [PATCH 14/59] Don't test internal python functions --- Tests/test_file_eps.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 96b39b286..a17ce274f 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -167,7 +167,10 @@ class TestFileEps(PillowTestCase): def _test_readline_io(self, test_string, ending): import io - t = io.StringIO(test_string) + if str is bytes: + t = io.StringIO(unicode(test_string)) + else: + t = io.StringIO(test_string) self._test_readline(t, ending) def _test_readline_file_universal(self, test_string, ending): @@ -202,14 +205,23 @@ class TestFileEps(PillowTestCase): for ending in line_endings: s = ending.join(strings) - self._test_readline_stringio(s, ending) - self._test_readline_io(s, ending) - self._test_readline_file_universal(s, ending) + # Native python versions will pass these endings. + #self._test_readline_stringio(s, ending) + #self._test_readline_io(s, ending) + #self._test_readline_file_universal(s, ending) + + self._test_readline_file_psfile(s, ending) for ending in not_working_endings: # these only work with the PSFile, while they're in spec, # they're not likely to be used - s = ending.join(strings) + s = ending.join(strings) + + # Native python versions may fail on these endings. + #self._test_readline_stringio(s, ending) + #self._test_readline_io(s, ending) + #self._test_readline_file_universal(s, ending) + self._test_readline_file_psfile(s, ending) From 5ca57520b6d028b1a3ad81cbf927aa3ab219c9f1 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 11:39:57 +1000 Subject: [PATCH 15/59] Added Test for JPEG2000 memory leak, before the fix is added --- Tests/test_jp2k_leak.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Tests/test_jp2k_leak.py diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py new file mode 100644 index 000000000..ffd061e00 --- /dev/null +++ b/Tests/test_jp2k_leak.py @@ -0,0 +1,30 @@ +from helper import unittest, PillowTestCase, tearDownModule +import sys + +from PIL import Image +from resource import setrlimit, RLIMIT_AS, RLIMIT_STACK + +# Limits for testing the leak +mem_limit = 512*1048576 +stack_size = 8*1048576 +iterations = (mem_limit/stack_size)*2 +codecs = dir(Image.core) +test_file = "Tests/images/rgb_trns_ycbc.jp2" + +@unittest.skipIf(sys.platform.startswith('win32'), "requires Unix or MacOS") +class TestJp2kLeak(PillowTestCase): + def setUp(self): + if "jpeg2k_encoder" not in codecs or "jpeg2k_decoder" not in codecs: + self.skipTest('JPEG 2000 support not available') + + def test_leak(self): + setrlimit(RLIMIT_STACK, (stack_size, stack_size)) + setrlimit(RLIMIT_AS, (mem_limit, mem_limit)) + for count in range(iterations): + with Image.open(test_file) as im: + im.load() + im.close() + + +if __name__ == '__main__': + unittest.main() From 3acb06e9d229f9d7208e4a93b74d821ac8a8a7e1 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 13:34:44 +1000 Subject: [PATCH 16/59] Added the JPEG2000 memory leak fix to decode.c --- decode.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/decode.c b/decode.c index e9aa6a387..c56f42592 100644 --- a/decode.c +++ b/decode.c @@ -103,6 +103,8 @@ PyImaging_DecoderNew(int contextsize) static void _dealloc(ImagingDecoderObject* decoder) { + if (decoder->cleanup) + decoder->cleanup(&decoder->state); free(decoder->state.buffer); free(decoder->state.context); Py_XDECREF(decoder->lock); From c134e5ab456f6b8ebd5a18c93d7554c9131fa39a Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 13:43:11 +1000 Subject: [PATCH 17/59] Moved resource import to inside the function --- Tests/test_jp2k_leak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index ffd061e00..35040df20 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -2,7 +2,6 @@ from helper import unittest, PillowTestCase, tearDownModule import sys from PIL import Image -from resource import setrlimit, RLIMIT_AS, RLIMIT_STACK # Limits for testing the leak mem_limit = 512*1048576 @@ -18,6 +17,7 @@ class TestJp2kLeak(PillowTestCase): self.skipTest('JPEG 2000 support not available') def test_leak(self): + from resource import setrlimit, RLIMIT_AS, RLIMIT_STACK setrlimit(RLIMIT_STACK, (stack_size, stack_size)) setrlimit(RLIMIT_AS, (mem_limit, mem_limit)) for count in range(iterations): From bc1e1c148cf339c9024fa0381fecab1019063401 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 13:44:38 +1000 Subject: [PATCH 18/59] Casting the iterations variable to integer --- Tests/test_jp2k_leak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index 35040df20..5b5fd6ab2 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -6,7 +6,7 @@ from PIL import Image # Limits for testing the leak mem_limit = 512*1048576 stack_size = 8*1048576 -iterations = (mem_limit/stack_size)*2 +iterations = int(mem_limit/stack_size)*2 codecs = dir(Image.core) test_file = "Tests/images/rgb_trns_ycbc.jp2" From f632baf4dc2497baa10b4a439c7dc6f95a55a1e1 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 14:04:24 +1000 Subject: [PATCH 19/59] Increased testing limit from 512MB to 1GB --- Tests/test_jp2k_leak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index 5b5fd6ab2..89419866c 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -4,7 +4,7 @@ import sys from PIL import Image # Limits for testing the leak -mem_limit = 512*1048576 +mem_limit = 1024*1048576 stack_size = 8*1048576 iterations = int(mem_limit/stack_size)*2 codecs = dir(Image.core) From a944ec92509ccc83a4802d890d70256f33e868cf Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 14:17:51 +1000 Subject: [PATCH 20/59] Increaded memory limit to 1.5Gb --- Tests/test_jp2k_leak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index 89419866c..7c91b4b85 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -4,7 +4,7 @@ import sys from PIL import Image # Limits for testing the leak -mem_limit = 1024*1048576 +mem_limit = 1536*1048576 stack_size = 8*1048576 iterations = int(mem_limit/stack_size)*2 codecs = dir(Image.core) From dea36ae610b9dcc1c3b5c9a55c661192d21c9cab Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 14:27:34 +1000 Subject: [PATCH 21/59] Tying 2GB max memory --- Tests/test_jp2k_leak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index 7c91b4b85..346416fe9 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -4,7 +4,7 @@ import sys from PIL import Image # Limits for testing the leak -mem_limit = 1536*1048576 +mem_limit = 2048*1048576 stack_size = 8*1048576 iterations = int(mem_limit/stack_size)*2 codecs = dir(Image.core) From a1f66bf402da265279b87e4e384949772a18b49f Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 14:44:09 +1000 Subject: [PATCH 22/59] Added a check for PyPy, as it needs far more memory as CPython --- Tests/test_jp2k_leak.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index 346416fe9..9b7db018f 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -1,10 +1,13 @@ from helper import unittest, PillowTestCase, tearDownModule import sys +from platform import python_implementation from PIL import Image # Limits for testing the leak -mem_limit = 2048*1048576 +mem_limit = 1024*1048576 +if python_implementation() == "PyPy": + mem_limit = 4096*1048576 stack_size = 8*1048576 iterations = int(mem_limit/stack_size)*2 codecs = dir(Image.core) From 0cd1d9526daa9ab221c423b1caa8d2c37e8f1e9c Mon Sep 17 00:00:00 2001 From: root Date: Tue, 26 Aug 2014 16:36:15 +1000 Subject: [PATCH 23/59] Reverted back to 512M, PyPy doesn't seem to work anyway --- Tests/test_jp2k_leak.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index 9b7db018f..70ae04c79 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -1,13 +1,11 @@ from helper import unittest, PillowTestCase, tearDownModule import sys -from platform import python_implementation +from os import getpid from PIL import Image # Limits for testing the leak -mem_limit = 1024*1048576 -if python_implementation() == "PyPy": - mem_limit = 4096*1048576 +mem_limit = 512*1048576 stack_size = 8*1048576 iterations = int(mem_limit/stack_size)*2 codecs = dir(Image.core) From 5a47b2bc845b899bab46589bc5a5b57389ec9637 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 26 Aug 2014 16:56:35 +1000 Subject: [PATCH 24/59] Sanity check :) --- Tests/test_jp2k_leak.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leak.py index 70ae04c79..4c3b5b365 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leak.py @@ -26,6 +26,18 @@ class TestJp2kLeak(PillowTestCase): im.load() im.close() + def self_sanity_check(self): + # Arrange + j2k = Image.open('Tests/images/rgb_trns_ycbc.j2k') + jp2 = Image.open('Tests/images/rgb_trns_ycbc.jp2') + + # Act + j2k.load() + jp2.load() + + # Assert + self.assertEqual(j2k.mode, 'RGBA') + self.assertEqual(jp2.mode, 'RGBA') if __name__ == '__main__': unittest.main() From 72e2a6cade4a17a964da6435eacb58478f783a08 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 22:22:35 +1000 Subject: [PATCH 25/59] Fix memory leak in JPEG2000 decoding, and JPEG decoding using PyPy --- .../{test_jp2k_leak.py => test_jp2k_leaks.py} | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) rename Tests/{test_jp2k_leak.py => test_jp2k_leaks.py} (63%) diff --git a/Tests/test_jp2k_leak.py b/Tests/test_jp2k_leaks.py similarity index 63% rename from Tests/test_jp2k_leak.py rename to Tests/test_jp2k_leaks.py index 4c3b5b365..1b7c68907 100644 --- a/Tests/test_jp2k_leak.py +++ b/Tests/test_jp2k_leaks.py @@ -1,18 +1,18 @@ from helper import unittest, PillowTestCase, tearDownModule import sys -from os import getpid +from StringIO import StringIO from PIL import Image # Limits for testing the leak -mem_limit = 512*1048576 +mem_limit = 768*1048576 stack_size = 8*1048576 -iterations = int(mem_limit/stack_size)*2 +iterations = int((mem_limit/stack_size)*20) codecs = dir(Image.core) test_file = "Tests/images/rgb_trns_ycbc.jp2" @unittest.skipIf(sys.platform.startswith('win32'), "requires Unix or MacOS") -class TestJp2kLeak(PillowTestCase): +class TestJpegLeaks(PillowTestCase): def setUp(self): if "jpeg2k_encoder" not in codecs or "jpeg2k_decoder" not in codecs: self.skipTest('JPEG 2000 support not available') @@ -24,20 +24,6 @@ class TestJp2kLeak(PillowTestCase): for count in range(iterations): with Image.open(test_file) as im: im.load() - im.close() - - def self_sanity_check(self): - # Arrange - j2k = Image.open('Tests/images/rgb_trns_ycbc.j2k') - jp2 = Image.open('Tests/images/rgb_trns_ycbc.jp2') - - # Act - j2k.load() - jp2.load() - - # Assert - self.assertEqual(j2k.mode, 'RGBA') - self.assertEqual(jp2.mode, 'RGBA') if __name__ == '__main__': unittest.main() From 7db19efe750ad9dc4b8dfc06d40d013e1ace51a9 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 22:25:23 +1000 Subject: [PATCH 26/59] Reduced JPEG2000 test iterations, and added cleanup for decoding using PyPy --- PIL/ImageFile.py | 2 ++ Tests/test_jp2k_leaks.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/PIL/ImageFile.py b/PIL/ImageFile.py index 5e4745d76..0550137ea 100644 --- a/PIL/ImageFile.py +++ b/PIL/ImageFile.py @@ -227,6 +227,8 @@ class ImageFile(Image.Image): break b = b[n:] t = t + n + # Need to cleanup here to prevent leaks in PyPy + d.cleanup() self.tile = [] self.readonly = readonly diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py index 1b7c68907..991a5cbf1 100644 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -7,7 +7,7 @@ from PIL import Image # Limits for testing the leak mem_limit = 768*1048576 stack_size = 8*1048576 -iterations = int((mem_limit/stack_size)*20) +iterations = int((mem_limit/stack_size)*2) codecs = dir(Image.core) test_file = "Tests/images/rgb_trns_ycbc.jp2" From b78e5444f4c4725d24d9720f3b05a4c20d4c432d Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 00:01:20 +1000 Subject: [PATCH 27/59] Prevent multiple calls to ImagingIncrementalCodecDestroy --- libImaging/Jpeg2KDecode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libImaging/Jpeg2KDecode.c b/libImaging/Jpeg2KDecode.c index 97ec81003..1477e68b9 100644 --- a/libImaging/Jpeg2KDecode.c +++ b/libImaging/Jpeg2KDecode.c @@ -800,6 +800,9 @@ ImagingJpeg2KDecodeCleanup(ImagingCodecState state) { if (context->decoder) ImagingIncrementalCodecDestroy(context->decoder); + /* Prevent multiple calls to ImagingIncrementalCodecDestroy */ + context->decoder = NULL; + return -1; } From bb738aef383a4df46b122cd910a4929a7fd8023c Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 00:15:19 +1000 Subject: [PATCH 28/59] Removed unnecessary StringIO dependency from JPEG2K leak test --- Tests/test_jp2k_leaks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py index 991a5cbf1..210e9ec62 100644 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -1,7 +1,5 @@ from helper import unittest, PillowTestCase, tearDownModule import sys -from StringIO import StringIO - from PIL import Image # Limits for testing the leak @@ -11,6 +9,7 @@ iterations = int((mem_limit/stack_size)*2) codecs = dir(Image.core) test_file = "Tests/images/rgb_trns_ycbc.jp2" + @unittest.skipIf(sys.platform.startswith('win32'), "requires Unix or MacOS") class TestJpegLeaks(PillowTestCase): def setUp(self): From caa95a26b2a0f1bf001a314b3efa4fb5824aed64 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 15:30:31 +1000 Subject: [PATCH 29/59] Added memory leak fix and testing for Encoder --- PIL/ImageFile.py | 2 ++ Tests/test_jp2k_leaks.py | 21 ++++++++++++++++++++- encode.c | 13 +++++++++++++ libImaging/Jpeg2KEncode.c | 4 ++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/PIL/ImageFile.py b/PIL/ImageFile.py index 0550137ea..bdcc769ec 100644 --- a/PIL/ImageFile.py +++ b/PIL/ImageFile.py @@ -473,6 +473,7 @@ def _save(im, fp, tile, bufsize=0): break if s < 0: raise IOError("encoder error %d when writing image file" % s) + e.cleanup() else: # slight speedup: compress to real file object for e, b, o, a in tile: @@ -483,6 +484,7 @@ def _save(im, fp, tile, bufsize=0): s = e.encode_to_file(fh, bufsize) if s < 0: raise IOError("encoder error %d when writing image file" % s) + e.cleanup() try: fp.flush() except: pass diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py index 210e9ec62..f99c1e73a 100644 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -8,6 +8,8 @@ stack_size = 8*1048576 iterations = int((mem_limit/stack_size)*2) codecs = dir(Image.core) test_file = "Tests/images/rgb_trns_ycbc.jp2" +from commands import getoutput +from os import getpid @unittest.skipIf(sys.platform.startswith('win32'), "requires Unix or MacOS") @@ -16,7 +18,7 @@ class TestJpegLeaks(PillowTestCase): if "jpeg2k_encoder" not in codecs or "jpeg2k_decoder" not in codecs: self.skipTest('JPEG 2000 support not available') - def test_leak(self): + def test_leak_load(self): from resource import setrlimit, RLIMIT_AS, RLIMIT_STACK setrlimit(RLIMIT_STACK, (stack_size, stack_size)) setrlimit(RLIMIT_AS, (mem_limit, mem_limit)) @@ -24,5 +26,22 @@ class TestJpegLeaks(PillowTestCase): with Image.open(test_file) as im: im.load() + def test_leak_save(self): + from resource import setrlimit, RLIMIT_AS, RLIMIT_STACK + try: + from cStringIO import StringIO + except ImportError: + from io import StringIO + setrlimit(RLIMIT_STACK, (stack_size, stack_size)) + setrlimit(RLIMIT_AS, (mem_limit, mem_limit)) + for count in range(iterations): + with Image.open(test_file) as im: + im.load() + test_output = StringIO() + im.save(test_output, "JPEG2000") + test_output.seek(0) + output = test_output.read() + + if __name__ == '__main__': unittest.main() diff --git a/encode.c b/encode.c index 3fa900b1d..34e3b8933 100644 --- a/encode.c +++ b/encode.c @@ -99,6 +99,18 @@ _dealloc(ImagingEncoderObject* encoder) PyObject_Del(encoder); } +static PyObject* +_encode_cleanup(ImagingEncoderObject* encoder, PyObject* args) +{ + int status = 0; + + if (encoder->cleanup){ + status = encoder->cleanup(&encoder->state); + } + + return Py_BuildValue("i", status); +} + static PyObject* _encode(ImagingEncoderObject* encoder, PyObject* args) { @@ -239,6 +251,7 @@ _setimage(ImagingEncoderObject* encoder, PyObject* args) static struct PyMethodDef methods[] = { {"encode", (PyCFunction)_encode, 1}, + {"cleanup", (PyCFunction)_encode_cleanup, 1}, {"encode_to_file", (PyCFunction)_encode_to_file, 1}, {"setimage", (PyCFunction)_setimage, 1}, {NULL, NULL} /* sentinel */ diff --git a/libImaging/Jpeg2KEncode.c b/libImaging/Jpeg2KEncode.c index e8eef08c1..b4c5284cb 100644 --- a/libImaging/Jpeg2KEncode.c +++ b/libImaging/Jpeg2KEncode.c @@ -585,6 +585,10 @@ ImagingJpeg2KEncodeCleanup(ImagingCodecState state) { if (context->encoder) ImagingIncrementalCodecDestroy(context->encoder); + /* Prevent multiple calls to ImagingIncrementalCodecDestroy */ + context->encoder = NULL; + + return -1; } From e791aa0325abd285b4b966d0a391fb3c1ee55bac Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 15:42:43 +1000 Subject: [PATCH 30/59] Removed unneeded dependencies --- Tests/test_jp2k_leaks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py index f99c1e73a..332520efb 100644 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -8,8 +8,6 @@ stack_size = 8*1048576 iterations = int((mem_limit/stack_size)*2) codecs = dir(Image.core) test_file = "Tests/images/rgb_trns_ycbc.jp2" -from commands import getoutput -from os import getpid @unittest.skipIf(sys.platform.startswith('win32'), "requires Unix or MacOS") From 11e34d695a651f155aea4828dc8bb7018f6f1204 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 16:03:07 +1000 Subject: [PATCH 31/59] Inwcreased max memory limit, and switched save test to use BytesIO instead of StringIO --- Tests/test_jp2k_leaks.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py index 332520efb..b0d84f47f 100644 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -1,9 +1,10 @@ from helper import unittest, PillowTestCase, tearDownModule import sys from PIL import Image +from io import BytesIO # Limits for testing the leak -mem_limit = 768*1048576 +mem_limit = 1024*1048576 stack_size = 8*1048576 iterations = int((mem_limit/stack_size)*2) codecs = dir(Image.core) @@ -26,16 +27,12 @@ class TestJpegLeaks(PillowTestCase): def test_leak_save(self): from resource import setrlimit, RLIMIT_AS, RLIMIT_STACK - try: - from cStringIO import StringIO - except ImportError: - from io import StringIO setrlimit(RLIMIT_STACK, (stack_size, stack_size)) setrlimit(RLIMIT_AS, (mem_limit, mem_limit)) for count in range(iterations): with Image.open(test_file) as im: im.load() - test_output = StringIO() + test_output = BytesIO() im.save(test_output, "JPEG2000") test_output.seek(0) output = test_output.read() From dace8913b89e9f6d79fcdd39a5a17940c50faf41 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 16:40:38 +1000 Subject: [PATCH 32/59] Increase memory yet again --- Tests/test_jp2k_leaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py index b0d84f47f..d93fde71f 100644 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -4,7 +4,7 @@ from PIL import Image from io import BytesIO # Limits for testing the leak -mem_limit = 1024*1048576 +mem_limit = 2048*1048576 stack_size = 8*1048576 iterations = int((mem_limit/stack_size)*2) codecs = dir(Image.core) From 3da6768a7227cacdc4b21d5292be0dd7ae5c4179 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 18:50:57 +1000 Subject: [PATCH 33/59] Testing whether e.cleanup causes segfaults --- PIL/ImageFile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PIL/ImageFile.py b/PIL/ImageFile.py index bdcc769ec..81549d8af 100644 --- a/PIL/ImageFile.py +++ b/PIL/ImageFile.py @@ -473,7 +473,7 @@ def _save(im, fp, tile, bufsize=0): break if s < 0: raise IOError("encoder error %d when writing image file" % s) - e.cleanup() + #e.cleanup() else: # slight speedup: compress to real file object for e, b, o, a in tile: @@ -484,7 +484,7 @@ def _save(im, fp, tile, bufsize=0): s = e.encode_to_file(fh, bufsize) if s < 0: raise IOError("encoder error %d when writing image file" % s) - e.cleanup() + #e.cleanup() try: fp.flush() except: pass From 7407371deb8b87e3049c9f1e8b04eac04a1fe6cc Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 19:13:42 +1000 Subject: [PATCH 34/59] NULLing pointers on cleanup --- PIL/ImageFile.py | 8 ++++---- libImaging/Jpeg2KDecode.c | 38 ++++++++++++++++++++------------------ libImaging/Jpeg2KEncode.c | 10 ++++++---- 3 files changed, 30 insertions(+), 26 deletions(-) diff --git a/PIL/ImageFile.py b/PIL/ImageFile.py index 81549d8af..1b5c2ebb2 100644 --- a/PIL/ImageFile.py +++ b/PIL/ImageFile.py @@ -135,8 +135,8 @@ class ImageFile(Image.Image): self.map = None use_mmap = self.filename and len(self.tile) == 1 # As of pypy 2.1.0, memory mapping was failing here. - use_mmap = use_mmap and not hasattr(sys, 'pypy_version_info') - + use_mmap = use_mmap and not hasattr(sys, 'pypy_version_info') + readonly = 0 # look for read/seek overrides @@ -473,7 +473,7 @@ def _save(im, fp, tile, bufsize=0): break if s < 0: raise IOError("encoder error %d when writing image file" % s) - #e.cleanup() + e.cleanup() else: # slight speedup: compress to real file object for e, b, o, a in tile: @@ -484,7 +484,7 @@ def _save(im, fp, tile, bufsize=0): s = e.encode_to_file(fh, bufsize) if s < 0: raise IOError("encoder error %d when writing image file" % s) - #e.cleanup() + e.cleanup() try: fp.flush() except: pass diff --git a/libImaging/Jpeg2KDecode.c b/libImaging/Jpeg2KDecode.c index 1477e68b9..bd6b59a62 100644 --- a/libImaging/Jpeg2KDecode.c +++ b/libImaging/Jpeg2KDecode.c @@ -68,7 +68,7 @@ j2k_skip(OPJ_OFF_T p_nb_bytes, void *p_user_data) typedef void (*j2k_unpacker_t)(opj_image_t *in, const JPEG2KTILEINFO *tileInfo, - const UINT8 *data, + const UINT8 *data, Imaging im); struct j2k_decode_unpacker { @@ -335,7 +335,7 @@ j2ku_srgb_rgb(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; for (n = 0; n < 3; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 3; ++n) { UINT32 word = 0; @@ -388,7 +388,7 @@ j2ku_sycc_rgb(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row_start = row; for (n = 0; n < 3; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 3; ++n) { UINT32 word = 0; @@ -442,7 +442,7 @@ j2ku_srgba_rgba(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; for (n = 0; n < 4; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 4; ++n) { UINT32 word = 0; @@ -494,7 +494,7 @@ j2ku_sycca_rgba(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row_start = row; for (n = 0; n < 4; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 4; ++n) { UINT32 word = 0; @@ -587,13 +587,13 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, /* Setup decompression context */ context->error_msg = NULL; - + opj_set_default_decoder_parameters(¶ms); params.cp_reduce = context->reduce; params.cp_layer = context->layers; - + codec = opj_create_decompress(context->format); - + if (!codec) { state->errcode = IMAGING_CODEC_BROKEN; state->state = J2K_STATE_FAILED; @@ -616,7 +616,7 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, state->state = J2K_STATE_FAILED; goto quick_exit; } - + for (n = 1; n < image->numcomps; ++n) { if (image->comps[n].dx != 1 || image->comps[n].dy != 1) { state->errcode = IMAGING_CODEC_BROKEN; @@ -624,8 +624,8 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, goto quick_exit; } } - - /* + + /* Colorspace Number of components PIL mode ------------------------------------------------------ sRGB 3 RGB @@ -633,22 +633,22 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, gray 1 L or I gray 2 LA YCC 3 YCbCr - - + + If colorspace is unspecified, we assume: - + Number of components Colorspace ----------------------------------------- 1 gray 2 gray (+ alpha) 3 sRGB 4 sRGB (+ alpha) - + */ - + /* Find the correct unpacker */ color_space = image->color_space; - + if (color_space == OPJ_CLRSPC_UNSPECIFIED) { switch (image->numcomps) { case 1: case 2: color_space = OPJ_CLRSPC_GRAY; break; @@ -668,7 +668,7 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, if (!unpack) { state->errcode = IMAGING_CODEC_BROKEN; state->state = J2K_STATE_FAILED; - goto quick_exit; + goto quick_exit; } /* Decode the image tile-by-tile; this means we only need use as much @@ -797,6 +797,8 @@ ImagingJpeg2KDecodeCleanup(ImagingCodecState state) { if (context->error_msg) free ((void *)context->error_msg); + context->error_msg = NULL; + if (context->decoder) ImagingIncrementalCodecDestroy(context->decoder); diff --git a/libImaging/Jpeg2KEncode.c b/libImaging/Jpeg2KEncode.c index b4c5284cb..5d6caf30c 100644 --- a/libImaging/Jpeg2KEncode.c +++ b/libImaging/Jpeg2KEncode.c @@ -303,7 +303,7 @@ j2k_encode_entry(Imaging im, ImagingCodecState state, prec = 16; bpp = 12; } else if (strcmp (im->mode, "LA") == 0) { - components = 2; + components = 2; color_space = OPJ_CLRSPC_GRAY; pack = j2k_pack_la; } else if (strcmp (im->mode, "RGB") == 0) { @@ -340,7 +340,7 @@ j2k_encode_entry(Imaging im, ImagingCodecState state, context->error_msg = NULL; opj_set_default_encoder_parameters(¶ms); - + params.image_offset_x0 = context->offset_x; params.image_offset_y0 = context->offset_y; @@ -546,8 +546,8 @@ ImagingJpeg2KEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) return -1; if (state->state == J2K_STATE_START) { - int seekable = (context->format != OPJ_CODEC_J2K - ? INCREMENTAL_CODEC_SEEKABLE + int seekable = (context->format != OPJ_CODEC_J2K + ? INCREMENTAL_CODEC_SEEKABLE : INCREMENTAL_CODEC_NOT_SEEKABLE); context->encoder = ImagingIncrementalCodecCreate(j2k_encode_entry, @@ -582,6 +582,8 @@ ImagingJpeg2KEncodeCleanup(ImagingCodecState state) { if (context->error_msg) free ((void *)context->error_msg); + context->error_msg = NULL; + if (context->encoder) ImagingIncrementalCodecDestroy(context->encoder); From 9ccc9307841bd8649d32fad91d2853bae6d436e0 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 19:21:40 +1000 Subject: [PATCH 35/59] Looking for cleanup segfault --- libImaging/Jpeg2KEncode.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 libImaging/Jpeg2KEncode.c diff --git a/libImaging/Jpeg2KEncode.c b/libImaging/Jpeg2KEncode.c old mode 100644 new mode 100755 index 5d6caf30c..1ed5e08fc --- a/libImaging/Jpeg2KEncode.c +++ b/libImaging/Jpeg2KEncode.c @@ -584,11 +584,11 @@ ImagingJpeg2KEncodeCleanup(ImagingCodecState state) { context->error_msg = NULL; - if (context->encoder) - ImagingIncrementalCodecDestroy(context->encoder); + //if (context->encoder) + // ImagingIncrementalCodecDestroy(context->encoder); /* Prevent multiple calls to ImagingIncrementalCodecDestroy */ - context->encoder = NULL; + //context->encoder = NULL; return -1; From e4e1f5c2d4146d42806684f7603819f196b3b67e Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 19:30:19 +1000 Subject: [PATCH 36/59] More testing... --- libImaging/Jpeg2KEncode.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libImaging/Jpeg2KEncode.c b/libImaging/Jpeg2KEncode.c index 1ed5e08fc..e7b5a78c3 100755 --- a/libImaging/Jpeg2KEncode.c +++ b/libImaging/Jpeg2KEncode.c @@ -576,13 +576,13 @@ int ImagingJpeg2KEncodeCleanup(ImagingCodecState state) { JPEG2KENCODESTATE *context = (JPEG2KENCODESTATE *)state->context; - if (context->quality_layers) - Py_DECREF(context->quality_layers); + //if (context->quality_layers) + // Py_DECREF(context->quality_layers); - if (context->error_msg) - free ((void *)context->error_msg); + //if (context->error_msg) + // free ((void *)context->error_msg); - context->error_msg = NULL; + //context->error_msg = NULL; //if (context->encoder) // ImagingIncrementalCodecDestroy(context->encoder); From dd221d9ec061d51b53c5f07daef044f92bb5a2e5 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Wed, 27 Aug 2014 19:54:33 +1000 Subject: [PATCH 37/59] Don't Py_DECREF context->quality_layers if there is no encoder --- libImaging/Jpeg2KEncode.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/libImaging/Jpeg2KEncode.c b/libImaging/Jpeg2KEncode.c index e7b5a78c3..d493cd637 100755 --- a/libImaging/Jpeg2KEncode.c +++ b/libImaging/Jpeg2KEncode.c @@ -555,7 +555,6 @@ ImagingJpeg2KEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) INCREMENTAL_CODEC_WRITE, seekable, context->fd); - if (!context->encoder) { state->errcode = IMAGING_CODEC_BROKEN; state->state = J2K_STATE_FAILED; @@ -576,19 +575,19 @@ int ImagingJpeg2KEncodeCleanup(ImagingCodecState state) { JPEG2KENCODESTATE *context = (JPEG2KENCODESTATE *)state->context; - //if (context->quality_layers) - // Py_DECREF(context->quality_layers); + if (context->quality_layers && context->encoder) + Py_DECREF(context->quality_layers); - //if (context->error_msg) - // free ((void *)context->error_msg); + if (context->error_msg) + free ((void *)context->error_msg); - //context->error_msg = NULL; + context->error_msg = NULL; - //if (context->encoder) - // ImagingIncrementalCodecDestroy(context->encoder); + if (context->encoder) + ImagingIncrementalCodecDestroy(context->encoder); - /* Prevent multiple calls to ImagingIncrementalCodecDestroy */ - //context->encoder = NULL; + /* Prevent multiple calls to ImagingIncrementalCodecDestroy */ + context->encoder = NULL; return -1; From 94194ed24803249c379e65c57e7aab6c3f208b47 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Thu, 28 Aug 2014 21:33:18 +1000 Subject: [PATCH 38/59] Cleaning up. Reduced memory limit to 1GB --- PIL/ImageFile.py | 4 ++-- Tests/test_jp2k_leaks.py | 2 +- libImaging/Jpeg2KDecode.c | 40 +++++++++++++++++++-------------------- libImaging/Jpeg2KEncode.c | 10 +++++----- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/PIL/ImageFile.py b/PIL/ImageFile.py index 1b5c2ebb2..bdcc769ec 100644 --- a/PIL/ImageFile.py +++ b/PIL/ImageFile.py @@ -135,8 +135,8 @@ class ImageFile(Image.Image): self.map = None use_mmap = self.filename and len(self.tile) == 1 # As of pypy 2.1.0, memory mapping was failing here. - use_mmap = use_mmap and not hasattr(sys, 'pypy_version_info') - + use_mmap = use_mmap and not hasattr(sys, 'pypy_version_info') + readonly = 0 # look for read/seek overrides diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py index d93fde71f..b0d84f47f 100644 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -4,7 +4,7 @@ from PIL import Image from io import BytesIO # Limits for testing the leak -mem_limit = 2048*1048576 +mem_limit = 1024*1048576 stack_size = 8*1048576 iterations = int((mem_limit/stack_size)*2) codecs = dir(Image.core) diff --git a/libImaging/Jpeg2KDecode.c b/libImaging/Jpeg2KDecode.c index bd6b59a62..abf8cebbe 100644 --- a/libImaging/Jpeg2KDecode.c +++ b/libImaging/Jpeg2KDecode.c @@ -68,7 +68,7 @@ j2k_skip(OPJ_OFF_T p_nb_bytes, void *p_user_data) typedef void (*j2k_unpacker_t)(opj_image_t *in, const JPEG2KTILEINFO *tileInfo, - const UINT8 *data, + const UINT8 *data, Imaging im); struct j2k_decode_unpacker { @@ -335,7 +335,7 @@ j2ku_srgb_rgb(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; for (n = 0; n < 3; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 3; ++n) { UINT32 word = 0; @@ -388,7 +388,7 @@ j2ku_sycc_rgb(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row_start = row; for (n = 0; n < 3; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 3; ++n) { UINT32 word = 0; @@ -442,7 +442,7 @@ j2ku_srgba_rgba(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; for (n = 0; n < 4; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 4; ++n) { UINT32 word = 0; @@ -494,7 +494,7 @@ j2ku_sycca_rgba(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, UINT8 *row_start = row; for (n = 0; n < 4; ++n) data[n] = &cdata[n][csiz[n] * y * w]; - + for (x = 0; x < w; ++x) { for (n = 0; n < 4; ++n) { UINT32 word = 0; @@ -587,13 +587,13 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, /* Setup decompression context */ context->error_msg = NULL; - + opj_set_default_decoder_parameters(¶ms); params.cp_reduce = context->reduce; params.cp_layer = context->layers; - + codec = opj_create_decompress(context->format); - + if (!codec) { state->errcode = IMAGING_CODEC_BROKEN; state->state = J2K_STATE_FAILED; @@ -616,7 +616,7 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, state->state = J2K_STATE_FAILED; goto quick_exit; } - + for (n = 1; n < image->numcomps; ++n) { if (image->comps[n].dx != 1 || image->comps[n].dy != 1) { state->errcode = IMAGING_CODEC_BROKEN; @@ -624,8 +624,8 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, goto quick_exit; } } - - /* + + /* Colorspace Number of components PIL mode ------------------------------------------------------ sRGB 3 RGB @@ -633,22 +633,22 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, gray 1 L or I gray 2 LA YCC 3 YCbCr - - + + If colorspace is unspecified, we assume: - + Number of components Colorspace ----------------------------------------- 1 gray 2 gray (+ alpha) 3 sRGB 4 sRGB (+ alpha) - + */ - + /* Find the correct unpacker */ color_space = image->color_space; - + if (color_space == OPJ_CLRSPC_UNSPECIFIED) { switch (image->numcomps) { case 1: case 2: color_space = OPJ_CLRSPC_GRAY; break; @@ -668,7 +668,7 @@ j2k_decode_entry(Imaging im, ImagingCodecState state, if (!unpack) { state->errcode = IMAGING_CODEC_BROKEN; state->state = J2K_STATE_FAILED; - goto quick_exit; + goto quick_exit; } /* Decode the image tile-by-tile; this means we only need use as much @@ -797,11 +797,11 @@ ImagingJpeg2KDecodeCleanup(ImagingCodecState state) { if (context->error_msg) free ((void *)context->error_msg); - context->error_msg = NULL; - if (context->decoder) ImagingIncrementalCodecDestroy(context->decoder); + context->error_msg = NULL; + /* Prevent multiple calls to ImagingIncrementalCodecDestroy */ context->decoder = NULL; diff --git a/libImaging/Jpeg2KEncode.c b/libImaging/Jpeg2KEncode.c index d493cd637..ea4bca2f2 100755 --- a/libImaging/Jpeg2KEncode.c +++ b/libImaging/Jpeg2KEncode.c @@ -303,7 +303,7 @@ j2k_encode_entry(Imaging im, ImagingCodecState state, prec = 16; bpp = 12; } else if (strcmp (im->mode, "LA") == 0) { - components = 2; + components = 2; color_space = OPJ_CLRSPC_GRAY; pack = j2k_pack_la; } else if (strcmp (im->mode, "RGB") == 0) { @@ -340,7 +340,7 @@ j2k_encode_entry(Imaging im, ImagingCodecState state, context->error_msg = NULL; opj_set_default_encoder_parameters(¶ms); - + params.image_offset_x0 = context->offset_x; params.image_offset_y0 = context->offset_y; @@ -546,8 +546,8 @@ ImagingJpeg2KEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) return -1; if (state->state == J2K_STATE_START) { - int seekable = (context->format != OPJ_CODEC_J2K - ? INCREMENTAL_CODEC_SEEKABLE + int seekable = (context->format != OPJ_CODEC_J2K + ? INCREMENTAL_CODEC_SEEKABLE : INCREMENTAL_CODEC_NOT_SEEKABLE); context->encoder = ImagingIncrementalCodecCreate(j2k_encode_entry, @@ -555,6 +555,7 @@ ImagingJpeg2KEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) INCREMENTAL_CODEC_WRITE, seekable, context->fd); + if (!context->encoder) { state->errcode = IMAGING_CODEC_BROKEN; state->state = J2K_STATE_FAILED; @@ -589,7 +590,6 @@ ImagingJpeg2KEncodeCleanup(ImagingCodecState state) { /* Prevent multiple calls to ImagingIncrementalCodecDestroy */ context->encoder = NULL; - return -1; } From b07baf4549829dd81703ad9e6c59c0dd7feb6de1 Mon Sep 17 00:00:00 2001 From: Josh Ware Date: Tue, 26 Aug 2014 13:22:53 +1000 Subject: [PATCH 39/59] Removed tearDownModule from Test import --- Tests/test_jp2k_leaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 Tests/test_jp2k_leaks.py diff --git a/Tests/test_jp2k_leaks.py b/Tests/test_jp2k_leaks.py old mode 100644 new mode 100755 index b0d84f47f..9dbb8c1f4 --- a/Tests/test_jp2k_leaks.py +++ b/Tests/test_jp2k_leaks.py @@ -1,4 +1,4 @@ -from helper import unittest, PillowTestCase, tearDownModule +from helper import unittest, PillowTestCase import sys from PIL import Image from io import BytesIO From 922a040629aeaeaa7700c0c7fd28b8abf955f7db Mon Sep 17 00:00:00 2001 From: wiredfool Date: Fri, 12 Sep 2014 22:45:02 -0700 Subject: [PATCH 40/59] Don't run memory leak test automatically. --- Tests/{test_jp2k_leaks.py => check_j2k_leaks.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Tests/{test_jp2k_leaks.py => check_j2k_leaks.py} (100%) diff --git a/Tests/test_jp2k_leaks.py b/Tests/check_j2k_leaks.py similarity index 100% rename from Tests/test_jp2k_leaks.py rename to Tests/check_j2k_leaks.py From 02c8188060abca0d5185828d8b824794bb254026 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 13 Sep 2014 18:32:18 +0300 Subject: [PATCH 41/59] Update CHANGES.rst [CI skip] --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 266da354d..cc92eba8d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,12 @@ Changelog (Pillow) 2.6.0 (unreleased) ------------------ +- Jpeg2k Decode/Encode Memory Leak Fix #898 + [joshware, wiredfool] + +- EpsFilePlugin Speed improvements #886 + [wiredfool, karstenw] + - Don't resize if already the right size. [radarhere] From b1187c7dc7d91086602c6c6546ce7d9df9cdcb21 Mon Sep 17 00:00:00 2001 From: tyrannosaurus Date: Sat, 13 Sep 2014 20:36:14 +0400 Subject: [PATCH 42/59] Fix heading --- docs/handbook/image-file-formats.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/handbook/image-file-formats.rst b/docs/handbook/image-file-formats.rst index 97caea722..cd187a4a2 100644 --- a/docs/handbook/image-file-formats.rst +++ b/docs/handbook/image-file-formats.rst @@ -670,6 +670,7 @@ files, using either JPEG or HEX encoding depending on the image mode (and whether JPEG support is available or not). PIXAR (read only) +^^^^ PIL provides limited support for PIXAR raster files. The library can identify and read “dumped” RGB files. From ba696ab7f8eae98881e218667e573fc89905cf63 Mon Sep 17 00:00:00 2001 From: hugovk Date: Sun, 14 Sep 2014 21:46:55 +0300 Subject: [PATCH 43/59] Re-enable fail fast for tests --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b89406fc0..704c0d949 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,9 +37,7 @@ script: - CFLAGS="-coverage" python setup.py build_ext --inplace - coverage run --append --include=PIL/* selftest.py - # FIXME: re-add -x option to fail fast - # - coverage run --append --include=PIL/* -m nose -vx Tests/test_*.py - - coverage run --append --include=PIL/* -m nose -v Tests/test_*.py + - coverage run --append --include=PIL/* -m nose -vx Tests/test_*.py after_success: # gather the coverage data From bdf845d6df8dbc73af98ebb0f55d6754c27cec98 Mon Sep 17 00:00:00 2001 From: hugovk Date: Mon, 15 Sep 2014 08:59:28 +0300 Subject: [PATCH 44/59] Fix typo: fihopperme -> filename --- Tests/test_image_draft.py | 10 +++++----- Tests/test_mode_i16.py | 6 +++--- Tests/test_pickle.py | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Tests/test_image_draft.py b/Tests/test_image_draft.py index c7cd90e95..68676687c 100644 --- a/Tests/test_image_draft.py +++ b/Tests/test_image_draft.py @@ -2,13 +2,13 @@ from helper import unittest, PillowTestCase, fromstring, tostring from PIL import Image -codecs = dir(Image.core) -fihopperme = "Tests/images/hopper.jpg" -data = tostring(Image.open(fihopperme).resize((512, 512)), "JPEG") +CODECS = dir(Image.core) +FILENAME = "Tests/images/hopper.jpg" +DATA = tostring(Image.open(FILENAME).resize((512, 512)), "JPEG") def draft(mode, size): - im = fromstring(data) + im = fromstring(DATA) im.draft(mode, size) return im @@ -16,7 +16,7 @@ def draft(mode, size): class TestImageDraft(PillowTestCase): def setUp(self): - if "jpeg_encoder" not in codecs or "jpeg_decoder" not in codecs: + if "jpeg_encoder" not in CODECS or "jpeg_decoder" not in CODECS: self.skipTest("jpeg support not available") def test_size(self): diff --git a/Tests/test_mode_i16.py b/Tests/test_mode_i16.py index 4c6c181a8..0bd2b9a86 100644 --- a/Tests/test_mode_i16.py +++ b/Tests/test_mode_i16.py @@ -39,10 +39,10 @@ class TestModeI16(PillowTestCase): imOut = imIn.transform((w, h), Image.EXTENT, (0, 0, w, h)) self.verify(imOut) # transform - fihopperme = self.tempfile("temp.im") - imIn.save(fihopperme) + filename = self.tempfile("temp.im") + imIn.save(filename) - imOut = Image.open(fihopperme) + imOut = Image.open(filename) self.verify(imIn) self.verify(imOut) diff --git a/Tests/test_pickle.py b/Tests/test_pickle.py index 595141cb7..59dfd5948 100644 --- a/Tests/test_pickle.py +++ b/Tests/test_pickle.py @@ -6,13 +6,14 @@ from PIL import Image class TestPickle(PillowTestCase): def helper_pickle_file(self, pickle, protocol=0): + # Arrange im = Image.open('Tests/images/hopper.jpg') - fihopperme = self.tempfile('temp.pkl') + filename = self.tempfile('temp.pkl') # Act - with open(fihopperme, 'wb') as f: + with open(filename, 'wb') as f: pickle.dump(im, f, protocol) - with open(fihopperme, 'rb') as f: + with open(filename, 'rb') as f: loaded_im = pickle.load(f) # Assert From 99887a546763c950233f691cf5dd0107f38cfc81 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 16 Sep 2014 16:05:22 +0300 Subject: [PATCH 45/59] Improve the error message when importing ImageGrab on non-Windows (#901) --- PIL/ImageGrab.py | 5 ++++- Tests/test_imagegrab.py | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/PIL/ImageGrab.py b/PIL/ImageGrab.py index 9bb190934..ef0135334 100644 --- a/PIL/ImageGrab.py +++ b/PIL/ImageGrab.py @@ -17,6 +17,9 @@ from PIL import Image +import sys +if sys.platform != "win32": + raise ImportError("ImageGrab is Windows only") try: # built-in driver (1.1.3 and later) @@ -40,7 +43,7 @@ def grab(bbox=None): def grabclipboard(): - debug = 0 # temporary interface + debug = 0 # temporary interface data = Image.core.grabclipboard(debug) if isinstance(data, bytes): from PIL import BmpImagePlugin diff --git a/Tests/test_imagegrab.py b/Tests/test_imagegrab.py index 13affe6b9..dd6f50fb9 100644 --- a/Tests/test_imagegrab.py +++ b/Tests/test_imagegrab.py @@ -1,5 +1,8 @@ from helper import unittest, PillowTestCase +import exceptions +import sys + try: from PIL import ImageGrab @@ -19,6 +22,28 @@ except ImportError: self.skipTest("ImportError") +class TestImageGrabImport(PillowTestCase): + + def test_import(self): + # Arrange + exception = None + + # Act + try: + from PIL import ImageGrab + ImageGrab.__name__ # dummy to prevent Pyflakes warning + except Exception as exception: + pass + + # Assert + if sys.platform == 'win32': + self.assertIsNone(exception, None) + else: + self.assertIsInstance(exception, exceptions.ImportError) + self.assertEqual(exception.message, + "ImageGrab is Windows only") + + if __name__ == '__main__': unittest.main() From 300a3f0e70c5399e8401f1770cc5c35d99e18789 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 16 Sep 2014 16:44:51 +0300 Subject: [PATCH 46/59] Fix for Python 3 --- Tests/test_imagegrab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/test_imagegrab.py b/Tests/test_imagegrab.py index dd6f50fb9..e1c35ac4e 100644 --- a/Tests/test_imagegrab.py +++ b/Tests/test_imagegrab.py @@ -1,6 +1,5 @@ from helper import unittest, PillowTestCase -import exceptions import sys try: @@ -39,7 +38,7 @@ class TestImageGrabImport(PillowTestCase): if sys.platform == 'win32': self.assertIsNone(exception, None) else: - self.assertIsInstance(exception, exceptions.ImportError) + self.assertIsInstance(exception, ImportError) self.assertEqual(exception.message, "ImageGrab is Windows only") From 311a0c6f68f2600bcd5ce7e35c66339d5400ce4a Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 16 Sep 2014 17:19:15 +0300 Subject: [PATCH 47/59] Another Python 3 fix --- Tests/test_imagegrab.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/test_imagegrab.py b/Tests/test_imagegrab.py index e1c35ac4e..ec4572adb 100644 --- a/Tests/test_imagegrab.py +++ b/Tests/test_imagegrab.py @@ -31,8 +31,8 @@ class TestImageGrabImport(PillowTestCase): try: from PIL import ImageGrab ImageGrab.__name__ # dummy to prevent Pyflakes warning - except Exception as exception: - pass + except Exception as e: + exception = e # Assert if sys.platform == 'win32': From e5a068de53c6b44aa8be6c5145da1a727b38202d Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 16 Sep 2014 17:41:03 +0300 Subject: [PATCH 48/59] Yet another Python 3 fix --- Tests/test_imagegrab.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Tests/test_imagegrab.py b/Tests/test_imagegrab.py index ec4572adb..ea6b499b2 100644 --- a/Tests/test_imagegrab.py +++ b/Tests/test_imagegrab.py @@ -39,8 +39,7 @@ class TestImageGrabImport(PillowTestCase): self.assertIsNone(exception, None) else: self.assertIsInstance(exception, ImportError) - self.assertEqual(exception.message, - "ImageGrab is Windows only") + self.assertEqual(str(exception), "ImageGrab is Windows only") if __name__ == '__main__': From 45073c85d6ae14670ac86c6776e6b8c6dcd9a07a Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 17 Sep 2014 10:06:57 +0300 Subject: [PATCH 49/59] Remove Landscape badge until Landscape fixed Pillow isn't at 100% code health, there's plenty of PEP8 and Pyflakes warnings in our Travis CI. This is a bug in Landscape: https://github.com/landscapeio/landscape-issues/issues/70 Let's remove the badge until Landscape's working properly. Re: #895. --- README.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.rst b/README.rst index d58666bc6..a5cb77785 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,3 @@ Pillow is the "friendly" PIL fork by `Alex Clark and Contributors Date: Wed, 17 Sep 2014 20:24:34 +0300 Subject: [PATCH 50/59] Mention Coveralls [CI skip] --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a31c9ee09..30c375a17 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ Send a pull request. We'll generally want documentation and [tests](Tests/README - Fork the repo - Make a branch - Add your changes + Tests -- Run the test suite. Try to run on both Python 2.x and 3.x, or you'll get tripped up. You can enable [Travis CI on your repo](https://travis-ci.org/profile/) to catch test failures prior to the pull request. +- Run the test suite. Try to run on both Python 2.x and 3.x, or you'll get tripped up. You can enable [Travis CI on your repo](https://travis-ci.org/profile/) to catch test failures prior to the pull request, and [Coveralls](https://coveralls.io/repos/new) to see if the changed code is covered by tests. - Push to your fork, and make a pull request. A few guidelines: From c85e0e915b3bd5f40814ec1ebdcff2b6d3b860d8 Mon Sep 17 00:00:00 2001 From: Jim Garrison Date: Wed, 17 Sep 2014 21:36:59 -0700 Subject: [PATCH 51/59] Image.close() docstring: use correct "its" --- PIL/Image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PIL/Image.py b/PIL/Image.py index cd1e450ee..a7491fd83 100644 --- a/PIL/Image.py +++ b/PIL/Image.py @@ -530,7 +530,7 @@ class Image: """ Closes the file pointer, if possible. - This operation will destroy the image core and release it's memory. + This operation will destroy the image core and release its memory. The image data will be unusable afterward. This function is only required to close images that have not From 758c8930679a5d123bfac267a345d1b93949f1cc Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 18 Sep 2014 08:46:22 -0700 Subject: [PATCH 52/59] Convert lena->hopper --- Tests/test_imageenhance.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tests/test_imageenhance.py b/Tests/test_imageenhance.py index 433c49cf6..5edf46b12 100644 --- a/Tests/test_imageenhance.py +++ b/Tests/test_imageenhance.py @@ -1,4 +1,4 @@ -from helper import unittest, PillowTestCase, lena +from helper import unittest, PillowTestCase, hopper from PIL import Image from PIL import ImageEnhance @@ -10,10 +10,10 @@ class TestImageEnhance(PillowTestCase): # FIXME: assert_image # Implicit asserts no exception: - ImageEnhance.Color(lena()).enhance(0.5) - ImageEnhance.Contrast(lena()).enhance(0.5) - ImageEnhance.Brightness(lena()).enhance(0.5) - ImageEnhance.Sharpness(lena()).enhance(0.5) + ImageEnhance.Color(hopper()).enhance(0.5) + ImageEnhance.Contrast(hopper()).enhance(0.5) + ImageEnhance.Brightness(hopper()).enhance(0.5) + ImageEnhance.Sharpness(hopper()).enhance(0.5) def test_crash(self): From 5ea966d8698ff2adcff9b514f053babc7463450f Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 18 Sep 2014 08:49:13 -0700 Subject: [PATCH 53/59] Test for alpha preservation in ImageEnhance, #899 --- Tests/test_imageenhance.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Tests/test_imageenhance.py b/Tests/test_imageenhance.py index 5edf46b12..58042db85 100644 --- a/Tests/test_imageenhance.py +++ b/Tests/test_imageenhance.py @@ -22,6 +22,34 @@ class TestImageEnhance(PillowTestCase): ImageEnhance.Sharpness(im).enhance(0.5) + def _half_transparent_image(self): + # returns an image, half transparent, half solid + im = hopper('RGB') + + transparent = Image.new('L', im.size, 0) + solid = Image.new('L', (im.size[0]//2, im.size[1]), 255) + transparent.paste(solid, (0,0)) + im.putalpha(transparent) + + return im + + def _check_alpha(self,im, original, op, amount): + self.assertEqual(im.getbands(), original.getbands()) + self.assert_image_equal(im.split()[-1], original.split()[-1], + "Diff on %s: %s" % (op, amount)) + + def test_alpha(self): + # Issue https://github.com/python-pillow/Pillow/issues/899 + # Is alpha preserved through image enhancement? + + original = self._half_transparent_image() + + for op in ['Color', 'Brightness', 'Contrast', 'Sharpness']: + for amount in [0,0.5,1.0]: + self._check_alpha(getattr(ImageEnhance,op)(original).enhance(amount), + original, op, amount) + + if __name__ == '__main__': unittest.main() From 394f6d32491bb7e0a0c87545e49768476bbc2793 Mon Sep 17 00:00:00 2001 From: wiredfool Date: Thu, 18 Sep 2014 08:49:37 -0700 Subject: [PATCH 54/59] Fix for alpha preservation in ImageEnhance, #899 --- PIL/ImageEnhance.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/PIL/ImageEnhance.py b/PIL/ImageEnhance.py index f802dc1d3..a196d5b09 100644 --- a/PIL/ImageEnhance.py +++ b/PIL/ImageEnhance.py @@ -47,8 +47,11 @@ class Color(_Enhance): """ def __init__(self, image): self.image = image - self.degenerate = image.convert("L").convert(image.mode) + self.intermediate_mode = 'L' + if 'A' in image.getbands(): + self.intermediate_mode = 'LA' + self.degenerate = image.convert(self.intermediate_mode).convert(image.mode) class Contrast(_Enhance): """Adjust image contrast. @@ -62,6 +65,9 @@ class Contrast(_Enhance): mean = int(ImageStat.Stat(image.convert("L")).mean[0] + 0.5) self.degenerate = Image.new("L", image.size, mean).convert(image.mode) + if 'A' in image.getbands(): + self.degenerate.putalpha(image.split()[-1]) + class Brightness(_Enhance): """Adjust image brightness. @@ -74,6 +80,9 @@ class Brightness(_Enhance): self.image = image self.degenerate = Image.new(image.mode, image.size, 0) + if 'A' in image.getbands(): + self.degenerate.putalpha(image.split()[-1]) + class Sharpness(_Enhance): """Adjust image sharpness. @@ -85,3 +94,6 @@ class Sharpness(_Enhance): def __init__(self, image): self.image = image self.degenerate = image.filter(ImageFilter.SMOOTH) + + if 'A' in image.getbands(): + self.degenerate.putalpha(image.split()[-1]) From 516957bfa6e5418efb0edd178cd11b6b04fd1536 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 19 Sep 2014 09:53:21 +0300 Subject: [PATCH 55/59] Update CHANGES.rst [CI skip] --- CHANGES.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index cc92eba8d..5aa419ef9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,13 +4,16 @@ Changelog (Pillow) 2.6.0 (unreleased) ------------------ -- Jpeg2k Decode/Encode Memory Leak Fix #898 +- Retain alpha in ImageEnhance operations #909 + [wiredfool] + +- Jpeg2k Decode/encode memory leak fix #898 [joshware, wiredfool] - EpsFilePlugin Speed improvements #886 [wiredfool, karstenw] -- Don't resize if already the right size. +- Don't resize if already the right size #892 [radarhere] - Fix for reading multipage TIFFs #885 From af672b1d7e8a4545400777acdd6ce514f37d6cee Mon Sep 17 00:00:00 2001 From: wiredfool Date: Sat, 20 Sep 2014 10:27:52 -0700 Subject: [PATCH 56/59] Fix for handling P + transparency -> RGBA conversions --- PIL/Image.py | 10 ++++++++++ Tests/test_file_png.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/PIL/Image.py b/PIL/Image.py index 5e1416a33..058511b18 100644 --- a/PIL/Image.py +++ b/PIL/Image.py @@ -867,7 +867,17 @@ class Image: trns = trns_im.getpixel((0,0)) elif self.mode == 'P' and mode == 'RGBA': + t = self.info['transparency'] delete_trns = True + + if isinstance(t, bytes): + self.im.putpalettealphas(t) + elif isinstance(t, int): + self.im.putpalettealpha(t,0) + else: + raise ValueError("Transparency for P mode should" + + " be bytes or int") + if mode == "P" and palette == ADAPTIVE: im = self.im.quantize(colors) diff --git a/Tests/test_file_png.py b/Tests/test_file_png.py index e4d495330..b8f5ceb08 100644 --- a/Tests/test_file_png.py +++ b/Tests/test_file_png.py @@ -347,8 +347,8 @@ class TestFilePng(PillowTestCase): im2 = Image.open(f) self.assertIn('transparency', im2.info) - self.assert_image_similar(im2.convert('RGBA'), im.convert('RGBA'), - 16) + self.assert_image_equal(im2.convert('RGBA'), + im.convert('RGBA')) def test_save_icc_profile_none(self): # check saving files with an ICC profile set to None (omit profile) From f3e3af66865a7d4cadb3eeaa79af417401aa740b Mon Sep 17 00:00:00 2001 From: Christoph Gohlke Date: Sun, 21 Sep 2014 16:48:35 -0700 Subject: [PATCH 57/59] Fix msvc build error --- libImaging/TiffDecode.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libImaging/TiffDecode.c b/libImaging/TiffDecode.c index 1d320e9bd..76bd887a7 100644 --- a/libImaging/TiffDecode.c +++ b/libImaging/TiffDecode.c @@ -221,9 +221,10 @@ int ImagingLibTiffDecode(Imaging im, ImagingCodecState state, UINT8* buffer, int } if (clientstate->ifd){ - unsigned int ifdoffset = clientstate->ifd; + int rv; + unsigned int ifdoffset = clientstate->ifd; TRACE(("reading tiff ifd %d\n", ifdoffset)); - int rv = TIFFSetSubDirectory(tiff, ifdoffset); + rv = TIFFSetSubDirectory(tiff, ifdoffset); if (!rv){ TRACE(("error in TIFFSetSubDirectory")); return -1; From 09ba5560d8a94dc2f633e7bc11df214e0298df74 Mon Sep 17 00:00:00 2001 From: Christoph Gohlke Date: Sun, 21 Sep 2014 17:50:07 -0700 Subject: [PATCH 58/59] On Windows, do not execute convert.exe without specifying path Convert.exe is a system tool that converts file systems --- Tests/helper.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Tests/helper.py b/Tests/helper.py index 3d875983e..e510f307c 100644 --- a/Tests/helper.py +++ b/Tests/helper.py @@ -155,7 +155,7 @@ class PillowTestCase(unittest.TestCase): raise IOError() outfile = self.tempfile("temp.png") - if command_succeeds(['convert', f, outfile]): + if command_succeeds([IMCONVERT, f, outfile]): from PIL import Image return Image.open(outfile) raise IOError() @@ -251,6 +251,14 @@ def netpbm_available(): def imagemagick_available(): - return command_succeeds(['convert', '-version']) + return IMCONVERT and command_succeeds([IMCONVERT, '-version']) + + +if sys.platform == 'win32': + IMCONVERT = os.environ.get('MAGICK_HOME', '') + if IMCONVERT: + IMCONVERT = os.path.join(IMCONVERT, 'convert.exe') +else: + IMCONVERT = 'convert' # End of file From d328b4832019508cbad93a9d95535be60c16f0a1 Mon Sep 17 00:00:00 2001 From: Hugo Date: Mon, 22 Sep 2014 09:42:31 +0300 Subject: [PATCH 59/59] Update CHANGES.rst [CI skip] --- CHANGES.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 5aa419ef9..cd00ce285 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,15 @@ Changelog (Pillow) 2.6.0 (unreleased) ------------------ +- On Windows, do not execute convert.exe without specifying path #912 + [cgohlke] + +- Fix msvc build error #911 + [cgohlke] + +- Fix for handling P + transparency -> RGBA conversions #904 + [wiredfool] + - Retain alpha in ImageEnhance operations #909 [wiredfool]