diff --git a/.gitignore b/.gitignore index a2a3dc417..a0ba1b4c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,62 @@ -*.pyc -*.egg-info -build -dist -.tox +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions *.so -docs/_build + +# Distribution / packaging +.Python +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +# Sphinx documentation +docs/_build/ + +# Vim cruft +.*.swp + +#emacs +*~ +\#*# +.#* + diff --git a/.travis.yml b/.travis.yml index 472f8a9fa..34ffcfe1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,19 +4,41 @@ language: python virtualenv: system_site_packages: true +notifications: + irc: "chat.freenode.net#pil" + python: - 2.6 - 2.7 - 3.2 - 3.3 + - "pypy" -install: - - "sudo apt-get -qq install libfreetype6-dev liblcms2-dev libwebp-dev python-qt4 ghostscript libffi-dev" +install: + - "sudo apt-get -qq install libfreetype6-dev liblcms2-dev python-qt4 ghostscript libffi-dev cmake" - "pip install cffi" + - "pip install coveralls" + # webp + - pushd depends && ./install_webp.sh && popd + + # openjpeg + - pushd depends && ./install_openjpeg.sh && popd script: + - coverage erase - python setup.py clean - python setup.py build_ext --inplace - - python selftest.py - - python Tests/run.py + - coverage run --append --include=PIL/* selftest.py + - python Tests/run.py --coverage + +after_success: + - coverage report + - coveralls + - pip install pep8 pyflakes + - pep8 PIL/*.py + - pyflakes PIL/*.py + +matrix: + allow_failures: + - python: "pypy" diff --git a/CHANGES.rst b/CHANGES.rst index abdf73b33..f83c6f339 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,9 +1,66 @@ Changelog (Pillow) ================== -2.4.0 (unreleased) +2.5.0 (unreleased) ------------------ +- Have the tempfile use a suffix with a dot + [wiredfool] + +- Fix variable name used for transparency manipulations + [nijel] + +2.4.0 (2014-04-01) +------------------ + +- Indexed Transparency handled for conversions between L, RGB, and P modes. Fixes #510 + [wiredfool] + +- Conversions enabled from RGBA->P, Fixes #544 + [wiredfool] + +- Improved icns support + [al45tair] + +- Fix libtiff leaking open files, fixes #580 + [wiredfool] + +- Fixes for Jpeg encoding in Python 3, fixes #577 + [wiredfool] + +- Added support for JPEG 2000 + [al45tair] + +- Add more detailed error messages to Image.py + [larsmans] + +- Avoid conflicting _expand functions in PIL & MINGW, fixes #538 + [aclark] + +- Merge from Philippe Lagadec’s OleFileIO_PL fork + [vadmium] + +- Fix ImageColor.getcolor + [homm] + +- Make ICO files work with the ImageFile.Parser interface, fixes #522 + [wiredfool] + +- Handle 32bit compiled python on 64bit architecture + [choppsv1] + +- Fix support for characters >128 using .pcf or .pil fonts in Py3k. Fixes #505 + [wiredfool] + +- Skip CFFI test earlier if it's not installed + [wiredfool] + +- Fixed opening and saving odd sized .pcx files, fixes #523 + [wiredfool] + +- Fixed palette handling when converting from mode P->RGB->P + [d_schmidt] + - Fixed saving mode P image as a PNG with transparency = palette color 0 [d-schmidt] @@ -45,6 +102,12 @@ Changelog (Pillow) - Prefer homebrew freetype over X11 freetype (but still allow both) [dmckeone] + +2.3.1 (2014-03-14) +------------------ + +- Fix insecure use of tempfile.mktemp (CVE-2014-1932 CVE-2014-1933) + [wiredfool] 2.3.0 (2014-01-01) ------------------ diff --git a/Images/pillow.icns b/Images/pillow.icns new file mode 100644 index 000000000..a461329f0 Binary files /dev/null and b/Images/pillow.icns differ diff --git a/Images/pillow.ico b/Images/pillow.ico new file mode 100644 index 000000000..78eef9ae3 Binary files /dev/null and b/Images/pillow.ico differ diff --git a/MANIFEST.in b/MANIFEST.in index 3769b645e..c2358f76f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,6 +9,7 @@ include tox.ini recursive-include Images *.bdf recursive-include Images *.fli recursive-include Images *.gif +recursive-include Images *.icns recursive-include Images *.ico recursive-include Images *.jpg recursive-include Images *.pbm @@ -19,6 +20,7 @@ recursive-include Images *.psd recursive-include Images *.tar recursive-include Images *.webp recursive-include Images *.xpm +recursive-include PIL *.md recursive-include Sane *.c recursive-include Sane *.py recursive-include Sane *.txt @@ -27,9 +29,15 @@ recursive-include Sane README recursive-include Scripts *.py recursive-include Scripts README recursive-include Tests *.bin +recursive-include Tests *.bmp recursive-include Tests *.eps +recursive-include Tests *.gif recursive-include Tests *.gnuplot +recursive-include Tests *.html recursive-include Tests *.icm +recursive-include Tests *.icns +recursive-include Tests *.ico +recursive-include Tests *.jp2 recursive-include Tests *.jpg recursive-include Tests *.pcf recursive-include Tests *.pcx @@ -42,6 +50,7 @@ recursive-include Tests *.ttf recursive-include Tests *.txt recursive-include Tk *.c recursive-include Tk *.txt +recursive-include depends *.sh recursive-include docs *.bat recursive-include docs *.gitignore recursive-include docs *.html diff --git a/PIL/EpsImagePlugin.py b/PIL/EpsImagePlugin.py index 94f3e27f4..4d19c1f20 100644 --- a/PIL/EpsImagePlugin.py +++ b/PIL/EpsImagePlugin.py @@ -50,6 +50,21 @@ if sys.platform.startswith('win'): else: gs_windows_binary = False +def has_ghostscript(): + if gs_windows_binary: + return True + if not sys.platform.startswith('win'): + import subprocess + try: + gs = subprocess.Popen(['gs','--version'], stdout=subprocess.PIPE) + gs.stdout.read() + return True + except OSError: + # no ghostscript + pass + return False + + def Ghostscript(tile, size, fp, scale=1): """Render an image using Ghostscript""" @@ -67,8 +82,10 @@ def Ghostscript(tile, size, fp, scale=1): import tempfile, os, subprocess - outfile = tempfile.mktemp() - infile = tempfile.mktemp() + out_fd, outfile = tempfile.mkstemp() + os.close(out_fd) + in_fd, infile = tempfile.mkstemp() + os.close(in_fd) with open(infile, 'wb') as f: fp.seek(offset) diff --git a/PIL/GifImagePlugin.py b/PIL/GifImagePlugin.py index aed525824..c6d449425 100644 --- a/PIL/GifImagePlugin.py +++ b/PIL/GifImagePlugin.py @@ -237,7 +237,10 @@ def _save(im, fp, filename): # convert on the fly (EXPERIMENTAL -- I'm not sure PIL # should automatically convert images on save...) if Image.getmodebase(im.mode) == "RGB": - imOut = im.convert("P") + palette_size = 256 + if im.palette: + palette_size = len(im.palette.getdata()[1]) // 3 + imOut = im.convert("P", palette=1, colors=palette_size) rawmode = "P" else: imOut = im.convert("L") @@ -248,9 +251,13 @@ def _save(im, fp, filename): palette = im.encoderinfo["palette"] except KeyError: palette = None - if im.palette: - # use existing if possible - palette = im.palette.getdata()[1] + im.encoderinfo["optimize"] = im.encoderinfo.get("optimize", True) + if im.encoderinfo["optimize"]: + # When the mode is L, and we optimize, we end up with + # im.mode == P and rawmode = L, which fails. + # If we're optimizing the palette, we're going to be + # in a rawmode of P anyway. + rawmode = 'P' header, usedPaletteColors = getheader(imOut, palette, im.encoderinfo) for s in header: @@ -391,6 +398,9 @@ def getheader(im, palette=None, info=None): for i in range(len(imageBytes)): imageBytes[i] = newPositions[imageBytes[i]] im.frombytes(bytes(imageBytes)) + newPaletteBytes = paletteBytes + (768 - len(paletteBytes)) * b'\x00' + im.putpalette(newPaletteBytes) + im.palette = ImagePalette.ImagePalette("RGB", palette = paletteBytes, size = len(paletteBytes)) if not paletteBytes: paletteBytes = sourcePalette diff --git a/PIL/IcnsImagePlugin.py b/PIL/IcnsImagePlugin.py index 8fac1308c..9a0864bad 100644 --- a/PIL/IcnsImagePlugin.py +++ b/PIL/IcnsImagePlugin.py @@ -10,12 +10,17 @@ # Copyright (c) 2004 by Bob Ippolito. # Copyright (c) 2004 by Secret Labs. # Copyright (c) 2004 by Fredrik Lundh. +# Copyright (c) 2014 by Alastair Houghton. # # See the README file for information on usage and redistribution. # -from PIL import Image, ImageFile, _binary -import struct +from PIL import Image, ImageFile, PngImagePlugin, _binary +import struct, io + +enable_jpeg2k = hasattr(Image.core, 'jp2klib_version') +if enable_jpeg2k: + from PIL import Jpeg2KImagePlugin i8 = _binary.i8 @@ -40,14 +45,15 @@ def read_32(fobj, start_length, size): """ (start, length) = start_length fobj.seek(start) - sizesq = size[0] * size[1] + pixel_size = (size[0] * size[2], size[1] * size[2]) + sizesq = pixel_size[0] * pixel_size[1] if length == sizesq * 3: # uncompressed ("RGBRGBGB") indata = fobj.read(length) - im = Image.frombuffer("RGB", size, indata, "raw", "RGB", 0, 1) + im = Image.frombuffer("RGB", pixel_size, indata, "raw", "RGB", 0, 1) else: # decode image - im = Image.new("RGB", size, None) + im = Image.new("RGB", pixel_size, None) for band_ix in range(3): data = [] bytesleft = sizesq @@ -72,7 +78,7 @@ def read_32(fobj, start_length, size): "Error reading channel [%r left]" % bytesleft ) band = Image.frombuffer( - "L", size, b"".join(data), "raw", "L", 0, 1 + "L", pixel_size, b"".join(data), "raw", "L", 0, 1 ) im.im.putband(band.im, band_ix) return {"RGB": im} @@ -81,27 +87,80 @@ def read_mk(fobj, start_length, size): # Alpha masks seem to be uncompressed (start, length) = start_length fobj.seek(start) + pixel_size = (size[0] * size[2], size[1] * size[2]) + sizesq = pixel_size[0] * pixel_size[1] band = Image.frombuffer( - "L", size, fobj.read(size[0]*size[1]), "raw", "L", 0, 1 + "L", pixel_size, fobj.read(sizesq), "raw", "L", 0, 1 ) return {"A": band} +def read_png_or_jpeg2000(fobj, start_length, size): + (start, length) = start_length + fobj.seek(start) + sig = fobj.read(12) + if sig[:8] == b'\x89PNG\x0d\x0a\x1a\x0a': + fobj.seek(start) + im = PngImagePlugin.PngImageFile(fobj) + return {"RGBA": im} + elif sig[:4] == b'\xff\x4f\xff\x51' \ + or sig[:4] == b'\x0d\x0a\x87\x0a' \ + or sig == b'\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a': + if not enable_jpeg2k: + raise ValueError('Unsupported icon subimage format (rebuild PIL with JPEG 2000 support to fix this)') + # j2k, jpc or j2c + fobj.seek(start) + jp2kstream = fobj.read(length) + f = io.BytesIO(jp2kstream) + im = Jpeg2KImagePlugin.Jpeg2KImageFile(f) + if im.mode != 'RGBA': + im = im.convert('RGBA') + return {"RGBA": im} + else: + raise ValueError('Unsupported icon subimage format') + class IcnsFile: SIZES = { - (128, 128): [ + (512, 512, 2): [ + (b'ic10', read_png_or_jpeg2000), + ], + (512, 512, 1): [ + (b'ic09', read_png_or_jpeg2000), + ], + (256, 256, 2): [ + (b'ic14', read_png_or_jpeg2000), + ], + (256, 256, 1): [ + (b'ic08', read_png_or_jpeg2000), + ], + (128, 128, 2): [ + (b'ic13', read_png_or_jpeg2000), + ], + (128, 128, 1): [ + (b'ic07', read_png_or_jpeg2000), (b'it32', read_32t), (b't8mk', read_mk), ], - (48, 48): [ + (64, 64, 1): [ + (b'icp6', read_png_or_jpeg2000), + ], + (32, 32, 2): [ + (b'ic12', read_png_or_jpeg2000), + ], + (48, 48, 1): [ (b'ih32', read_32), (b'h8mk', read_mk), ], - (32, 32): [ + (32, 32, 1): [ + (b'icp5', read_png_or_jpeg2000), (b'il32', read_32), (b'l8mk', read_mk), ], - (16, 16): [ + (16, 16, 2): [ + (b'ic11', read_png_or_jpeg2000), + ], + (16, 16, 1): [ + (b'icp4', read_png_or_jpeg2000), (b'is32', read_32), (b's8mk', read_mk), ], @@ -115,7 +174,7 @@ class IcnsFile: self.dct = dct = {} self.fobj = fobj sig, filesize = nextheader(fobj) - if sig != 'icns': + if sig != b'icns': raise SyntaxError('not an icns file') i = HEADERSIZE while i < filesize: @@ -157,7 +216,14 @@ class IcnsFile: def getimage(self, size=None): if size is None: size = self.bestsize() + if len(size) == 2: + size = (size[0], size[1], 1) channels = self.dataforsize(size) + + im = channels.get('RGBA', None) + if im: + return im + im = channels.get("RGB").copy() try: im.putalpha(channels["A"]) @@ -185,18 +251,29 @@ class IcnsImageFile(ImageFile.ImageFile): def _open(self): self.icns = IcnsFile(self.fp) self.mode = 'RGBA' - self.size = self.icns.bestsize() + self.best_size = self.icns.bestsize() + self.size = (self.best_size[0] * self.best_size[2], + self.best_size[1] * self.best_size[2]) self.info['sizes'] = self.icns.itersizes() # Just use this to see if it's loaded or not yet. self.tile = ('',) def load(self): + if len(self.size) == 3: + self.best_size = self.size + self.size = (self.best_size[0] * self.best_size[2], + self.best_size[1] * self.best_size[2]) + Image.Image.load(self) if not self.tile: return self.load_prepare() # This is likely NOT the best way to do it, but whatever. - im = self.icns.getimage(self.size) + im = self.icns.getimage(self.best_size) + + # If this is a PNG or JPEG 2000, it won't be loaded yet + im.load() + self.im = im.im self.mode = im.mode self.size = im.size @@ -205,12 +282,18 @@ class IcnsImageFile(ImageFile.ImageFile): self.tile = () self.load_end() - Image.register_open("ICNS", IcnsImageFile, lambda x: x[:4] == b'icns') Image.register_extension("ICNS", '.icns') if __name__ == '__main__': import os, sys + imf = IcnsImageFile(open(sys.argv[1], 'rb')) + for size in imf.info['sizes']: + imf.size = size + imf.load() + im = imf.im + im.save('out-%s-%s-%s.png' % size) im = Image.open(open(sys.argv[1], "rb")) im.save("out.png") - os.startfile("out.png") + if sys.platform == 'windows': + os.startfile("out.png") diff --git a/PIL/IcoImagePlugin.py b/PIL/IcoImagePlugin.py index 82d33e383..268e93d6c 100644 --- a/PIL/IcoImagePlugin.py +++ b/PIL/IcoImagePlugin.py @@ -222,6 +222,10 @@ class IcoImageFile(ImageFile.ImageFile): self.mode = im.mode self.size = im.size + + def load_seek(self): + # Flage the ImageFile.Parser so that it just does all the decode at the end. + pass # # -------------------------------------------------------------------- diff --git a/PIL/Image.py b/PIL/Image.py index b93ce24a4..359aae716 100644 --- a/PIL/Image.py +++ b/PIL/Image.py @@ -504,14 +504,20 @@ class Image: self.readonly = 0 def _dump(self, file=None, format=None): - import tempfile + import tempfile, os + suffix = '' + if format: + suffix = '.'+format if not file: - file = tempfile.mktemp() + f, file = tempfile.mkstemp(suffix) + os.close(f) + self.load() if not format or format == "PPM": self.im.save_ppm(file) else: - file = file + "." + format + if not file.endswith(format): + file = file + "." + format self.save(file, format) return file @@ -732,18 +738,65 @@ class Image: im = self.im.convert_matrix(mode, matrix) return self._new(im) + if mode == "P" and self.mode == "RGBA": + return self.quantize(colors) + + trns = None + delete_trns = False + # transparency handling + if "transparency" in self.info and self.info['transparency'] is not None: + if self.mode in ('L', 'RGB') and mode == 'RGBA': + # Use transparent conversion to promote from transparent + # color to an alpha channel. + return self._new(self.im.convert_transparent( + mode, self.info['transparency'])) + elif self.mode in ('L', 'RGB', 'P') and mode in ('L', 'RGB', 'P'): + t = self.info['transparency'] + if isinstance(t, bytes): + # Dragons. This can't be represented by a single color + warnings.warn('Palette images with Transparency expressed '+ + ' in bytes should be converted to RGBA images') + delete_trns = True + else: + # get the new transparency color. + # use existing conversions + trns_im = Image()._new(core.new(self.mode, (1,1))) + if self.mode == 'P': + trns_im.putpalette(self.palette) + trns_im.putpixel((0,0), t) + + if mode in ('L','RGB'): + trns_im = trns_im.convert(mode) + else: + # can't just retrieve the palette number, got to do it + # after quantization. + trns_im = trns_im.convert('RGB') + trns = trns_im.getpixel((0,0)) + + if mode == "P" and palette == ADAPTIVE: im = self.im.quantize(colors) - return self._new(im) + new = self._new(im) + from PIL import ImagePalette + new.palette = ImagePalette.raw("RGB", new.im.getpalette("RGB")) + if delete_trns: + # This could possibly happen if we requantize to fewer colors. + # The transparency would be totally off in that case. + del(new.info['transparency']) + if trns is not None: + try: + new.info['transparency'] = new.palette.getcolor(trns) + except: + # if we can't make a transparent color, don't leave the old + # transparency hanging around to mess us up. + del(new.info['transparency']) + warnings.warn("Couldn't allocate palette entry for transparency") + return new # colorspace conversion if dither is None: dither = FLOYDSTEINBERG - - # Use transparent conversion to promote from transparent color to an alpha channel. - if self.mode in ("L", "RGB") and mode == "RGBA" and "transparency" in self.info: - return self._new(self.im.convert_transparent(mode, self.info['transparency'])) - + try: im = self.im.convert(mode, dither) except ValueError: @@ -754,9 +807,22 @@ class Image: except KeyError: raise ValueError("illegal conversion") - return self._new(im) + new_im = self._new(im) + if delete_trns: + #crash fail if we leave a bytes transparency in an rgb/l mode. + del(new_im.info['transparency']) + if trns is not None: + if new_im.mode == 'P': + try: + new_im.info['transparency'] = new_im.palette.getcolor(trns) + except: + del(new_im.info['transparency']) + warnings.warn("Couldn't allocate palette entry for transparency") + else: + new_im.info['transparency'] = trns + return new_im - def quantize(self, colors=256, method=0, kmeans=0, palette=None): + def quantize(self, colors=256, method=None, kmeans=0, palette=None): # methods: # 0 = median cut @@ -767,7 +833,18 @@ class Image: # quantizer interface in a later version of PIL. self.load() + + if method is None: + # defaults: + method = 0 + if self.mode == 'RGBA': + method = 2 + if self.mode == 'RGBA' and method != 2: + # Caller specified an invalid mode. + raise ValueError('Fast Octree (method == 2) is the ' + + ' only valid method for quantizing RGBA images') + if palette: # use palette from reference image palette.load() @@ -1959,7 +2036,7 @@ def fromarray(obj, mode=None): else: ndmax = 4 if ndim > ndmax: - raise ValueError("Too many dimensions.") + raise ValueError("Too many dimensions: %d > %d." % (ndim, ndmax)) size = shape[1], shape[0] if strides is not None: @@ -2012,7 +2089,7 @@ def open(fp, mode="r"): """ if mode != "r": - raise ValueError("bad mode") + raise ValueError("bad mode %r" % mode) if isPath(fp): filename = fp @@ -2048,7 +2125,8 @@ def open(fp, mode="r"): #traceback.print_exc() pass - raise IOError("cannot identify image file") + raise IOError("cannot identify image file %r" + % (filename if filename else fp)) # # Image processing. diff --git a/PIL/ImageCms.py b/PIL/ImageCms.py index 20ba6a11f..c875712c1 100644 --- a/PIL/ImageCms.py +++ b/PIL/ImageCms.py @@ -84,7 +84,13 @@ VERSION = "1.0.0 pil" # --------------------------------------------------------------------. from PIL import Image -from PIL import _imagingcms +try: + from PIL import _imagingcms +except ImportError as ex: + # Allow error import for doc purposes, but error out when accessing + # anything in core. + from _util import import_err + _imagingcms = import_err(ex) from PIL._util import isStringType core = _imagingcms @@ -159,11 +165,10 @@ class ImageCmsProfile: self.product_name = None self.product_info = None -## -# Transform. This can be used with the procedural API, or with the -# standard {@link Image.point} method. - class ImageCmsTransform(Image.ImagePointHandler): + """Transform. This can be used with the procedural API, or with the + standard Image.point() method. + """ def __init__(self, input, output, input_mode, output_mode, intent=INTENT_PERCEPTUAL, @@ -203,11 +208,11 @@ class ImageCmsTransform(Image.ImagePointHandler): result = self.transform.apply(im.im.id, im.im.id) return im -## -# (experimental) Fetches the profile for the current display device. -# @return None if the profile is not known. - def get_display_profile(handle=None): + """ (experimental) Fetches the profile for the current display device. + :returns: None if the profile is not known. + """ + import sys if sys.platform == "win32": from PIL import ImageWin @@ -228,59 +233,58 @@ def get_display_profile(handle=None): # pyCMS compatible layer # --------------------------------------------------------------------. -## -# (pyCMS) Exception class. This is used for all errors in the pyCMS API. - class PyCMSError(Exception): + """ (pyCMS) Exception class. This is used for all errors in the pyCMS API. """ pass -## -# (pyCMS) Applies an ICC transformation to a given image, mapping from -# inputProfile to outputProfile. -# -# If the input or output profiles specified are not valid filenames, a -# PyCMSError will be raised. If inPlace == TRUE and outputMode != im.mode, -# a PyCMSError will be raised. If an error occurs during application of -# the profiles, a PyCMSError will be raised. If outputMode is not a mode -# supported by the outputProfile (or by pyCMS), a PyCMSError will be -# raised. -# -# This function applies an ICC transformation to im from inputProfile's -# color space to outputProfile's color space using the specified rendering -# intent to decide how to handle out-of-gamut colors. -# -# OutputMode can be used to specify that a color mode conversion is to -# be done using these profiles, but the specified profiles must be able -# to handle that mode. I.e., if converting im from RGB to CMYK using -# profiles, the input profile must handle RGB data, and the output -# profile must handle CMYK data. -# -# @param im An open PIL image object (i.e. Image.new(...) or Image.open(...), etc.) -# @param inputProfile String, as a valid filename path to the ICC input profile -# you wish to use for this image, or a profile object -# @param outputProfile String, as a valid filename path to the ICC output -# profile you wish to use for this image, or a profile object -# @param renderingIntent Integer (0-3) specifying the rendering intent you wish -# to use for the transform -# -# INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) -# INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) -# INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) -# INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) -# -# see the pyCMS documentation for details on rendering intents and what they do. -# @param outputMode A valid PIL mode for the output image (i.e. "RGB", "CMYK", -# etc.). Note: if rendering the image "inPlace", outputMode MUST be the -# same mode as the input, or omitted completely. If omitted, the outputMode -# will be the same as the mode of the input image (im.mode) -# @param inPlace Boolean (1 = True, None or 0 = False). If True, the original -# image is modified in-place, and None is returned. If False (default), a -# new Image object is returned with the transform applied. -# @param flags Integer (0-...) specifying additional flags -# @return Either None or a new PIL image object, depending on value of inPlace -# @exception PyCMSError - def profileToProfile(im, inputProfile, outputProfile, renderingIntent=INTENT_PERCEPTUAL, outputMode=None, inPlace=0, flags=0): + """ + (pyCMS) Applies an ICC transformation to a given image, mapping from + inputProfile to outputProfile. + + If the input or output profiles specified are not valid filenames, a + PyCMSError will be raised. If inPlace == TRUE and outputMode != im.mode, + a PyCMSError will be raised. If an error occurs during application of + the profiles, a PyCMSError will be raised. If outputMode is not a mode + supported by the outputProfile (or by pyCMS), a PyCMSError will be + raised. + + This function applies an ICC transformation to im from inputProfile's + color space to outputProfile's color space using the specified rendering + intent to decide how to handle out-of-gamut colors. + + OutputMode can be used to specify that a color mode conversion is to + be done using these profiles, but the specified profiles must be able + to handle that mode. I.e., if converting im from RGB to CMYK using + profiles, the input profile must handle RGB data, and the output + profile must handle CMYK data. + + :param im: An open PIL image object (i.e. Image.new(...) or Image.open(...), etc.) + :param inputProfile: String, as a valid filename path to the ICC input profile + you wish to use for this image, or a profile object + :param outputProfile: String, as a valid filename path to the ICC output + profile you wish to use for this image, or a profile object + :param renderingIntent: Integer (0-3) specifying the rendering intent you wish + to use for the transform + + INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) + INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) + INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) + INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) + + see the pyCMS documentation for details on rendering intents and what they do. + :param outputMode: A valid PIL mode for the output image (i.e. "RGB", "CMYK", + etc.). Note: if rendering the image "inPlace", outputMode MUST be the + same mode as the input, or omitted completely. If omitted, the outputMode + will be the same as the mode of the input image (im.mode) + :param inPlace: Boolean (1 = True, None or 0 = False). If True, the original + image is modified in-place, and None is returned. If False (default), a + new Image object is returned with the transform applied. + :param flags: Integer (0-...) specifying additional flags + :returns: Either None or a new PIL image object, depending on value of inPlace + :exception PyCMSError: + """ + if outputMode is None: outputMode = im.mode @@ -308,80 +312,83 @@ def profileToProfile(im, inputProfile, outputProfile, renderingIntent=INTENT_PER return imOut -## -# (pyCMS) Opens an ICC profile file. -# -# The PyCMSProfile object can be passed back into pyCMS for use in creating -# transforms and such (as in ImageCms.buildTransformFromOpenProfiles()). -# -# If profileFilename is not a vaild filename for an ICC profile, a PyCMSError -# will be raised. -# -# @param profileFilename String, as a valid filename path to the ICC profile you -# wish to open, or a file-like object. -# @return A CmsProfile class object. -# @exception PyCMSError def getOpenProfile(profileFilename): + """ + (pyCMS) Opens an ICC profile file. + + The PyCMSProfile object can be passed back into pyCMS for use in creating + transforms and such (as in ImageCms.buildTransformFromOpenProfiles()). + + If profileFilename is not a vaild filename for an ICC profile, a PyCMSError + will be raised. + + :param profileFilename: String, as a valid filename path to the ICC profile you + wish to open, or a file-like object. + :returns: A CmsProfile class object. + :exception PyCMSError: + """ + try: return ImageCmsProfile(profileFilename) except (IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Builds an ICC transform mapping from the inputProfile to the -# outputProfile. Use applyTransform to apply the transform to a given -# image. -# -# If the input or output profiles specified are not valid filenames, a -# PyCMSError will be raised. If an error occurs during creation of the -# transform, a PyCMSError will be raised. -# -# If inMode or outMode are not a mode supported by the outputProfile (or -# by pyCMS), a PyCMSError will be raised. -# -# This function builds and returns an ICC transform from the inputProfile -# to the outputProfile using the renderingIntent to determine what to do -# with out-of-gamut colors. It will ONLY work for converting images that -# are in inMode to images that are in outMode color format (PIL mode, -# i.e. "RGB", "RGBA", "CMYK", etc.). -# -# Building the transform is a fair part of the overhead in -# ImageCms.profileToProfile(), so if you're planning on converting multiple -# images using the same input/output settings, this can save you time. -# Once you have a transform object, it can be used with -# ImageCms.applyProfile() to convert images without the need to re-compute -# the lookup table for the transform. -# -# The reason pyCMS returns a class object rather than a handle directly -# to the transform is that it needs to keep track of the PIL input/output -# modes that the transform is meant for. These attributes are stored in -# the "inMode" and "outMode" attributes of the object (which can be -# manually overridden if you really want to, but I don't know of any -# time that would be of use, or would even work). -# -# @param inputProfile String, as a valid filename path to the ICC input profile -# you wish to use for this transform, or a profile object -# @param outputProfile String, as a valid filename path to the ICC output -# profile you wish to use for this transform, or a profile object -# @param inMode String, as a valid PIL mode that the appropriate profile also -# supports (i.e. "RGB", "RGBA", "CMYK", etc.) -# @param outMode String, as a valid PIL mode that the appropriate profile also -# supports (i.e. "RGB", "RGBA", "CMYK", etc.) -# @param renderingIntent Integer (0-3) specifying the rendering intent you -# wish to use for the transform -# -# INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) -# INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) -# INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) -# INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) -# -# see the pyCMS documentation for details on rendering intents and what they do. -# @param flags Integer (0-...) specifying additional flags -# @return A CmsTransform class object. -# @exception PyCMSError - def buildTransform(inputProfile, outputProfile, inMode, outMode, renderingIntent=INTENT_PERCEPTUAL, flags=0): + """ + (pyCMS) Builds an ICC transform mapping from the inputProfile to the + outputProfile. Use applyTransform to apply the transform to a given + image. + + If the input or output profiles specified are not valid filenames, a + PyCMSError will be raised. If an error occurs during creation of the + transform, a PyCMSError will be raised. + + If inMode or outMode are not a mode supported by the outputProfile (or + by pyCMS), a PyCMSError will be raised. + + This function builds and returns an ICC transform from the inputProfile + to the outputProfile using the renderingIntent to determine what to do + with out-of-gamut colors. It will ONLY work for converting images that + are in inMode to images that are in outMode color format (PIL mode, + i.e. "RGB", "RGBA", "CMYK", etc.). + + Building the transform is a fair part of the overhead in + ImageCms.profileToProfile(), so if you're planning on converting multiple + images using the same input/output settings, this can save you time. + Once you have a transform object, it can be used with + ImageCms.applyProfile() to convert images without the need to re-compute + the lookup table for the transform. + + The reason pyCMS returns a class object rather than a handle directly + to the transform is that it needs to keep track of the PIL input/output + modes that the transform is meant for. These attributes are stored in + the "inMode" and "outMode" attributes of the object (which can be + manually overridden if you really want to, but I don't know of any + time that would be of use, or would even work). + + :param inputProfile: String, as a valid filename path to the ICC input profile + you wish to use for this transform, or a profile object + :param outputProfile: String, as a valid filename path to the ICC output + profile you wish to use for this transform, or a profile object + :param inMode: String, as a valid PIL mode that the appropriate profile also + supports (i.e. "RGB", "RGBA", "CMYK", etc.) + :param outMode: String, as a valid PIL mode that the appropriate profile also + supports (i.e. "RGB", "RGBA", "CMYK", etc.) + :param renderingIntent: Integer (0-3) specifying the rendering intent you + wish to use for the transform + + INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) + INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) + INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) + INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) + + see the pyCMS documentation for details on rendering intents and what they do. + :param flags: Integer (0-...) specifying additional flags + :returns: A CmsTransform class object. + :exception PyCMSError: + """ + if not isinstance(renderingIntent, int) or not (0 <= renderingIntent <=3): raise PyCMSError("renderingIntent must be an integer between 0 and 3") @@ -397,78 +404,79 @@ def buildTransform(inputProfile, outputProfile, inMode, outMode, renderingIntent except (IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Builds an ICC transform mapping from the inputProfile to the -# outputProfile, but tries to simulate the result that would be -# obtained on the proofProfile device. -# -# If the input, output, or proof profiles specified are not valid -# filenames, a PyCMSError will be raised. -# -# If an error occurs during creation of the transform, a PyCMSError will -# be raised. -# -# If inMode or outMode are not a mode supported by the outputProfile -# (or by pyCMS), a PyCMSError will be raised. -# -# This function builds and returns an ICC transform from the inputProfile -# to the outputProfile, but tries to simulate the result that would be -# obtained on the proofProfile device using renderingIntent and -# proofRenderingIntent to determine what to do with out-of-gamut -# colors. This is known as "soft-proofing". It will ONLY work for -# converting images that are in inMode to images that are in outMode -# color format (PIL mode, i.e. "RGB", "RGBA", "CMYK", etc.). -# -# Usage of the resulting transform object is exactly the same as with -# ImageCms.buildTransform(). -# -# Proof profiling is generally used when using an output device to get a -# good idea of what the final printed/displayed image would look like on -# the proofProfile device when it's quicker and easier to use the -# output device for judging color. Generally, this means that the -# output device is a monitor, or a dye-sub printer (etc.), and the simulated -# device is something more expensive, complicated, or time consuming -# (making it difficult to make a real print for color judgement purposes). -# -# Soft-proofing basically functions by adjusting the colors on the -# output device to match the colors of the device being simulated. However, -# when the simulated device has a much wider gamut than the output -# device, you may obtain marginal results. -# -# @param inputProfile String, as a valid filename path to the ICC input profile -# you wish to use for this transform, or a profile object -# @param outputProfile String, as a valid filename path to the ICC output -# (monitor, usually) profile you wish to use for this transform, or a -# profile object -# @param proofProfile String, as a valid filename path to the ICC proof profile -# you wish to use for this transform, or a profile object -# @param inMode String, as a valid PIL mode that the appropriate profile also -# supports (i.e. "RGB", "RGBA", "CMYK", etc.) -# @param outMode String, as a valid PIL mode that the appropriate profile also -# supports (i.e. "RGB", "RGBA", "CMYK", etc.) -# @param renderingIntent Integer (0-3) specifying the rendering intent you -# wish to use for the input->proof (simulated) transform -# -# INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) -# INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) -# INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) -# INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) -# -# see the pyCMS documentation for details on rendering intents and what they do. -# @param proofRenderingIntent Integer (0-3) specifying the rendering intent you -# wish to use for proof->output transform -# -# INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) -# INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) -# INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) -# INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) -# -# see the pyCMS documentation for details on rendering intents and what they do. -# @param flags Integer (0-...) specifying additional flags -# @return A CmsTransform class object. -# @exception PyCMSError - def buildProofTransform(inputProfile, outputProfile, proofProfile, inMode, outMode, renderingIntent=INTENT_PERCEPTUAL, proofRenderingIntent=INTENT_ABSOLUTE_COLORIMETRIC, flags=FLAGS["SOFTPROOFING"]): + """ + (pyCMS) Builds an ICC transform mapping from the inputProfile to the + outputProfile, but tries to simulate the result that would be + obtained on the proofProfile device. + + If the input, output, or proof profiles specified are not valid + filenames, a PyCMSError will be raised. + + If an error occurs during creation of the transform, a PyCMSError will + be raised. + + If inMode or outMode are not a mode supported by the outputProfile + (or by pyCMS), a PyCMSError will be raised. + + This function builds and returns an ICC transform from the inputProfile + to the outputProfile, but tries to simulate the result that would be + obtained on the proofProfile device using renderingIntent and + proofRenderingIntent to determine what to do with out-of-gamut + colors. This is known as "soft-proofing". It will ONLY work for + converting images that are in inMode to images that are in outMode + color format (PIL mode, i.e. "RGB", "RGBA", "CMYK", etc.). + + Usage of the resulting transform object is exactly the same as with + ImageCms.buildTransform(). + + Proof profiling is generally used when using an output device to get a + good idea of what the final printed/displayed image would look like on + the proofProfile device when it's quicker and easier to use the + output device for judging color. Generally, this means that the + output device is a monitor, or a dye-sub printer (etc.), and the simulated + device is something more expensive, complicated, or time consuming + (making it difficult to make a real print for color judgement purposes). + + Soft-proofing basically functions by adjusting the colors on the + output device to match the colors of the device being simulated. However, + when the simulated device has a much wider gamut than the output + device, you may obtain marginal results. + + :param inputProfile: String, as a valid filename path to the ICC input profile + you wish to use for this transform, or a profile object + :param outputProfile: String, as a valid filename path to the ICC output + (monitor, usually) profile you wish to use for this transform, or a + profile object + :param proofProfile: String, as a valid filename path to the ICC proof profile + you wish to use for this transform, or a profile object + :param inMode: String, as a valid PIL mode that the appropriate profile also + supports (i.e. "RGB", "RGBA", "CMYK", etc.) + :param outMode: String, as a valid PIL mode that the appropriate profile also + supports (i.e. "RGB", "RGBA", "CMYK", etc.) + :param renderingIntent: Integer (0-3) specifying the rendering intent you + wish to use for the input->proof (simulated) transform + + INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) + INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) + INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) + INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) + + see the pyCMS documentation for details on rendering intents and what they do. + :param proofRenderingIntent: Integer (0-3) specifying the rendering intent you + wish to use for proof->output transform + + INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) + INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) + INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) + INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) + + see the pyCMS documentation for details on rendering intents and what they do. + :param flags: Integer (0-...) specifying additional flags + :returns: A CmsTransform class object. + :exception PyCMSError: + """ + if not isinstance(renderingIntent, int) or not (0 <= renderingIntent <=3): raise PyCMSError("renderingIntent must be an integer between 0 and 3") @@ -489,43 +497,44 @@ def buildProofTransform(inputProfile, outputProfile, proofProfile, inMode, outMo buildTransformFromOpenProfiles = buildTransform buildProofTransformFromOpenProfiles = buildProofTransform -## -# (pyCMS) Applies a transform to a given image. -# -# If im.mode != transform.inMode, a PyCMSError is raised. -# -# If inPlace == TRUE and transform.inMode != transform.outMode, a -# PyCMSError is raised. -# -# If im.mode, transfer.inMode, or transfer.outMode is not supported by -# pyCMSdll or the profiles you used for the transform, a PyCMSError is -# raised. -# -# If an error occurs while the transform is being applied, a PyCMSError -# is raised. -# -# This function applies a pre-calculated transform (from -# ImageCms.buildTransform() or ImageCms.buildTransformFromOpenProfiles()) to an -# image. The transform can be used for multiple images, saving -# considerable calcuation time if doing the same conversion multiple times. -# -# If you want to modify im in-place instead of receiving a new image as -# the return value, set inPlace to TRUE. This can only be done if -# transform.inMode and transform.outMode are the same, because we can't -# change the mode in-place (the buffer sizes for some modes are -# different). The default behavior is to return a new Image object of -# the same dimensions in mode transform.outMode. -# -# @param im A PIL Image object, and im.mode must be the same as the inMode -# supported by the transform. -# @param transform A valid CmsTransform class object -# @param inPlace Bool (1 == True, 0 or None == False). If True, im is modified -# in place and None is returned, if False, a new Image object with the -# transform applied is returned (and im is not changed). The default is False. -# @return Either None, or a new PIL Image object, depending on the value of inPlace -# @exception PyCMSError - def applyTransform(im, transform, inPlace=0): + """ + (pyCMS) Applies a transform to a given image. + + If im.mode != transform.inMode, a PyCMSError is raised. + + If inPlace == TRUE and transform.inMode != transform.outMode, a + PyCMSError is raised. + + If im.mode, transfer.inMode, or transfer.outMode is not supported by + pyCMSdll or the profiles you used for the transform, a PyCMSError is + raised. + + If an error occurs while the transform is being applied, a PyCMSError + is raised. + + This function applies a pre-calculated transform (from + ImageCms.buildTransform() or ImageCms.buildTransformFromOpenProfiles()) to an + image. The transform can be used for multiple images, saving + considerable calcuation time if doing the same conversion multiple times. + + If you want to modify im in-place instead of receiving a new image as + the return value, set inPlace to TRUE. This can only be done if + transform.inMode and transform.outMode are the same, because we can't + change the mode in-place (the buffer sizes for some modes are + different). The default behavior is to return a new Image object of + the same dimensions in mode transform.outMode. + + :param im: A PIL Image object, and im.mode must be the same as the inMode + supported by the transform. + :param transform: A valid CmsTransform class object + :param inPlace: Bool (1 == True, 0 or None == False). If True, im is modified + in place and None is returned, if False, a new Image object with the + transform applied is returned (and im is not changed). The default is False. + :returns: Either None, or a new PIL Image object, depending on the value of inPlace + :exception PyCMSError: + """ + try: if inPlace: transform.apply_in_place(im) @@ -537,31 +546,32 @@ def applyTransform(im, transform, inPlace=0): return imOut -## -# (pyCMS) Creates a profile. -# -# If colorSpace not in ["LAB", "XYZ", "sRGB"], a PyCMSError is raised -# -# If using LAB and colorTemp != a positive integer, a PyCMSError is raised. -# -# If an error occurs while creating the profile, a PyCMSError is raised. -# -# Use this function to create common profiles on-the-fly instead of -# having to supply a profile on disk and knowing the path to it. It -# returns a normal CmsProfile object that can be passed to -# ImageCms.buildTransformFromOpenProfiles() to create a transform to apply -# to images. -# -# @param colorSpace String, the color space of the profile you wish to create. -# Currently only "LAB", "XYZ", and "sRGB" are supported. -# @param colorTemp Positive integer for the white point for the profile, in -# degrees Kelvin (i.e. 5000, 6500, 9600, etc.). The default is for D50 -# illuminant if omitted (5000k). colorTemp is ONLY applied to LAB profiles, -# and is ignored for XYZ and sRGB. -# @return A CmsProfile class object -# @exception PyCMSError - def createProfile(colorSpace, colorTemp=-1): + """ + (pyCMS) Creates a profile. + + If colorSpace not in ["LAB", "XYZ", "sRGB"], a PyCMSError is raised + + If using LAB and colorTemp != a positive integer, a PyCMSError is raised. + + If an error occurs while creating the profile, a PyCMSError is raised. + + Use this function to create common profiles on-the-fly instead of + having to supply a profile on disk and knowing the path to it. It + returns a normal CmsProfile object that can be passed to + ImageCms.buildTransformFromOpenProfiles() to create a transform to apply + to images. + + :param colorSpace: String, the color space of the profile you wish to create. + Currently only "LAB", "XYZ", and "sRGB" are supported. + :param colorTemp: Positive integer for the white point for the profile, in + degrees Kelvin (i.e. 5000, 6500, 9600, etc.). The default is for D50 + illuminant if omitted (5000k). colorTemp is ONLY applied to LAB profiles, + and is ignored for XYZ and sRGB. + :returns: A CmsProfile class object + :exception PyCMSError: + """ + if colorSpace not in ["LAB", "XYZ", "sRGB"]: raise PyCMSError("Color space not supported for on-the-fly profile creation (%s)" % colorSpace) @@ -576,25 +586,27 @@ def createProfile(colorSpace, colorTemp=-1): except (TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Gets the internal product name for the given profile. -# -# If profile isn't a valid CmsProfile object or filename to a profile, -# a PyCMSError is raised If an error occurs while trying to obtain the -# name tag, a PyCMSError is raised. -# -# Use this function to obtain the INTERNAL name of the profile (stored -# in an ICC tag in the profile itself), usually the one used when the -# profile was originally created. Sometimes this tag also contains -# additional information supplied by the creator. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @return A string containing the internal name of the profile as stored in an -# ICC tag. -# @exception PyCMSError - def getProfileName(profile): + """ + + (pyCMS) Gets the internal product name for the given profile. + + If profile isn't a valid CmsProfile object or filename to a profile, + a PyCMSError is raised If an error occurs while trying to obtain the + name tag, a PyCMSError is raised. + + Use this function to obtain the INTERNAL name of the profile (stored + in an ICC tag in the profile itself), usually the one used when the + profile was originally created. Sometimes this tag also contains + additional information supplied by the creator. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :returns: A string containing the internal name of the profile as stored in an + ICC tag. + :exception PyCMSError: + """ + try: # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): @@ -615,26 +627,27 @@ def getProfileName(profile): except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Gets the internal product information for the given profile. -# -# If profile isn't a valid CmsProfile object or filename to a profile, -# a PyCMSError is raised. -# -# If an error occurs while trying to obtain the info tag, a PyCMSError -# is raised -# -# Use this function to obtain the information stored in the profile's -# info tag. This often contains details about the profile, and how it -# was created, as supplied by the creator. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @return A string containing the internal profile information stored in an ICC -# tag. -# @exception PyCMSError - def getProfileInfo(profile): + """ + (pyCMS) Gets the internal product information for the given profile. + + If profile isn't a valid CmsProfile object or filename to a profile, + a PyCMSError is raised. + + If an error occurs while trying to obtain the info tag, a PyCMSError + is raised + + Use this function to obtain the information stored in the profile's + info tag. This often contains details about the profile, and how it + was created, as supplied by the creator. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :returns: A string containing the internal profile information stored in an ICC + tag. + :exception PyCMSError: + """ + try: if not isinstance(profile, ImageCmsProfile): profile = ImageCmsProfile(profile) @@ -653,25 +666,25 @@ def getProfileInfo(profile): raise PyCMSError(v) -## -# (pyCMS) Gets the copyright for the given profile. -# -# If profile isn't a valid CmsProfile object or filename to a profile, -# a PyCMSError is raised. -# -# If an error occurs while trying to obtain the copyright tag, a PyCMSError -# is raised -# -# Use this function to obtain the information stored in the profile's -# copyright tag. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @return A string containing the internal profile information stored in an ICC -# tag. -# @exception PyCMSError - def getProfileCopyright(profile): + """ + (pyCMS) Gets the copyright for the given profile. + + If profile isn't a valid CmsProfile object or filename to a profile, + a PyCMSError is raised. + + If an error occurs while trying to obtain the copyright tag, a PyCMSError + is raised + + Use this function to obtain the information stored in the profile's + copyright tag. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :returns: A string containing the internal profile information stored in an ICC + tag. + :exception PyCMSError: + """ try: # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): @@ -680,25 +693,25 @@ def getProfileCopyright(profile): except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Gets the manufacturer for the given profile. -# -# If profile isn't a valid CmsProfile object or filename to a profile, -# a PyCMSError is raised. -# -# If an error occurs while trying to obtain the manufacturer tag, a PyCMSError -# is raised -# -# Use this function to obtain the information stored in the profile's -# manufacturer tag. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @return A string containing the internal profile information stored in an ICC -# tag. -# @exception PyCMSError - def getProfileManufacturer(profile): + """ + (pyCMS) Gets the manufacturer for the given profile. + + If profile isn't a valid CmsProfile object or filename to a profile, + a PyCMSError is raised. + + If an error occurs while trying to obtain the manufacturer tag, a PyCMSError + is raised + + Use this function to obtain the information stored in the profile's + manufacturer tag. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :returns: A string containing the internal profile information stored in an ICC + tag. + :exception PyCMSError: + """ try: # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): @@ -707,25 +720,26 @@ def getProfileManufacturer(profile): except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Gets the model for the given profile. -# -# If profile isn't a valid CmsProfile object or filename to a profile, -# a PyCMSError is raised. -# -# If an error occurs while trying to obtain the model tag, a PyCMSError -# is raised -# -# Use this function to obtain the information stored in the profile's -# model tag. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @return A string containing the internal profile information stored in an ICC -# tag. -# @exception PyCMSError - def getProfileModel(profile): + """ + (pyCMS) Gets the model for the given profile. + + If profile isn't a valid CmsProfile object or filename to a profile, + a PyCMSError is raised. + + If an error occurs while trying to obtain the model tag, a PyCMSError + is raised + + Use this function to obtain the information stored in the profile's + model tag. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :returns: A string containing the internal profile information stored in an ICC + tag. + :exception PyCMSError: + """ + try: # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): @@ -734,25 +748,26 @@ def getProfileModel(profile): except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Gets the description for the given profile. -# -# If profile isn't a valid CmsProfile object or filename to a profile, -# a PyCMSError is raised. -# -# If an error occurs while trying to obtain the description tag, a PyCMSError -# is raised -# -# Use this function to obtain the information stored in the profile's -# description tag. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @return A string containing the internal profile information stored in an ICC -# tag. -# @exception PyCMSError - def getProfileDescription(profile): + """ + (pyCMS) Gets the description for the given profile. + + If profile isn't a valid CmsProfile object or filename to a profile, + a PyCMSError is raised. + + If an error occurs while trying to obtain the description tag, a PyCMSError + is raised + + Use this function to obtain the information stored in the profile's + description tag. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :returns: A string containing the internal profile information stored in an ICC + tag. + :exception PyCMSError: + """ + try: # add an extra newline to preserve pyCMS compatibility if not isinstance(profile, ImageCmsProfile): @@ -762,35 +777,35 @@ def getProfileDescription(profile): raise PyCMSError(v) - -## -# (pyCMS) Gets the default intent name for the given profile. -# -# If profile isn't a valid CmsProfile object or filename to a profile, -# a PyCMSError is raised. -# -# If an error occurs while trying to obtain the default intent, a -# PyCMSError is raised. -# -# Use this function to determine the default (and usually best optomized) -# rendering intent for this profile. Most profiles support multiple -# rendering intents, but are intended mostly for one type of conversion. -# If you wish to use a different intent than returned, use -# ImageCms.isIntentSupported() to verify it will work first. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @return Integer 0-3 specifying the default rendering intent for this profile. -# -# INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) -# INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) -# INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) -# INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) -# -# see the pyCMS documentation for details on rendering intents and what they do. -# @exception PyCMSError - def getDefaultIntent(profile): + """ + (pyCMS) Gets the default intent name for the given profile. + + If profile isn't a valid CmsProfile object or filename to a profile, + a PyCMSError is raised. + + If an error occurs while trying to obtain the default intent, a + PyCMSError is raised. + + Use this function to determine the default (and usually best optomized) + rendering intent for this profile. Most profiles support multiple + rendering intents, but are intended mostly for one type of conversion. + If you wish to use a different intent than returned, use + ImageCms.isIntentSupported() to verify it will work first. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :returns: Integer 0-3 specifying the default rendering intent for this profile. + + INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) + INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) + INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) + INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) + + see the pyCMS documentation for details on rendering intents and what they do. + :exception PyCMSError: + """ + try: if not isinstance(profile, ImageCmsProfile): profile = ImageCmsProfile(profile) @@ -798,42 +813,43 @@ def getDefaultIntent(profile): except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Checks if a given intent is supported. -# -# Use this function to verify that you can use your desired -# renderingIntent with profile, and that profile can be used for the -# input/output/proof profile as you desire. -# -# Some profiles are created specifically for one "direction", can cannot -# be used for others. Some profiles can only be used for certain -# rendering intents... so it's best to either verify this before trying -# to create a transform with them (using this function), or catch the -# potential PyCMSError that will occur if they don't support the modes -# you select. -# -# @param profile EITHER a valid CmsProfile object, OR a string of the filename -# of an ICC profile. -# @param intent Integer (0-3) specifying the rendering intent you wish to use -# with this profile -# -# INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) -# INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) -# INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) -# INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) -# -# see the pyCMS documentation for details on rendering intents and what they do. -# @param direction Integer specifing if the profile is to be used for input, -# output, or proof -# -# INPUT = 0 (or use ImageCms.DIRECTION_INPUT) -# OUTPUT = 1 (or use ImageCms.DIRECTION_OUTPUT) -# PROOF = 2 (or use ImageCms.DIRECTION_PROOF) -# -# @return 1 if the intent/direction are supported, -1 if they are not. -# @exception PyCMSError - def isIntentSupported(profile, intent, direction): + """ + (pyCMS) Checks if a given intent is supported. + + Use this function to verify that you can use your desired + renderingIntent with profile, and that profile can be used for the + input/output/proof profile as you desire. + + Some profiles are created specifically for one "direction", can cannot + be used for others. Some profiles can only be used for certain + rendering intents... so it's best to either verify this before trying + to create a transform with them (using this function), or catch the + potential PyCMSError that will occur if they don't support the modes + you select. + + :param profile: EITHER a valid CmsProfile object, OR a string of the filename + of an ICC profile. + :param intent: Integer (0-3) specifying the rendering intent you wish to use + with this profile + + INTENT_PERCEPTUAL = 0 (DEFAULT) (ImageCms.INTENT_PERCEPTUAL) + INTENT_RELATIVE_COLORIMETRIC = 1 (ImageCms.INTENT_RELATIVE_COLORIMETRIC) + INTENT_SATURATION = 2 (ImageCms.INTENT_SATURATION) + INTENT_ABSOLUTE_COLORIMETRIC = 3 (ImageCms.INTENT_ABSOLUTE_COLORIMETRIC) + + see the pyCMS documentation for details on rendering intents and what they do. + :param direction: Integer specifing if the profile is to be used for input, + output, or proof + + INPUT = 0 (or use ImageCms.DIRECTION_INPUT) + OUTPUT = 1 (or use ImageCms.DIRECTION_OUTPUT) + PROOF = 2 (or use ImageCms.DIRECTION_PROOF) + + :returns: 1 if the intent/direction are supported, -1 if they are not. + :exception PyCMSError: + """ + try: if not isinstance(profile, ImageCmsProfile): profile = ImageCmsProfile(profile) @@ -846,10 +862,11 @@ def isIntentSupported(profile, intent, direction): except (AttributeError, IOError, TypeError, ValueError) as v: raise PyCMSError(v) -## -# (pyCMS) Fetches versions. - def versions(): + """ + (pyCMS) Fetches versions. + """ + import sys return ( VERSION, core.littlecms_version, sys.version.split()[0], Image.VERSION diff --git a/PIL/ImageColor.py b/PIL/ImageColor.py index c14257151..98a241bb0 100644 --- a/PIL/ImageColor.py +++ b/PIL/ImageColor.py @@ -20,15 +20,6 @@ from PIL import Image import re - -## -# Convert color string to RGB tuple. -# -# @param color A CSS3-style colour string. -# @return An RGB-tuple. -# @exception ValueError If the color string could not be interpreted -# as an RGB value. - def getrgb(color): """ Convert a color string to an RGB tuple. If the string cannot be parsed, @@ -37,7 +28,7 @@ def getrgb(color): .. versionadded:: 1.1.4 :param color: A color string - :return: ``(red, green, blue)`` + :return: ``(red, green, blue[, alpha])`` """ try: rgb = colormap[color] @@ -114,20 +105,21 @@ def getcolor(color, mode): .. versionadded:: 1.1.4 :param color: A color string - :return: ``(red, green, blue)`` + :return: ``(graylevel [, alpha]) or (red, green, blue[, alpha])`` """ # same as getrgb, but converts the result to the given mode - color = getrgb(color) - if mode == "RGB": - return color - if mode == "RGBA": - if len(color) == 3: - color = (color + (255,)) - r, g, b, a = color - return r, g, b, a + color, alpha = getrgb(color), 255 + if len(color) == 4: + color, alpha = color[0:3], color[3] + if Image.getmodebase(mode) == "L": r, g, b = color - return (r*299 + g*587 + b*114)//1000 + color = (r*299 + g*587 + b*114)//1000 + if mode[-1] == 'A': + return (color, alpha) + else: + if mode[-1] == 'A': + return color + (alpha,) return color colormap = { diff --git a/PIL/ImageFile.py b/PIL/ImageFile.py index a63fe757e..501e16b00 100644 --- a/PIL/ImageFile.py +++ b/PIL/ImageFile.py @@ -205,7 +205,7 @@ class ImageFile(Image.Image): else: raise IndexError(ie) - if not s: # truncated jpeg + if not s and not d.handles_eof: # truncated jpeg self.tile = [] # JpegDecode needs to clean things up here either way diff --git a/PIL/ImagePalette.py b/PIL/ImagePalette.py index 61affdb19..d5b9d04eb 100644 --- a/PIL/ImagePalette.py +++ b/PIL/ImagePalette.py @@ -23,13 +23,14 @@ from PIL import Image, ImageColor class ImagePalette: "Color palette for palette mapped images" - def __init__(self, mode = "RGB", palette = None): + def __init__(self, mode = "RGB", palette = None, size = 0): self.mode = mode self.rawmode = None # if set, palette contains raw data self.palette = palette or list(range(256))*len(self.mode) self.colors = {} self.dirty = None - if len(self.mode)*256 != len(self.palette): + if ((size == 0 and len(self.mode)*256 != len(self.palette)) or + (size != 0 and size != len(self.palette))): raise ValueError("wrong palette size") def getdata(self): diff --git a/PIL/IptcImagePlugin.py b/PIL/IptcImagePlugin.py index 157b73509..104153002 100644 --- a/PIL/IptcImagePlugin.py +++ b/PIL/IptcImagePlugin.py @@ -172,8 +172,8 @@ class IptcImageFile(ImageFile.ImageFile): self.fp.seek(offset) # Copy image data to temporary file - outfile = tempfile.mktemp() - o = open(outfile, "wb") + o_fd, outfile = tempfile.mkstemp(text=False) + o = os.fdopen(o_fd) if encoding == "raw": # To simplify access to the extracted file, # prepend a PPM header diff --git a/PIL/Jpeg2KImagePlugin.py b/PIL/Jpeg2KImagePlugin.py new file mode 100644 index 000000000..f57f4a784 --- /dev/null +++ b/PIL/Jpeg2KImagePlugin.py @@ -0,0 +1,249 @@ +# +# The Python Imaging Library +# $Id$ +# +# JPEG2000 file handling +# +# History: +# 2014-03-12 ajh Created +# +# Copyright (c) 2014 Coriolis Systems Limited +# Copyright (c) 2014 Alastair Houghton +# +# See the README file for information on usage and redistribution. +# + +__version__ = "0.1" + +from PIL import Image, ImageFile, _binary +import struct +import os +import io + +def _parse_codestream(fp): + """Parse the JPEG 2000 codestream to extract the size and component + count from the SIZ marker segment, returning a PIL (size, mode) tuple.""" + + hdr = fp.read(2) + lsiz = struct.unpack('>H', hdr)[0] + siz = hdr + fp.read(lsiz - 2) + lsiz, rsiz, xsiz, ysiz, xosiz, yosiz, xtsiz, ytsiz, \ + xtosiz, ytosiz, csiz \ + = struct.unpack('>HHIIIIIIIIH', siz[:38]) + ssiz = [None]*csiz + xrsiz = [None]*csiz + yrsiz = [None]*csiz + for i in range(csiz): + ssiz[i], xrsiz[i], yrsiz[i] \ + = struct.unpack('>BBB', siz[36 + 3 * i:39 + 3 * i]) + + size = (xsiz - xosiz, ysiz - yosiz) + if csiz == 1: + mode = 'L' + elif csiz == 2: + mode = 'LA' + elif csiz == 3: + mode = 'RGB' + elif csiz == 4: + mode == 'RGBA' + else: + mode = None + + return (size, mode) + +def _parse_jp2_header(fp): + """Parse the JP2 header box to extract size, component count and + color space information, returning a PIL (size, mode) tuple.""" + + # Find the JP2 header box + header = None + while True: + lbox, tbox = struct.unpack('>I4s', fp.read(8)) + if lbox == 1: + lbox = struct.unpack('>Q', fp.read(8))[0] + hlen = 16 + else: + hlen = 8 + + if tbox == b'jp2h': + header = fp.read(lbox - hlen) + break + else: + fp.seek(lbox - hlen, os.SEEK_CUR) + + if header is None: + raise SyntaxError('could not find JP2 header') + + size = None + mode = None + + hio = io.BytesIO(header) + while True: + lbox, tbox = struct.unpack('>I4s', hio.read(8)) + if lbox == 1: + lbox = struct.unpack('>Q', hio.read(8))[0] + hlen = 16 + else: + hlen = 8 + + content = hio.read(lbox - hlen) + + if tbox == b'ihdr': + height, width, nc, bpc, c, unkc, ipr \ + = struct.unpack('>IIHBBBB', content) + size = (width, height) + if unkc: + if nc == 1: + mode = 'L' + elif nc == 2: + mode = 'LA' + elif nc == 3: + mode = 'RGB' + elif nc == 4: + mode = 'RGBA' + break + elif tbox == b'colr': + meth, prec, approx = struct.unpack('>BBB', content[:3]) + if meth == 1: + cs = struct.unpack('>I', content[3:7])[0] + if cs == 16: # sRGB + if nc == 3: + mode = 'RGB' + elif nc == 4: + mode = 'RGBA' + break + elif cs == 17: # grayscale + if nc == 1: + mode = 'L' + elif nc == 2: + mode = 'LA' + break + elif cs == 18: # sYCC + if nc == 3: + mode = 'RGB' + elif nc == 4: + mode == 'RGBA' + break + + return (size, mode) + +## +# Image plugin for JPEG2000 images. + +class Jpeg2KImageFile(ImageFile.ImageFile): + format = "JPEG2000" + format_description = "JPEG 2000 (ISO 15444)" + + def _open(self): + sig = self.fp.read(4) + if sig == b'\xff\x4f\xff\x51': + self.codec = "j2k" + self.size, self.mode = _parse_codestream(self.fp) + else: + sig = sig + self.fp.read(8) + + if sig == b'\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a': + self.codec = "jp2" + self.size, self.mode = _parse_jp2_header(self.fp) + else: + raise SyntaxError('not a JPEG 2000 file') + + if self.size is None or self.mode is None: + raise SyntaxError('unable to determine size/mode') + + self.reduce = 0 + self.layers = 0 + + fd = -1 + + if hasattr(self.fp, "fileno"): + try: + fd = self.fp.fileno() + except: + fd = -1 + + self.tile = [('jpeg2k', (0, 0) + self.size, 0, + (self.codec, self.reduce, self.layers, fd))] + + def load(self): + if self.reduce: + power = 1 << self.reduce + adjust = power >> 1 + self.size = (int((self.size[0] + adjust) / power), + int((self.size[1] + adjust) / power)) + + if self.tile: + # Update the reduce and layers settings + t = self.tile[0] + t3 = (t[3][0], self.reduce, self.layers, t[3][3]) + self.tile = [(t[0], (0, 0) + self.size, t[2], t3)] + + ImageFile.ImageFile.load(self) + +def _accept(prefix): + return (prefix[:4] == b'\xff\x4f\xff\x51' + or prefix[:12] == b'\x00\x00\x00\x0cjP \x0d\x0a\x87\x0a') + +# ------------------------------------------------------------ +# Save support + +def _save(im, fp, filename): + if filename.endswith('.j2k'): + kind = 'j2k' + else: + kind = 'jp2' + + # Get the keyword arguments + info = im.encoderinfo + + offset = info.get('offset', None) + tile_offset = info.get('tile_offset', None) + tile_size = info.get('tile_size', None) + quality_mode = info.get('quality_mode', 'rates') + quality_layers = info.get('quality_layers', None) + num_resolutions = info.get('num_resolutions', 0) + cblk_size = info.get('codeblock_size', None) + precinct_size = info.get('precinct_size', None) + irreversible = info.get('irreversible', False) + progression = info.get('progression', 'LRCP') + cinema_mode = info.get('cinema_mode', 'no') + fd = -1 + + if hasattr(fp, "fileno"): + try: + fd = fp.fileno() + except: + fd = -1 + + im.encoderconfig = ( + offset, + tile_offset, + tile_size, + quality_mode, + quality_layers, + num_resolutions, + cblk_size, + precinct_size, + irreversible, + progression, + cinema_mode, + fd + ) + + ImageFile._save(im, fp, [('jpeg2k', (0, 0)+im.size, 0, kind)]) + +# ------------------------------------------------------------ +# Registry stuff + +Image.register_open('JPEG2000', Jpeg2KImageFile, _accept) +Image.register_save('JPEG2000', _save) + +Image.register_extension('JPEG2000', '.jp2') +Image.register_extension('JPEG2000', '.j2k') +Image.register_extension('JPEG2000', '.jpc') +Image.register_extension('JPEG2000', '.jpf') +Image.register_extension('JPEG2000', '.jpx') +Image.register_extension('JPEG2000', '.j2c') + +Image.register_mime('JPEG2000', 'image/jp2') +Image.register_mime('JPEG2000', 'image/jpx') diff --git a/PIL/JpegImagePlugin.py b/PIL/JpegImagePlugin.py index 9563f9723..da52006ca 100644 --- a/PIL/JpegImagePlugin.py +++ b/PIL/JpegImagePlugin.py @@ -344,13 +344,17 @@ class JpegImageFile(ImageFile.ImageFile): # ALTERNATIVE: handle JPEGs via the IJG command line utilities import tempfile, os - file = tempfile.mktemp() - os.system("djpeg %s >%s" % (self.filename, file)) + f, path = tempfile.mkstemp() + os.close(f) + if os.path.exists(self.filename): + os.system("djpeg '%s' >'%s'" % (self.filename, path)) + else: + raise ValueError("Invalid Filename") try: - self.im = Image.core.open_ppm(file) + self.im = Image.core.open_ppm(path) finally: - try: os.unlink(file) + try: os.unlink(path) except: pass self.mode = self.im.mode @@ -438,7 +442,7 @@ samplings = { } def convert_dict_qtables(qtables): - qtables = [qtables[key] for key in xrange(len(qtables)) if qtables.has_key(key)] + qtables = [qtables[key] for key in range(len(qtables)) if key in qtables] for idx, table in enumerate(qtables): qtables[idx] = [table[i] for i in zigzag_index] return qtables @@ -500,7 +504,7 @@ def _save(im, fp, filename): except ValueError: raise ValueError("Invalid quantization table") else: - qtables = [lines[s:s+64] for s in xrange(0, len(lines), 64)] + qtables = [lines[s:s+64] for s in range(0, len(lines), 64)] if isinstance(qtables, (tuple, list, dict)): if isinstance(qtables, dict): qtables = convert_dict_qtables(qtables) diff --git a/PIL/OleFileIO-README.md b/PIL/OleFileIO-README.md new file mode 100644 index 000000000..f02a548d6 --- /dev/null +++ b/PIL/OleFileIO-README.md @@ -0,0 +1,141 @@ +OleFileIO_PL +============ + +[OleFileIO_PL](http://www.decalage.info/python/olefileio) is a Python module to read [Microsoft OLE2 files (also called Structured Storage, Compound File Binary Format or Compound Document File Format)](http://en.wikipedia.org/wiki/Compound_File_Binary_Format), such as Microsoft Office documents, Image Composer and FlashPix files, Outlook messages, ... + +This is an improved version of the OleFileIO module from [PIL](http://www.pythonware.com/products/pil/index.htm), the excellent Python Imaging Library, created and maintained by Fredrik Lundh. The API is still compatible with PIL, but I have improved the internal implementation significantly, with new features, bugfixes and a more robust design. + +As far as I know, this module is now the most complete and robust Python implementation to read MS OLE2 files, portable on several operating systems. (please tell me if you know other similar Python modules) + +WARNING: THIS IS (STILL) WORK IN PROGRESS. + +Main improvements over PIL version of OleFileIO: +------------------------------------------------ + +- Better compatibility with Python 2.6 (also compatible with Python 3.0+) +- Support for files larger than 6.8MB +- Robust: many checks to detect malformed files +- Improved API +- New features: metadata extraction, stream/storage timestamps +- Added setup.py and install.bat to ease installation + +News +---- + +- 2013-07-24 v0.26: added methods to parse stream/storage timestamps, improved listdir to include storages, fixed parsing of direntry timestamps +- 2013-05-27 v0.25: improved metadata extraction, properties parsing and exception handling, fixed [issue #12](https://bitbucket.org/decalage/olefileio_pl/issue/12/error-when-converting-timestamps-in-ole) +- 2013-05-07 v0.24: new features to extract metadata (get\_metadata method and OleMetadata class), improved getproperties to convert timestamps to Python datetime +- 2012-10-09: published [python-oletools](http://www.decalage.info/python/oletools), a package of analysis tools based on OleFileIO_PL +- 2012-09-11 v0.23: added support for file-like objects, fixed [issue #8](https://bitbucket.org/decalage/olefileio_pl/issue/8/bug-with-file-object) +- 2012-02-17 v0.22: fixed issues #7 (bug in getproperties) and #2 (added close method) +- 2011-10-20: code hosted on bitbucket to ease contributions and bug tracking +- 2010-01-24 v0.21: fixed support for big-endian CPUs, such as PowerPC Macs. +- 2009-12-11 v0.20: small bugfix in OleFileIO.open when filename is not plain str. +- 2009-12-10 v0.19: fixed support for 64 bits platforms (thanks to Ben G. and Martijn for reporting the bug) +- see changelog in source code for more info. + +Download: +--------- + +The archive is available on [the project page](https://bitbucket.org/decalage/olefileio_pl/downloads). + + +How to use this module: +----------------------- + +See sample code at the end of the module, and also docstrings. + +Here are a few examples: + + :::python + import OleFileIO_PL + + # Test if a file is an OLE container: + assert OleFileIO_PL.isOleFile('myfile.doc') + + # Open an OLE file from disk: + ole = OleFileIO_PL.OleFileIO('myfile.doc') + + # Get list of streams: + print(ole.listdir()) + + # Test if known streams/storages exist: + if ole.exists('worddocument'): + print("This is a Word document.") + print("size :", ole.get_size('worddocument')) + if ole.exists('macros/vba'): + print("This document seems to contain VBA macros.") + + # Extract the "Pictures" stream from a PPT file: + if ole.exists('Pictures'): + pics = ole.openstream('Pictures') + data = pics.read() + f = open('Pictures.bin', 'wb') + f.write(data) + f.close() + + # Extract metadata (new in v0.24) - see source code for all attributes: + meta = ole.get_metadata() + print('Author:', meta.author) + print('Title:', meta.title) + print('Creation date:', meta.create_time) + # print all metadata: + meta.dump() + + # Close the OLE file: + ole.close() + + # Work with a file-like object (e.g. StringIO) instead of a file on disk: + data = open('myfile.doc', 'rb').read() + f = io.BytesIO(data) + ole = OleFileIO_PL.OleFileIO(f) + print(ole.listdir()) + ole.close() + + +It can also be used as a script from the command-line to display the structure of an OLE file, for example: + + OleFileIO_PL.py myfile.doc + +A real-life example: [using OleFileIO_PL for malware analysis and forensics](http://blog.gregback.net/2011/03/using-remnux-for-forensic-puzzle-6/). + +How to contribute: +------------------ + +The code is available in [a Mercurial repository on bitbucket](https://bitbucket.org/decalage/olefileio_pl). You may use it to submit enhancements or to report any issue. + +If you would like to help us improve this module, or simply provide feedback, you may also send an e-mail to decalage(at)laposte.net. You can help in many ways: + +- test this module on different platforms / Python versions +- find and report bugs +- improve documentation, code samples, docstrings +- write unittest test cases +- provide tricky malformed files + +How to report bugs: +------------------- + +To report a bug, for example a normal file which is not parsed correctly, please use the [issue reporting page](https://bitbucket.org/decalage/olefileio_pl/issues?status=new&status=open), or send an e-mail with an attachment containing the debugging output of OleFileIO_PL. + +For this, launch the following command : + + OleFileIO_PL.py -d -c file >debug.txt + +License +------- + +OleFileIO_PL is open-source. + +OleFileIO_PL changes are Copyright (c) 2005-2013 by Philippe Lagadec. + +The Python Imaging Library (PIL) is + +- Copyright (c) 1997-2005 by Secret Labs AB + +- Copyright (c) 1995-2005 by Fredrik Lundh + +By obtaining, using, and/or copying this software and/or its associated documentation, you agree that you have read, understood, and will comply with the following terms and conditions: + +Permission to use, copy, modify, and distribute this software and its associated documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appears in all copies, and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Secret Labs AB or the author not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/PIL/OleFileIO.py b/PIL/OleFileIO.py index 2367841e0..91074be6b 100644 --- a/PIL/OleFileIO.py +++ b/PIL/OleFileIO.py @@ -1,3 +1,185 @@ +#!/usr/local/bin/python +# -*- coding: latin-1 -*- +""" +OleFileIO_PL: + Module to read Microsoft OLE2 files (also called Structured Storage or + Microsoft Compound Document File Format), such as Microsoft Office + documents, Image Composer and FlashPix files, Outlook messages, ... + +version 0.26 2013-07-24 Philippe Lagadec - http://www.decalage.info + +Project website: http://www.decalage.info/python/olefileio + +Improved version of the OleFileIO module from PIL library v1.1.6 +See: http://www.pythonware.com/products/pil/index.htm + +The Python Imaging Library (PIL) is + Copyright (c) 1997-2005 by Secret Labs AB + Copyright (c) 1995-2005 by Fredrik Lundh +OleFileIO_PL changes are Copyright (c) 2005-2013 by Philippe Lagadec + +See source code and LICENSE.txt for information on usage and redistribution. + +WARNING: THIS IS (STILL) WORK IN PROGRESS. +""" + +from __future__ import print_function + +__author__ = "Philippe Lagadec, Fredrik Lundh (Secret Labs AB)" +__date__ = "2013-07-24" +__version__ = '0.26' + +#--- LICENSE ------------------------------------------------------------------ + +# OleFileIO_PL is an improved version of the OleFileIO module from the +# Python Imaging Library (PIL). + +# OleFileIO_PL changes are Copyright (c) 2005-2013 by Philippe Lagadec +# +# The Python Imaging Library (PIL) is +# Copyright (c) 1997-2005 by Secret Labs AB +# Copyright (c) 1995-2005 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its associated +# documentation, you agree that you have read, understood, and will comply with +# the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and its +# associated documentation for any purpose and without fee is hereby granted, +# provided that the above copyright notice appears in all copies, and that both +# that copyright notice and this permission notice appear in supporting +# documentation, and that the name of Secret Labs AB or the author(s) not be used +# in advertising or publicity pertaining to distribution of the software +# without specific, written prior permission. +# +# SECRET LABS AB AND THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +# SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +# IN NO EVENT SHALL SECRET LABS AB OR THE AUTHORS BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +# OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +#----------------------------------------------------------------------------- +# CHANGELOG: (only OleFileIO_PL changes compared to PIL 1.1.6) +# 2005-05-11 v0.10 PL: - a few fixes for Python 2.4 compatibility +# (all changes flagged with [PL]) +# 2006-02-22 v0.11 PL: - a few fixes for some Office 2003 documents which raise +# exceptions in _OleStream.__init__() +# 2006-06-09 v0.12 PL: - fixes for files above 6.8MB (DIFAT in loadfat) +# - added some constants +# - added header values checks +# - added some docstrings +# - getsect: bugfix in case sectors >512 bytes +# - getsect: added conformity checks +# - DEBUG_MODE constant to activate debug display +# 2007-09-04 v0.13 PL: - improved/translated (lots of) comments +# - updated license +# - converted tabs to 4 spaces +# 2007-11-19 v0.14 PL: - added OleFileIO._raise_defect() to adapt sensitivity +# - improved _unicode() to use Python 2.x unicode support +# - fixed bug in _OleDirectoryEntry +# 2007-11-25 v0.15 PL: - added safety checks to detect FAT loops +# - fixed _OleStream which didn't check stream size +# - added/improved many docstrings and comments +# - moved helper functions _unicode and _clsid out of +# OleFileIO class +# - improved OleFileIO._find() to add Unix path syntax +# - OleFileIO._find() is now case-insensitive +# - added get_type() and get_rootentry_name() +# - rewritten loaddirectory and _OleDirectoryEntry +# 2007-11-27 v0.16 PL: - added _OleDirectoryEntry.kids_dict +# - added detection of duplicate filenames in storages +# - added detection of duplicate references to streams +# - added get_size() and exists() to _OleDirectoryEntry +# - added isOleFile to check header before parsing +# - added __all__ list to control public keywords in pydoc +# 2007-12-04 v0.17 PL: - added _load_direntry to fix a bug in loaddirectory +# - improved _unicode(), added workarounds for Python <2.3 +# - added set_debug_mode and -d option to set debug mode +# - fixed bugs in OleFileIO.open and _OleDirectoryEntry +# - added safety check in main for large or binary +# properties +# - allow size>0 for storages for some implementations +# 2007-12-05 v0.18 PL: - fixed several bugs in handling of FAT, MiniFAT and +# streams +# - added option '-c' in main to check all streams +# 2009-12-10 v0.19 PL: - bugfix for 32 bit arrays on 64 bits platforms +# (thanks to Ben G. and Martijn for reporting the bug) +# 2009-12-11 v0.20 PL: - bugfix in OleFileIO.open when filename is not plain str +# 2010-01-22 v0.21 PL: - added support for big-endian CPUs such as PowerPC Macs +# 2012-02-16 v0.22 PL: - fixed bug in getproperties, patch by chuckleberryfinn +# (https://bitbucket.org/decalage/olefileio_pl/issue/7) +# - added close method to OleFileIO (fixed issue #2) +# 2012-07-25 v0.23 PL: - added support for file-like objects (patch by mete0r_kr) +# 2013-05-05 v0.24 PL: - getproperties: added conversion from filetime to python +# datetime +# - main: displays properties with date format +# - new class OleMetadata to parse standard properties +# - added get_metadata method +# 2013-05-07 v0.24 PL: - a few improvements in OleMetadata +# 2013-05-24 v0.25 PL: - getproperties: option to not convert some timestamps +# - OleMetaData: total_edit_time is now a number of seconds, +# not a timestamp +# - getproperties: added support for VT_BOOL, VT_INT, V_UINT +# - getproperties: filter out null chars from strings +# - getproperties: raise non-fatal defects instead of +# exceptions when properties cannot be parsed properly +# 2013-05-27 PL: - getproperties: improved exception handling +# - _raise_defect: added option to set exception type +# - all non-fatal issues are now recorded, and displayed +# when run as a script +# 2013-07-11 v0.26 PL: - added methods to get modification and creation times +# of a directory entry or a storage/stream +# - fixed parsing of direntry timestamps +# 2013-07-24 PL: - new options in listdir to list storages and/or streams + +#----------------------------------------------------------------------------- +# TODO (for version 1.0): +# + add path attrib to _OleDirEntry, set it once and for all in init or +# append_kids (then listdir/_list can be simplified) +# - TESTS with Linux, MacOSX, Python 1.5.2, various files, PIL, ... +# - add underscore to each private method, to avoid their display in +# pydoc/epydoc documentation - Remove it for classes to be documented +# - replace all raised exceptions with _raise_defect (at least in OleFileIO) +# - merge code from _OleStream and OleFileIO.getsect to read sectors +# (maybe add a class for FAT and MiniFAT ?) +# - add method to check all streams (follow sectors chains without storing all +# stream in memory, and report anomalies) +# - use _OleDirectoryEntry.kids_dict to improve _find and _list ? +# - fix Unicode names handling (find some way to stay compatible with Py1.5.2) +# => if possible avoid converting names to Latin-1 +# - review DIFAT code: fix handling of DIFSECT blocks in FAT (not stop) +# - rewrite OleFileIO.getproperties +# - improve docstrings to show more sample uses +# - see also original notes and FIXME below +# - remove all obsolete FIXMEs +# - OleMetadata: fix version attrib according to +# http://msdn.microsoft.com/en-us/library/dd945671%28v=office.12%29.aspx + +# IDEAS: +# - in OleFileIO._open and _OleStream, use size=None instead of 0x7FFFFFFF for +# streams with unknown size +# - use arrays of int instead of long integers for FAT/MiniFAT, to improve +# performance and reduce memory usage ? (possible issue with values >2^31) +# - provide tests with unittest (may need write support to create samples) +# - move all debug code (and maybe dump methods) to a separate module, with +# a class which inherits OleFileIO ? +# - fix docstrings to follow epydoc format +# - add support for 4K sectors ? +# - add support for big endian byte order ? +# - create a simple OLE explorer with wxPython + +# FUTURE EVOLUTIONS to add write support: +# 1) add ability to write a stream back on disk from BytesIO (same size, no +# change in FAT/MiniFAT). +# 2) rename a stream/storage if it doesn't change the RB tree +# 3) use rbtree module to update the red-black tree + any rename +# 4) remove a stream/storage: free sectors in FAT/MiniFAT +# 5) allocate new sectors in FAT/MiniFAT +# 6) create new storage/stream +#----------------------------------------------------------------------------- + # # THIS IS WORK IN PROGRESS # @@ -36,23 +218,92 @@ # See the README file for information on usage and redistribution. # -from __future__ import print_function +#------------------------------------------------------------------------------ import io import sys from PIL import _binary -from PIL._util import isPath +import struct, array, os.path, datetime + +#[PL] Define explicitly the public API to avoid private objects in pydoc: +__all__ = ['OleFileIO', 'isOleFile', 'MAGIC'] if str is not bytes: long = int -i8 = _binary.i8 -i16 = _binary.i16le -i32 = _binary.i32le +#[PL] workaround to fix an issue with array item size on 64 bits systems: +if array.array('L').itemsize == 4: + # on 32 bits platforms, long integers in an array are 32 bits: + UINT32 = 'L' +elif array.array('I').itemsize == 4: + # on 64 bits platforms, integers in an array are 32 bits: + UINT32 = 'I' +else: + raise ValueError('Need to fix a bug with 32 bit arrays, please contact author...') +#[PL] These workarounds were inspired from the Path module +# (see http://www.jorendorff.com/articles/python/path/) +#TODO: test with old Python versions + +# Pre-2.3 workaround for basestring. +try: + basestring +except NameError: + try: + # is Unicode supported (Python >2.0 or >1.6 ?) + basestring = (str, unicode) + except NameError: + basestring = str + +#[PL] Experimental setting: if True, OLE filenames will be kept in Unicode +# if False (default PIL behaviour), all filenames are converted to Latin-1. +KEEP_UNICODE_NAMES = False + +#[PL] DEBUG display mode: False by default, use set_debug_mode() or "-d" on +# command line to change it. +DEBUG_MODE = False +def debug_print(msg): + print(msg) +def debug_pass(msg): + pass +debug = debug_pass + +def set_debug_mode(debug_mode): + """ + Set debug mode on or off, to control display of debugging messages. + mode: True or False + """ + global DEBUG_MODE, debug + DEBUG_MODE = debug_mode + if debug_mode: + debug = debug_print + else: + debug = debug_pass + +#TODO: convert this to hex MAGIC = b'\320\317\021\340\241\261\032\341' +#[PL]: added constants for Sector IDs (from AAF specifications) +MAXREGSECT = 0xFFFFFFFA; # maximum SECT +DIFSECT = 0xFFFFFFFC; # (-4) denotes a DIFAT sector in a FAT +FATSECT = 0xFFFFFFFD; # (-3) denotes a FAT sector in a FAT +ENDOFCHAIN = 0xFFFFFFFE; # (-2) end of a virtual stream chain +FREESECT = 0xFFFFFFFF; # (-1) unallocated sector + +#[PL]: added constants for Directory Entry IDs (from AAF specifications) +MAXREGSID = 0xFFFFFFFA; # maximum directory entry ID +NOSTREAM = 0xFFFFFFFF; # (-1) unallocated directory entry + +#[PL] object types in storage (from AAF specifications) +STGTY_EMPTY = 0 # empty directory entry (according to OpenOffice.org doc) +STGTY_STORAGE = 1 # element is a storage object +STGTY_STREAM = 2 # element is a stream object +STGTY_LOCKBYTES = 3 # element is an ILockBytes object +STGTY_PROPERTY = 4 # element is an IPropertyStorage object +STGTY_ROOT = 5 # element is a root storage + + # # -------------------------------------------------------------------- # property types @@ -70,173 +321,652 @@ VT_VECTOR=0x1000; # map property id to name (for debugging purposes) VT = {} -for k, v in list(vars().items()): - if k[:3] == "VT_": - VT[v] = k +for keyword, var in list(vars().items()): + if keyword[:3] == "VT_": + VT[var] = keyword # # -------------------------------------------------------------------- # Some common document types (root.clsid fields) WORD_CLSID = "00020900-0000-0000-C000-000000000046" +#TODO: check Excel, PPT, ... + +#[PL]: Defect levels to classify parsing errors - see OleFileIO._raise_defect() +DEFECT_UNSURE = 10 # a case which looks weird, but not sure it's a defect +DEFECT_POTENTIAL = 20 # a potential defect +DEFECT_INCORRECT = 30 # an error according to specifications, but parsing + # can go on +DEFECT_FATAL = 40 # an error which cannot be ignored, parsing is + # impossible + +#[PL] add useful constants to __all__: +for key in list(vars().keys()): + if key.startswith('STGTY_') or key.startswith('DEFECT_'): + __all__.append(key) -# -# -------------------------------------------------------------------- +#--- FUNCTIONS ---------------------------------------------------------------- + +def isOleFile (filename): + """ + Test if file is an OLE container (according to its header). + filename: file name or path (str, unicode) + return: True if OLE, False otherwise. + """ + f = open(filename, 'rb') + header = f.read(len(MAGIC)) + if header == MAGIC: + return True + else: + return False + + +i8 = _binary.i8 +i16 = _binary.i16le +i32 = _binary.i32le + + +def _clsid(clsid): + """ + Converts a CLSID to a human-readable string. + clsid: string of length 16. + """ + assert len(clsid) == 16 + if clsid == bytearray(16): + return "" + return (("%08X-%04X-%04X-%02X%02X-" + "%02X" * 6) % + ((i32(clsid, 0), i16(clsid, 4), i16(clsid, 6)) + + tuple(map(i8, clsid[8:16])))) + + + +# UNICODE support: +# (necessary to handle storages/streams names which use Unicode) + +def _unicode(s, errors='replace'): + """ + Map unicode string to Latin 1. (Python with Unicode support) + + s: UTF-16LE unicode string to convert to Latin-1 + errors: 'replace', 'ignore' or 'strict'. + """ + #TODO: test if it OleFileIO works with Unicode strings, instead of + # converting to Latin-1. + try: + # First the string is converted to plain Unicode: + # (assuming it is encoded as UTF-16 little-endian) + u = s.decode('UTF-16LE', errors) + if bytes is not str or KEEP_UNICODE_NAMES: + return u + else: + # Second the unicode string is converted to Latin-1 + return u.encode('latin_1', errors) + except: + # there was an error during Unicode to Latin-1 conversion: + raise IOError('incorrect Unicode name') + + +def filetime2datetime(filetime): + """ + convert FILETIME (64 bits int) to Python datetime.datetime + """ + # TODO: manage exception when microseconds is too large + # inspired from http://code.activestate.com/recipes/511425-filetime-to-datetime/ + _FILETIME_null_date = datetime.datetime(1601, 1, 1, 0, 0, 0) + #debug('timedelta days=%d' % (filetime//(10*1000000*3600*24))) + return _FILETIME_null_date + datetime.timedelta(microseconds=filetime//10) + + + +#=== CLASSES ================================================================== + +class OleMetadata: + """ + class to parse and store metadata from standard properties of OLE files. + + Available attributes: + codepage, title, subject, author, keywords, comments, template, + last_saved_by, revision_number, total_edit_time, last_printed, create_time, + last_saved_time, num_pages, num_words, num_chars, thumbnail, + creating_application, security, codepage_doc, category, presentation_target, + bytes, lines, paragraphs, slides, notes, hidden_slides, mm_clips, + scale_crop, heading_pairs, titles_of_parts, manager, company, links_dirty, + chars_with_spaces, unused, shared_doc, link_base, hlinks, hlinks_changed, + version, dig_sig, content_type, content_status, language, doc_version + + Note: an attribute is set to None when not present in the properties of the + OLE file. + + References for SummaryInformation stream: + - http://msdn.microsoft.com/en-us/library/dd942545.aspx + - http://msdn.microsoft.com/en-us/library/dd925819%28v=office.12%29.aspx + - http://msdn.microsoft.com/en-us/library/windows/desktop/aa380376%28v=vs.85%29.aspx + - http://msdn.microsoft.com/en-us/library/aa372045.aspx + - http://sedna-soft.de/summary-information-stream/ + - http://poi.apache.org/apidocs/org/apache/poi/hpsf/SummaryInformation.html + + References for DocumentSummaryInformation stream: + - http://msdn.microsoft.com/en-us/library/dd945671%28v=office.12%29.aspx + - http://msdn.microsoft.com/en-us/library/windows/desktop/aa380374%28v=vs.85%29.aspx + - http://poi.apache.org/apidocs/org/apache/poi/hpsf/DocumentSummaryInformation.html + + new in version 0.25 + """ + + # attribute names for SummaryInformation stream properties: + # (ordered by property id, starting at 1) + SUMMARY_ATTRIBS = ['codepage', 'title', 'subject', 'author', 'keywords', 'comments', + 'template', 'last_saved_by', 'revision_number', 'total_edit_time', + 'last_printed', 'create_time', 'last_saved_time', 'num_pages', + 'num_words', 'num_chars', 'thumbnail', 'creating_application', + 'security'] + + # attribute names for DocumentSummaryInformation stream properties: + # (ordered by property id, starting at 1) + DOCSUM_ATTRIBS = ['codepage_doc', 'category', 'presentation_target', 'bytes', 'lines', 'paragraphs', + 'slides', 'notes', 'hidden_slides', 'mm_clips', + 'scale_crop', 'heading_pairs', 'titles_of_parts', 'manager', + 'company', 'links_dirty', 'chars_with_spaces', 'unused', 'shared_doc', + 'link_base', 'hlinks', 'hlinks_changed', 'version', 'dig_sig', + 'content_type', 'content_status', 'language', 'doc_version'] + + def __init__(self): + """ + Constructor for OleMetadata + All attributes are set to None by default + """ + # properties from SummaryInformation stream + self.codepage = None + self.title = None + self.subject = None + self.author = None + self.keywords = None + self.comments = None + self.template = None + self.last_saved_by = None + self.revision_number = None + self.total_edit_time = None + self.last_printed = None + self.create_time = None + self.last_saved_time = None + self.num_pages = None + self.num_words = None + self.num_chars = None + self.thumbnail = None + self.creating_application = None + self.security = None + # properties from DocumentSummaryInformation stream + self.codepage_doc = None + self.category = None + self.presentation_target = None + self.bytes = None + self.lines = None + self.paragraphs = None + self.slides = None + self.notes = None + self.hidden_slides = None + self.mm_clips = None + self.scale_crop = None + self.heading_pairs = None + self.titles_of_parts = None + self.manager = None + self.company = None + self.links_dirty = None + self.chars_with_spaces = None + self.unused = None + self.shared_doc = None + self.link_base = None + self.hlinks = None + self.hlinks_changed = None + self.version = None + self.dig_sig = None + self.content_type = None + self.content_status = None + self.language = None + self.doc_version = None + + + def parse_properties(self, olefile): + """ + Parse standard properties of an OLE file, from the streams + "\x05SummaryInformation" and "\x05DocumentSummaryInformation", + if present. + Properties are converted to strings, integers or python datetime objects. + If a property is not present, its value is set to None. + """ + # first set all attributes to None: + for attrib in (self.SUMMARY_ATTRIBS + self.DOCSUM_ATTRIBS): + setattr(self, attrib, None) + if olefile.exists("\x05SummaryInformation"): + # get properties from the stream: + # (converting timestamps to python datetime, except total_edit_time, + # which is property #10) + props = olefile.getproperties("\x05SummaryInformation", + convert_time=True, no_conversion=[10]) + # store them into this object's attributes: + for i in range(len(self.SUMMARY_ATTRIBS)): + # ids for standards properties start at 0x01, until 0x13 + value = props.get(i+1, None) + setattr(self, self.SUMMARY_ATTRIBS[i], value) + if olefile.exists("\x05DocumentSummaryInformation"): + # get properties from the stream: + props = olefile.getproperties("\x05DocumentSummaryInformation", + convert_time=True) + # store them into this object's attributes: + for i in range(len(self.DOCSUM_ATTRIBS)): + # ids for standards properties start at 0x01, until 0x13 + value = props.get(i+1, None) + setattr(self, self.DOCSUM_ATTRIBS[i], value) + + def dump(self): + """ + Dump all metadata, for debugging purposes. + """ + print('Properties from SummaryInformation stream:') + for prop in self.SUMMARY_ATTRIBS: + value = getattr(self, prop) + print('- %s: %s' % (prop, repr(value))) + print('Properties from DocumentSummaryInformation stream:') + for prop in self.DOCSUM_ATTRIBS: + value = getattr(self, prop) + print('- %s: %s' % (prop, repr(value))) + + +#--- _OleStream --------------------------------------------------------------- class _OleStream(io.BytesIO): - - """OLE2 Stream + """ + OLE2 Stream Returns a read-only file object which can be used to read - the contents of a OLE stream. To open a stream, use the - openstream method in the OleFile class. + the contents of a OLE stream (instance of the BytesIO class). + To open a stream, use the openstream method in the OleFile class. This function can be used with either ordinary streams, or ministreams, depending on the offset, sectorsize, and fat table arguments. + + Attributes: + - size: actual size of data stream, after it was opened. """ # FIXME: should store the list of sects obtained by following # the fat chain, and load new sectors on demand instead of # loading it all in one go. - def __init__(self, fp, sect, size, offset, sectorsize, fat): + def __init__(self, fp, sect, size, offset, sectorsize, fat, filesize): + """ + Constructor for _OleStream class. + fp : file object, the OLE container or the MiniFAT stream + sect : sector index of first sector in the stream + size : total size of the stream + offset : offset in bytes for the first FAT or MiniFAT sector + sectorsize: size of one sector + fat : array/list of sector indexes (FAT or MiniFAT) + filesize : size of OLE file (for debugging) + return : a BytesIO instance containing the OLE stream + """ + debug('_OleStream.__init__:') + debug(' sect=%d (%X), size=%d, offset=%d, sectorsize=%d, len(fat)=%d, fp=%s' + %(sect,sect,size,offset,sectorsize,len(fat), repr(fp))) + #[PL] To detect malformed documents with FAT loops, we compute the + # expected number of sectors in the stream: + unknown_size = False + if size==0x7FFFFFFF: + # this is the case when called from OleFileIO._open(), and stream + # size is not known in advance (for example when reading the + # Directory stream). Then we can only guess maximum size: + size = len(fat)*sectorsize + # and we keep a record that size was unknown: + unknown_size = True + debug(' stream with UNKNOWN SIZE') + nb_sectors = (size + (sectorsize-1)) // sectorsize + debug('nb_sectors = %d' % nb_sectors) + # This number should (at least) be less than the total number of + # sectors in the given FAT: + if nb_sectors > len(fat): + raise IOError('malformed OLE document, stream too large') + # optimization(?): data is first a list of strings, and join() is called + # at the end to concatenate all in one string. + # (this may not be really useful with recent Python versions) data = [] - - while sect != -2: # 0xFFFFFFFEL: - fp.seek(offset + sectorsize * sect) - data.append(fp.read(sectorsize)) - sect = fat[sect] - + # if size is zero, then first sector index should be ENDOFCHAIN: + if size == 0 and sect != ENDOFCHAIN: + debug('size == 0 and sect != ENDOFCHAIN:') + raise IOError('incorrect OLE sector index for empty stream') + #[PL] A fixed-length for loop is used instead of an undefined while + # loop to avoid DoS attacks: + for i in range(nb_sectors): + # Sector index may be ENDOFCHAIN, but only if size was unknown + if sect == ENDOFCHAIN: + if unknown_size: + break + else: + # else this means that the stream is smaller than declared: + debug('sect=ENDOFCHAIN before expected size') + raise IOError('incomplete OLE stream') + # sector index should be within FAT: + if sect<0 or sect>=len(fat): + debug('sect=%d (%X) / len(fat)=%d' % (sect, sect, len(fat))) + debug('i=%d / nb_sectors=%d' %(i, nb_sectors)) +## tmp_data = b"".join(data) +## f = open('test_debug.bin', 'wb') +## f.write(tmp_data) +## f.close() +## debug('data read so far: %d bytes' % len(tmp_data)) + raise IOError('incorrect OLE FAT, sector index out of range') + #TODO: merge this code with OleFileIO.getsect() ? + #TODO: check if this works with 4K sectors: + try: + fp.seek(offset + sectorsize * sect) + except: + debug('sect=%d, seek=%d, filesize=%d' % + (sect, offset+sectorsize*sect, filesize)) + raise IOError('OLE sector index out of range') + sector_data = fp.read(sectorsize) + # [PL] check if there was enough data: + # Note: if sector is the last of the file, sometimes it is not a + # complete sector (of 512 or 4K), so we may read less than + # sectorsize. + if len(sector_data)!=sectorsize and sect!=(len(fat)-1): + debug('sect=%d / len(fat)=%d, seek=%d / filesize=%d, len read=%d' % + (sect, len(fat), offset+sectorsize*sect, filesize, len(sector_data))) + debug('seek+len(read)=%d' % (offset+sectorsize*sect+len(sector_data))) + raise IOError('incomplete OLE sector') + data.append(sector_data) + # jump to next sector in the FAT: + try: + sect = fat[sect] + except IndexError: + # [PL] if pointer is out of the FAT an exception is raised + raise IOError('incorrect OLE FAT, sector index out of range') + #[PL] Last sector should be a "end of chain" marker: + if sect != ENDOFCHAIN: + raise IOError('incorrect last sector index in OLE stream') data = b"".join(data) + # Data is truncated to the actual stream size: + if len(data) >= size: + data = data[:size] + # actual stream size is stored for future use: + self.size = size + elif unknown_size: + # actual stream size was not known, now we know the size of read + # data: + self.size = len(data) + else: + # read data is less than expected: + debug('len(data)=%d, size=%d' % (len(data), size)) + raise IOError('OLE stream size is less than declared') + # when all data is read in memory, BytesIO constructor is called + io.BytesIO.__init__(self, data) + # Then the _OleStream object can be used as a read-only file object. - # print len(data), size - io.BytesIO.__init__(self, data[:size]) - -# -# -------------------------------------------------------------------- - -# FIXME: should add a counter in here to avoid looping forever -# if the tree is broken. +#--- _OleDirectoryEntry ------------------------------------------------------- class _OleDirectoryEntry: - """OLE2 Directory Entry - - Encapsulates a stream directory entry. Note that the - constructor builds a tree of all subentries, so we only - have to call it with the root object. """ + OLE2 Directory Entry + """ + #[PL] parsing code moved from OleFileIO.loaddirectory - def __init__(self, sidlist, sid): + # struct to parse directory entries: + # <: little-endian byte order, standard sizes + # (note: this should guarantee that Q returns a 64 bits int) + # 64s: string containing entry name in unicode (max 31 chars) + null char + # H: uint16, number of bytes used in name buffer, including null = (len+1)*2 + # B: uint8, dir entry type (between 0 and 5) + # B: uint8, color: 0=black, 1=red + # I: uint32, index of left child node in the red-black tree, NOSTREAM if none + # I: uint32, index of right child node in the red-black tree, NOSTREAM if none + # I: uint32, index of child root node if it is a storage, else NOSTREAM + # 16s: CLSID, unique identifier (only used if it is a storage) + # I: uint32, user flags + # Q (was 8s): uint64, creation timestamp or zero + # Q (was 8s): uint64, modification timestamp or zero + # I: uint32, SID of first sector if stream or ministream, SID of 1st sector + # of stream containing ministreams if root entry, 0 otherwise + # I: uint32, total stream size in bytes if stream (low 32 bits), 0 otherwise + # I: uint32, total stream size in bytes if stream (high 32 bits), 0 otherwise + STRUCT_DIRENTRY = '<64sHBBIII16sIQQIII' + # size of a directory entry: 128 bytes + DIRENTRY_SIZE = 128 + assert struct.calcsize(STRUCT_DIRENTRY) == DIRENTRY_SIZE - # store directory parameters. the caller provides - # a complete list of directory entries, as read from - # the directory stream. - name, type, sect, size, sids, clsid = sidlist[sid] - - self.sid = sid - self.name = name - self.type = type # 1=storage 2=stream - self.sect = sect - self.size = size - self.clsid = clsid - - # process child nodes, if any + def __init__(self, entry, sid, olefile): + """ + Constructor for an _OleDirectoryEntry object. + Parses a 128-bytes entry from the OLE Directory stream. + entry : string (must be 128 bytes long) + sid : index of this directory entry in the OLE file directory + olefile: OleFileIO containing this directory entry + """ + self.sid = sid + # ref to olefile is stored for future use + self.olefile = olefile + # kids is a list of children entries, if this entry is a storage: + # (list of _OleDirectoryEntry objects) self.kids = [] + # kids_dict is a dictionary of children entries, indexed by their + # name in lowercase: used to quickly find an entry, and to detect + # duplicates + self.kids_dict = {} + # flag used to detect if the entry is referenced more than once in + # directory: + self.used = False + # decode DirEntry + ( + name, + namelength, + self.entry_type, + self.color, + self.sid_left, + self.sid_right, + self.sid_child, + clsid, + self.dwUserFlags, + self.createTime, + self.modifyTime, + self.isectStart, + sizeLow, + sizeHigh + ) = struct.unpack(_OleDirectoryEntry.STRUCT_DIRENTRY, entry) + if self.entry_type not in [STGTY_ROOT, STGTY_STORAGE, STGTY_STREAM, STGTY_EMPTY]: + olefile._raise_defect(DEFECT_INCORRECT, 'unhandled OLE storage type') + # only first directory entry can (and should) be root: + if self.entry_type == STGTY_ROOT and sid != 0: + olefile._raise_defect(DEFECT_INCORRECT, 'duplicate OLE root entry') + if sid == 0 and self.entry_type != STGTY_ROOT: + olefile._raise_defect(DEFECT_INCORRECT, 'incorrect OLE root entry') + #debug (struct.unpack(fmt_entry, entry[:len_entry])) + # name should be at most 31 unicode characters + null character, + # so 64 bytes in total (31*2 + 2): + if namelength>64: + olefile._raise_defect(DEFECT_INCORRECT, 'incorrect DirEntry name length') + # if exception not raised, namelength is set to the maximum value: + namelength = 64 + # only characters without ending null char are kept: + name = name[:(namelength-2)] + # name is converted from unicode to Latin-1: + self.name = _unicode(name) - sid = sidlist[sid][4][2] + debug('DirEntry SID=%d: %s' % (self.sid, repr(self.name))) + debug(' - type: %d' % self.entry_type) + debug(' - sect: %d' % self.isectStart) + debug(' - SID left: %d, right: %d, child: %d' % (self.sid_left, + self.sid_right, self.sid_child)) - if sid != -1: + # sizeHigh is only used for 4K sectors, it should be zero for 512 bytes + # sectors, BUT apparently some implementations set it as 0xFFFFFFFF, 1 + # or some other value so it cannot be raised as a defect in general: + if olefile.sectorsize == 512: + if sizeHigh != 0 and sizeHigh != 0xFFFFFFFF: + debug('sectorsize=%d, sizeLow=%d, sizeHigh=%d (%X)' % + (olefile.sectorsize, sizeLow, sizeHigh, sizeHigh)) + olefile._raise_defect(DEFECT_UNSURE, 'incorrect OLE stream size') + self.size = sizeLow + else: + self.size = sizeLow + (long(sizeHigh)<<32) + debug(' - size: %d (sizeLow=%d, sizeHigh=%d)' % (self.size, sizeLow, sizeHigh)) - # the directory entries are organized as a red-black tree. - # the following piece of code does an ordered traversal of - # such a tree (at least that's what I hope ;-) + self.clsid = _clsid(clsid) + # a storage should have a null size, BUT some implementations such as + # Word 8 for Mac seem to allow non-null values => Potential defect: + if self.entry_type == STGTY_STORAGE and self.size != 0: + olefile._raise_defect(DEFECT_POTENTIAL, 'OLE storage with size>0') + # check if stream is not already referenced elsewhere: + if self.entry_type in (STGTY_ROOT, STGTY_STREAM) and self.size>0: + if self.size < olefile.minisectorcutoff \ + and self.entry_type==STGTY_STREAM: # only streams can be in MiniFAT + # ministream object + minifat = True + else: + minifat = False + olefile._check_duplicate_stream(self.isectStart, minifat) - stack = [self.sid] - # start at leftmost position - left, right, child = sidlist[sid][4] + def build_storage_tree(self): + """ + Read and build the red-black tree attached to this _OleDirectoryEntry + object, if it is a storage. + Note that this method builds a tree of all subentries, so it should + only be called for the root object once. + """ + debug('build_storage_tree: SID=%d - %s - sid_child=%d' + % (self.sid, repr(self.name), self.sid_child)) + if self.sid_child != NOSTREAM: + # if child SID is not NOSTREAM, then this entry is a storage. + # Let's walk through the tree of children to fill the kids list: + self.append_kids(self.sid_child) - while left != -1: # 0xFFFFFFFFL: - stack.append(sid) - sid = left - left, right, child = sidlist[sid][4] - - while sid != self.sid: - - self.kids.append(_OleDirectoryEntry(sidlist, sid)) - - # try to move right - left, right, child = sidlist[sid][4] - if right != -1: # 0xFFFFFFFFL: - # and then back to the left - sid = right - while True: - left, right, child = sidlist[sid][4] - if left == -1: # 0xFFFFFFFFL: - break - stack.append(sid) - sid = left - else: - # couldn't move right; move up instead - while True: - ptr = stack[-1] - del stack[-1] - left, right, child = sidlist[ptr][4] - if right != sid: - break - sid = right - left, right, child = sidlist[sid][4] - if right != ptr: - sid = ptr + # Note from OpenOffice documentation: the safest way is to + # recreate the tree because some implementations may store broken + # red-black trees... # in the OLE file, entries are sorted on (length, name). - # for convenience, we sort them on name instead. - + # for convenience, we sort them on name instead: + # (see rich comparison methods in this class) self.kids.sort() - def __cmp__(self, other): - "Compare entries by name" - return cmp(self.name, other.name) + def append_kids(self, child_sid): + """ + Walk through red-black tree of children of this directory entry to add + all of them to the kids list. (recursive method) + + child_sid : index of child directory entry to use, or None when called + first time for the root. (only used during recursion) + """ + #[PL] this method was added to use simple recursion instead of a complex + # algorithm. + # if this is not a storage or a leaf of the tree, nothing to do: + if child_sid == NOSTREAM: + return + # check if child SID is in the proper range: + if child_sid<0 or child_sid>=len(self.olefile.direntries): + self.olefile._raise_defect(DEFECT_FATAL, 'OLE DirEntry index out of range') + # get child direntry: + child = self.olefile._load_direntry(child_sid) #direntries[child_sid] + debug('append_kids: child_sid=%d - %s - sid_left=%d, sid_right=%d, sid_child=%d' + % (child.sid, repr(child.name), child.sid_left, child.sid_right, child.sid_child)) + # the directory entries are organized as a red-black tree. + # (cf. Wikipedia for details) + # First walk through left side of the tree: + self.append_kids(child.sid_left) + # Check if its name is not already used (case-insensitive): + name_lower = child.name.lower() + if name_lower in self.kids_dict: + self.olefile._raise_defect(DEFECT_INCORRECT, + "Duplicate filename in OLE storage") + # Then the child_sid _OleDirectoryEntry object is appended to the + # kids list and dictionary: + self.kids.append(child) + self.kids_dict[name_lower] = child + # Check if kid was not already referenced in a storage: + if child.used: + self.olefile._raise_defect(DEFECT_INCORRECT, + 'OLE Entry referenced more than once') + child.used = True + # Finally walk through right side of the tree: + self.append_kids(child.sid_right) + # Afterwards build kid's own tree if it's also a storage: + child.build_storage_tree() + + + def __eq__(self, other): + "Compare entries by name" + return self.name == other.name + def __lt__(self, other): + "Compare entries by name" + return self.name < other.name + #TODO: replace by the same function as MS implementation ? + # (order by name length first, then case-insensitive order) + + def __ne__(self, other): + return not self.__eq__(other) + def __le__(self, other): + return self.__eq__(other) or self.__lt__(other) + # Reflected __lt__() and __le__() will be used for __gt__() and __ge__() + def dump(self, tab = 0): "Dump this entry, and all its subentries (for debug purposes only)" - TYPES = ["(invalid)", "(storage)", "(stream)", "(lockbytes)", "(property)", "(root)"] - - print(" "*tab + repr(self.name), TYPES[self.type], end=' ') - if self.type in (2, 5): + print(" "*tab + repr(self.name), TYPES[self.entry_type], end=' ') + if self.entry_type in (STGTY_STREAM, STGTY_ROOT): print(self.size, "bytes", end=' ') print() - if self.type in (1, 5) and self.clsid: + if self.entry_type in (STGTY_STORAGE, STGTY_ROOT) and self.clsid: print(" "*tab + "{%s}" % self.clsid) for kid in self.kids: kid.dump(tab + 2) -# -# -------------------------------------------------------------------- -## -# This class encapsulates the interface to an OLE 2 structured -# storage file. Use the {@link listdir} and {@link openstream} -# methods to access the contents of this file. + def getmtime(self): + """ + Return modification time of a directory entry. + + return: None if modification time is null, a python datetime object + otherwise (UTC timezone) + + new in version 0.26 + """ + if self.modifyTime == 0: + return None + return filetime2datetime(self.modifyTime) + + + def getctime(self): + """ + Return creation time of a directory entry. + + return: None if modification time is null, a python datetime object + otherwise (UTC timezone) + + new in version 0.26 + """ + if self.createTime == 0: + return None + return filetime2datetime(self.createTime) + + +#--- OleFileIO ---------------------------------------------------------------- class OleFileIO: - """OLE container object + """ + OLE container object This class encapsulates the interface to an OLE 2 structured - storage file. Use the listdir and openstream methods to access - the contents of this file. + storage file. Use the {@link listdir} and {@link openstream} methods to + access the contents of this file. Object names are given as a list of strings, one for each subentry level. The root entry should be omitted. For example, the following @@ -259,255 +989,926 @@ class OleFileIO: TIFF files). """ - def __init__(self, filename = None): + def __init__(self, filename = None, raise_defects=DEFECT_FATAL): + """ + Constructor for OleFileIO class. + filename: file to open. + raise_defects: minimal level for defects to be raised as exceptions. + (use DEFECT_FATAL for a typical application, DEFECT_INCORRECT for a + security-oriented application, see source code for details) + """ + # minimal level for defects to be raised as exceptions: + self._raise_defects_level = raise_defects + # list of defects/issues not raised as exceptions: + # tuples of (exception type, message) + self.parsing_issues = [] if filename: self.open(filename) - ## - # Open an OLE2 file. + + def _raise_defect(self, defect_level, message, exception_type=IOError): + """ + This method should be called for any defect found during file parsing. + It may raise an IOError exception according to the minimal level chosen + for the OleFileIO object. + + defect_level: defect level, possible values are: + DEFECT_UNSURE : a case which looks weird, but not sure it's a defect + DEFECT_POTENTIAL : a potential defect + DEFECT_INCORRECT : an error according to specifications, but parsing can go on + DEFECT_FATAL : an error which cannot be ignored, parsing is impossible + message: string describing the defect, used with raised exception. + exception_type: exception class to be raised, IOError by default + """ + # added by [PL] + if defect_level >= self._raise_defects_level: + raise exception_type(message) + else: + # just record the issue, no exception raised: + self.parsing_issues.append((exception_type, message)) + def open(self, filename): - """Open an OLE2 file""" + """ + Open an OLE2 file. + Reads the header, FAT and directory. - if isPath(filename): - self.fp = open(filename, "rb") - else: + filename: string-like or file-like object + """ + #[PL] check if filename is a string-like or file-like object: + # (it is better to check for a read() method) + if hasattr(filename, 'read'): + # file-like object self.fp = filename + else: + # string-like object: filename of file on disk + #TODO: if larger than 1024 bytes, this could be the actual data => BytesIO + self.fp = open(filename, "rb") + # old code fails if filename is not a plain string: + #if isPath(filename): + # self.fp = open(filename, "rb") + #else: + # self.fp = filename + # obtain the filesize by using seek and tell, which should work on most + # file-like objects: + #TODO: do it above, using getsize with filename when possible? + #TODO: fix code to fail with clear exception when filesize cannot be obtained + self.fp.seek(0, os.SEEK_END) + try: + filesize = self.fp.tell() + finally: + self.fp.seek(0) + self._filesize = filesize + + # lists of streams in FAT and MiniFAT, to detect duplicate references + # (list of indexes of first sectors of each stream) + self._used_streams_fat = [] + self._used_streams_minifat = [] header = self.fp.read(512) if len(header) != 512 or header[:8] != MAGIC: - raise IOError("not an OLE2 structured storage file") + self._raise_defect(DEFECT_FATAL, "not an OLE2 structured storage file") + + # [PL] header structure according to AAF specifications: + ##Header + ##struct StructuredStorageHeader { // [offset from start (bytes), length (bytes)] + ##BYTE _abSig[8]; // [00H,08] {0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, + ## // 0x1a, 0xe1} for current version + ##CLSID _clsid; // [08H,16] reserved must be zero (WriteClassStg/ + ## // GetClassFile uses root directory class id) + ##USHORT _uMinorVersion; // [18H,02] minor version of the format: 33 is + ## // written by reference implementation + ##USHORT _uDllVersion; // [1AH,02] major version of the dll/format: 3 for + ## // 512-byte sectors, 4 for 4 KB sectors + ##USHORT _uByteOrder; // [1CH,02] 0xFFFE: indicates Intel byte-ordering + ##USHORT _uSectorShift; // [1EH,02] size of sectors in power-of-two; + ## // typically 9 indicating 512-byte sectors + ##USHORT _uMiniSectorShift; // [20H,02] size of mini-sectors in power-of-two; + ## // typically 6 indicating 64-byte mini-sectors + ##USHORT _usReserved; // [22H,02] reserved, must be zero + ##ULONG _ulReserved1; // [24H,04] reserved, must be zero + ##FSINDEX _csectDir; // [28H,04] must be zero for 512-byte sectors, + ## // number of SECTs in directory chain for 4 KB + ## // sectors + ##FSINDEX _csectFat; // [2CH,04] number of SECTs in the FAT chain + ##SECT _sectDirStart; // [30H,04] first SECT in the directory chain + ##DFSIGNATURE _signature; // [34H,04] signature used for transactions; must + ## // be zero. The reference implementation + ## // does not support transactions + ##ULONG _ulMiniSectorCutoff; // [38H,04] maximum size for a mini stream; + ## // typically 4096 bytes + ##SECT _sectMiniFatStart; // [3CH,04] first SECT in the MiniFAT chain + ##FSINDEX _csectMiniFat; // [40H,04] number of SECTs in the MiniFAT chain + ##SECT _sectDifStart; // [44H,04] first SECT in the DIFAT chain + ##FSINDEX _csectDif; // [48H,04] number of SECTs in the DIFAT chain + ##SECT _sectFat[109]; // [4CH,436] the SECTs of first 109 FAT sectors + ##}; + + # [PL] header decoding: + # '<' indicates little-endian byte ordering for Intel (cf. struct module help) + fmt_header = '<8s16sHHHHHHLLLLLLLLLL' + header_size = struct.calcsize(fmt_header) + debug( "fmt_header size = %d, +FAT = %d" % (header_size, header_size + 109*4) ) + header1 = header[:header_size] + ( + self.Sig, + self.clsid, + self.MinorVersion, + self.DllVersion, + self.ByteOrder, + self.SectorShift, + self.MiniSectorShift, + self.Reserved, self.Reserved1, + self.csectDir, + self.csectFat, + self.sectDirStart, + self.signature, + self.MiniSectorCutoff, + self.MiniFatStart, + self.csectMiniFat, + self.sectDifStart, + self.csectDif + ) = struct.unpack(fmt_header, header1) + debug( struct.unpack(fmt_header, header1)) + + if self.Sig != b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1': + # OLE signature should always be present + self._raise_defect(DEFECT_FATAL, "incorrect OLE signature") + if self.clsid != bytearray(16): + # according to AAF specs, CLSID should always be zero + self._raise_defect(DEFECT_INCORRECT, "incorrect CLSID in OLE header") + debug( "MinorVersion = %d" % self.MinorVersion ) + debug( "DllVersion = %d" % self.DllVersion ) + if self.DllVersion not in [3, 4]: + # version 3: usual format, 512 bytes per sector + # version 4: large format, 4K per sector + self._raise_defect(DEFECT_INCORRECT, "incorrect DllVersion in OLE header") + debug( "ByteOrder = %X" % self.ByteOrder ) + if self.ByteOrder != 0xFFFE: + # For now only common little-endian documents are handled correctly + self._raise_defect(DEFECT_FATAL, "incorrect ByteOrder in OLE header") + # TODO: add big-endian support for documents created on Mac ? + self.SectorSize = 2**self.SectorShift + debug( "SectorSize = %d" % self.SectorSize ) + if self.SectorSize not in [512, 4096]: + self._raise_defect(DEFECT_INCORRECT, "incorrect SectorSize in OLE header") + if (self.DllVersion==3 and self.SectorSize!=512) \ + or (self.DllVersion==4 and self.SectorSize!=4096): + self._raise_defect(DEFECT_INCORRECT, "SectorSize does not match DllVersion in OLE header") + self.MiniSectorSize = 2**self.MiniSectorShift + debug( "MiniSectorSize = %d" % self.MiniSectorSize ) + if self.MiniSectorSize not in [64]: + self._raise_defect(DEFECT_INCORRECT, "incorrect MiniSectorSize in OLE header") + if self.Reserved != 0 or self.Reserved1 != 0: + self._raise_defect(DEFECT_INCORRECT, "incorrect OLE header (non-null reserved bytes)") + debug( "csectDir = %d" % self.csectDir ) + if self.SectorSize==512 and self.csectDir!=0: + self._raise_defect(DEFECT_INCORRECT, "incorrect csectDir in OLE header") + debug( "csectFat = %d" % self.csectFat ) + debug( "sectDirStart = %X" % self.sectDirStart ) + debug( "signature = %d" % self.signature ) + # Signature should be zero, BUT some implementations do not follow this + # rule => only a potential defect: + if self.signature != 0: + self._raise_defect(DEFECT_POTENTIAL, "incorrect OLE header (signature>0)") + debug( "MiniSectorCutoff = %d" % self.MiniSectorCutoff ) + debug( "MiniFatStart = %X" % self.MiniFatStart ) + debug( "csectMiniFat = %d" % self.csectMiniFat ) + debug( "sectDifStart = %X" % self.sectDifStart ) + debug( "csectDif = %d" % self.csectDif ) + + # calculate the number of sectors in the file + # (-1 because header doesn't count) + self.nb_sect = ( (filesize + self.SectorSize-1) // self.SectorSize) - 1 + debug( "Number of sectors in the file: %d" % self.nb_sect ) # file clsid (probably never used, so we don't store it) - clsid = self._clsid(header[8:24]) + clsid = _clsid(header[8:24]) + self.sectorsize = self.SectorSize #1 << i16(header, 30) + self.minisectorsize = self.MiniSectorSize #1 << i16(header, 32) + self.minisectorcutoff = self.MiniSectorCutoff # i32(header, 56) - # FIXME: could check version and byte order fields - - self.sectorsize = 1 << i16(header, 30) - self.minisectorsize = 1 << i16(header, 32) - - self.minisectorcutoff = i32(header, 56) + # check known streams for duplicate references (these are always in FAT, + # never in MiniFAT): + self._check_duplicate_stream(self.sectDirStart) + # check MiniFAT only if it is not empty: + if self.csectMiniFat: + self._check_duplicate_stream(self.MiniFatStart) + # check DIFAT only if it is not empty: + if self.csectDif: + self._check_duplicate_stream(self.sectDifStart) # Load file allocation tables self.loadfat(header) - - # Load direcory. This sets both the sidlist (ordered by id) + # Load direcory. This sets both the direntries list (ordered by sid) # and the root (ordered by hierarchy) members. - self.loaddirectory(i32(header, 48)) - + self.loaddirectory(self.sectDirStart)#i32(header, 48)) self.ministream = None - self.minifatsect = i32(header, 60) + self.minifatsect = self.MiniFatStart #i32(header, 60) + + + def close(self): + """ + close the OLE file, to release the file object + """ + self.fp.close() + + + def _check_duplicate_stream(self, first_sect, minifat=False): + """ + Checks if a stream has not been already referenced elsewhere. + This method should only be called once for each known stream, and only + if stream size is not null. + first_sect: index of first sector of the stream in FAT + minifat: if True, stream is located in the MiniFAT, else in the FAT + """ + if minifat: + debug('_check_duplicate_stream: sect=%d in MiniFAT' % first_sect) + used_streams = self._used_streams_minifat + else: + debug('_check_duplicate_stream: sect=%d in FAT' % first_sect) + # some values can be safely ignored (not a real stream): + if first_sect in (DIFSECT,FATSECT,ENDOFCHAIN,FREESECT): + return + used_streams = self._used_streams_fat + #TODO: would it be more efficient using a dict or hash values, instead + # of a list of long ? + if first_sect in used_streams: + self._raise_defect(DEFECT_INCORRECT, 'Stream referenced twice') + else: + used_streams.append(first_sect) + + + def dumpfat(self, fat, firstindex=0): + "Displays a part of FAT in human-readable form for debugging purpose" + # [PL] added only for debug + if not DEBUG_MODE: + return + # dictionary to convert special FAT values in human-readable strings + VPL=8 # valeurs par ligne (8+1 * 8+1 = 81) + fatnames = { + FREESECT: "..free..", + ENDOFCHAIN: "[ END. ]", + FATSECT: "FATSECT ", + DIFSECT: "DIFSECT " + } + nbsect = len(fat) + nlines = (nbsect+VPL-1)//VPL + print("index", end=" ") + for i in range(VPL): + print("%8X" % i, end=" ") + print() + for l in range(nlines): + index = l*VPL + print("%8X:" % (firstindex+index), end=" ") + for i in range(index, index+VPL): + if i>=nbsect: + break + sect = fat[i] + if sect in fatnames: + nom = fatnames[sect] + else: + if sect == i+1: + nom = " --->" + else: + nom = "%8X" % sect + print(nom, end=" ") + print() + + + def dumpsect(self, sector, firstindex=0): + "Displays a sector in a human-readable form, for debugging purpose." + if not DEBUG_MODE: + return + VPL=8 # number of values per line (8+1 * 8+1 = 81) + tab = array.array(UINT32, sector) + nbsect = len(tab) + nlines = (nbsect+VPL-1)//VPL + print("index", end=" ") + for i in range(VPL): + print("%8X" % i, end=" ") + print() + for l in range(nlines): + index = l*VPL + print("%8X:" % (firstindex+index), end=" ") + for i in range(index, index+VPL): + if i>=nbsect: + break + sect = tab[i] + nom = "%8X" % sect + print(nom, end=" ") + print() + + def sect2array(self, sect): + """ + convert a sector to an array of 32 bits unsigned integers, + swapping bytes on big endian CPUs such as PowerPC (old Macs) + """ + a = array.array(UINT32, sect) + # if CPU is big endian, swap bytes: + if sys.byteorder == 'big': + a.byteswap() + return a + + + def loadfat_sect(self, sect): + """ + Adds the indexes of the given sector to the FAT + sect: string containing the first FAT sector, or array of long integers + return: index of last FAT sector. + """ + # a FAT sector is an array of ulong integers. + if isinstance(sect, array.array): + # if sect is already an array it is directly used + fat1 = sect + else: + # if it's a raw sector, it is parsed in an array + fat1 = self.sect2array(sect) + self.dumpsect(sect) + # The FAT is a sector chain starting at the first index of itself. + for isect in fat1: + #print("isect = %X" % isect) + if isect == ENDOFCHAIN or isect == FREESECT: + # the end of the sector chain has been reached + break + # read the FAT sector + s = self.getsect(isect) + # parse it as an array of 32 bits integers, and add it to the + # global FAT array + nextfat = self.sect2array(s) + self.fat = self.fat + nextfat + return isect + def loadfat(self, header): - # Load the FAT table. The header contains a sector numbers + """ + Load the FAT table. + """ + # The header contains a sector numbers # for the first 109 FAT sectors. Additional sectors are - # described by DIF blocks (FIXME: not yet implemented) + # described by DIF blocks sect = header[76:512] - fat = [] - for i in range(0, len(sect), 4): - ix = i32(sect, i) - if ix == -2 or ix == -1: # ix == 0xFFFFFFFEL or ix == 0xFFFFFFFFL: - break - s = self.getsect(ix) - fat = fat + [i32(s, i) for i in range(0, len(s), 4)] - self.fat = fat + debug( "len(sect)=%d, so %d integers" % (len(sect), len(sect)//4) ) + #fat = [] + # [PL] FAT is an array of 32 bits unsigned ints, it's more effective + # to use an array than a list in Python. + # It's initialized as empty first: + self.fat = array.array(UINT32) + self.loadfat_sect(sect) + #self.dumpfat(self.fat) +## for i in range(0, len(sect), 4): +## ix = i32(sect, i) +## #[PL] if ix == -2 or ix == -1: # ix == 0xFFFFFFFE or ix == 0xFFFFFFFF: +## if ix == 0xFFFFFFFE or ix == 0xFFFFFFFF: +## break +## s = self.getsect(ix) +## #fat = fat + [i32(s, i) for i in range(0, len(s), 4)] +## fat = fat + array.array(UINT32, s) + if self.csectDif != 0: + # [PL] There's a DIFAT because file is larger than 6.8MB + # some checks just in case: + if self.csectFat <= 109: + # there must be at least 109 blocks in header and the rest in + # DIFAT, so number of sectors must be >109. + self._raise_defect(DEFECT_INCORRECT, 'incorrect DIFAT, not enough sectors') + if self.sectDifStart >= self.nb_sect: + # initial DIFAT block index must be valid + self._raise_defect(DEFECT_FATAL, 'incorrect DIFAT, first index out of range') + debug( "DIFAT analysis..." ) + # We compute the necessary number of DIFAT sectors : + # (each DIFAT sector = 127 pointers + 1 towards next DIFAT sector) + nb_difat = (self.csectFat-109 + 126)//127 + debug( "nb_difat = %d" % nb_difat ) + if self.csectDif != nb_difat: + raise IOError('incorrect DIFAT') + isect_difat = self.sectDifStart + for i in range(nb_difat): + debug( "DIFAT block %d, sector %X" % (i, isect_difat) ) + #TODO: check if corresponding FAT SID = DIFSECT + sector_difat = self.getsect(isect_difat) + difat = self.sect2array(sector_difat) + self.dumpsect(sector_difat) + self.loadfat_sect(difat[:127]) + # last DIFAT pointer is next DIFAT sector: + isect_difat = difat[127] + debug( "next DIFAT sector: %X" % isect_difat ) + # checks: + if isect_difat not in [ENDOFCHAIN, FREESECT]: + # last DIFAT pointer value must be ENDOFCHAIN or FREESECT + raise IOError('incorrect end of DIFAT') +## if len(self.fat) != self.csectFat: +## # FAT should contain csectFat blocks +## print("FAT length: %d instead of %d" % (len(self.fat), self.csectFat)) +## raise IOError('incorrect DIFAT') + # since FAT is read from fixed-size sectors, it may contain more values + # than the actual number of sectors in the file. + # Keep only the relevant sector indexes: + if len(self.fat) > self.nb_sect: + debug('len(fat)=%d, shrunk to nb_sect=%d' % (len(self.fat), self.nb_sect)) + self.fat = self.fat[:self.nb_sect] + debug('\nFAT:') + self.dumpfat(self.fat) + def loadminifat(self): - # Load the MINIFAT table. This is stored in a standard sub- - # stream, pointed to by a header field. - - s = self._open(self.minifatsect).read() - - self.minifat = [i32(s, i) for i in range(0, len(s), 4)] + """ + Load the MiniFAT table. + """ + # MiniFAT is stored in a standard sub-stream, pointed to by a header + # field. + # NOTE: there are two sizes to take into account for this stream: + # 1) Stream size is calculated according to the number of sectors + # declared in the OLE header. This allocated stream may be more than + # needed to store the actual sector indexes. + # (self.csectMiniFat is the number of sectors of size self.SectorSize) + stream_size = self.csectMiniFat * self.SectorSize + # 2) Actually used size is calculated by dividing the MiniStream size + # (given by root entry size) by the size of mini sectors, *4 for + # 32 bits indexes: + nb_minisectors = (self.root.size + self.MiniSectorSize-1) // self.MiniSectorSize + used_size = nb_minisectors * 4 + debug('loadminifat(): minifatsect=%d, nb FAT sectors=%d, used_size=%d, stream_size=%d, nb MiniSectors=%d' % + (self.minifatsect, self.csectMiniFat, used_size, stream_size, nb_minisectors)) + if used_size > stream_size: + # This is not really a problem, but may indicate a wrong implementation: + self._raise_defect(DEFECT_INCORRECT, 'OLE MiniStream is larger than MiniFAT') + # In any case, first read stream_size: + s = self._open(self.minifatsect, stream_size, force_FAT=True).read() + #[PL] Old code replaced by an array: + #self.minifat = [i32(s, i) for i in range(0, len(s), 4)] + self.minifat = self.sect2array(s) + # Then shrink the array to used size, to avoid indexes out of MiniStream: + debug('MiniFAT shrunk from %d to %d sectors' % (len(self.minifat), nb_minisectors)) + self.minifat = self.minifat[:nb_minisectors] + debug('loadminifat(): len=%d' % len(self.minifat)) + debug('\nMiniFAT:') + self.dumpfat(self.minifat) def getsect(self, sect): - # Read given sector + """ + Read given sector from file on disk. + sect: sector index + returns a string containing the sector data. + """ + # [PL] this original code was wrong when sectors are 4KB instead of + # 512 bytes: + #self.fp.seek(512 + self.sectorsize * sect) + #[PL]: added safety checks: + #print("getsect(%X)" % sect) + try: + self.fp.seek(self.sectorsize * (sect+1)) + except: + debug('getsect(): sect=%X, seek=%d, filesize=%d' % + (sect, self.sectorsize*(sect+1), self._filesize)) + self._raise_defect(DEFECT_FATAL, 'OLE sector index out of range') + sector = self.fp.read(self.sectorsize) + if len(sector) != self.sectorsize: + debug('getsect(): sect=%X, read=%d, sectorsize=%d' % + (sect, len(sector), self.sectorsize)) + self._raise_defect(DEFECT_FATAL, 'incomplete OLE sector') + return sector - self.fp.seek(512 + self.sectorsize * sect) - return self.fp.read(self.sectorsize) - - def _unicode(self, s): - # Map unicode string to Latin 1 - - if bytes is str: - # Old version tried to produce a Latin-1 str - return s.decode('utf-16').encode('latin-1', 'replace') - else: - # Provide actual Unicode string - return s.decode('utf-16') def loaddirectory(self, sect): - # Load the directory. The directory is stored in a standard + """ + Load the directory. + sect: sector index of directory stream. + """ + # The directory is stored in a standard # substream, independent of its size. - # read directory stream - fp = self._open(sect) + # open directory stream as a read-only file: + # (stream size is not known in advance) + self.directory_fp = self._open(sect) - # create list of sid entries - self.sidlist = [] - while True: - entry = fp.read(128) - if not entry: - break - type = i8(entry[66]) - name = self._unicode(entry[0:0+i16(entry, 64)]) - ptrs = i32(entry, 68), i32(entry, 72), i32(entry, 76) - sect, size = i32(entry, 116), i32(entry, 120) - clsid = self._clsid(entry[80:96]) - self.sidlist.append((name, type, sect, size, ptrs, clsid)) + #[PL] to detect malformed documents and avoid DoS attacks, the maximum + # number of directory entries can be calculated: + max_entries = self.directory_fp.size // 128 + debug('loaddirectory: size=%d, max_entries=%d' % + (self.directory_fp.size, max_entries)) + + # Create list of directory entries + #self.direntries = [] + # We start with a list of "None" object + self.direntries = [None] * max_entries +## for sid in range(max_entries): +## entry = fp.read(128) +## if not entry: +## break +## self.direntries.append(_OleDirectoryEntry(entry, sid, self)) + # load root entry: + root_entry = self._load_direntry(0) + # Root entry is the first entry: + self.root = self.direntries[0] + # read and build all storage trees, starting from the root: + self.root.build_storage_tree() + + + def _load_direntry (self, sid): + """ + Load a directory entry from the directory. + This method should only be called once for each storage/stream when + loading the directory. + sid: index of storage/stream in the directory. + return: a _OleDirectoryEntry object + raise: IOError if the entry has always been referenced. + """ + # check if SID is OK: + if sid<0 or sid>=len(self.direntries): + self._raise_defect(DEFECT_FATAL, "OLE directory index out of range") + # check if entry was already referenced: + if self.direntries[sid] is not None: + self._raise_defect(DEFECT_INCORRECT, + "double reference for OLE stream/storage") + # if exception not raised, return the object + return self.direntries[sid] + self.directory_fp.seek(sid * 128) + entry = self.directory_fp.read(128) + self.direntries[sid] = _OleDirectoryEntry(entry, sid, self) + return self.direntries[sid] - # create hierarchical list of directory entries - self.root = _OleDirectoryEntry(self.sidlist, 0) def dumpdirectory(self): - # Dump directory (for debugging only) - + """ + Dump directory (for debugging only) + """ self.root.dump() - def _clsid(self, clsid): - if clsid == "\0" * len(clsid): - return "" - return (("%08X-%04X-%04X-%02X%02X-" + "%02X" * 6) % - ((i32(clsid, 0), i16(clsid, 4), i16(clsid, 6)) + - tuple(map(i8, clsid[8:16])))) - def _list(self, files, prefix, node): - # listdir helper + def _open(self, start, size = 0x7FFFFFFF, force_FAT=False): + """ + Open a stream, either in FAT or MiniFAT according to its size. + (openstream helper) + start: index of first sector + size: size of stream (or nothing if size is unknown) + force_FAT: if False (default), stream will be opened in FAT or MiniFAT + according to size. If True, it will always be opened in FAT. + """ + debug('OleFileIO.open(): sect=%d, size=%d, force_FAT=%s' % + (start, size, str(force_FAT))) + # stream size is compared to the MiniSectorCutoff threshold: + if size < self.minisectorcutoff and not force_FAT: + # ministream object + if not self.ministream: + # load MiniFAT if it wasn't already done: + self.loadminifat() + # The first sector index of the miniFAT stream is stored in the + # root directory entry: + size_ministream = self.root.size + debug('Opening MiniStream: sect=%d, size=%d' % + (self.root.isectStart, size_ministream)) + self.ministream = self._open(self.root.isectStart, + size_ministream, force_FAT=True) + return _OleStream(self.ministream, start, size, 0, + self.minisectorsize, self.minifat, + self.ministream.size) + else: + # standard stream + return _OleStream(self.fp, start, size, 512, + self.sectorsize, self.fat, self._filesize) + + + def _list(self, files, prefix, node, streams=True, storages=False): + """ + (listdir helper) + files: list of files to fill in + prefix: current location in storage tree (list of names) + node: current node (_OleDirectoryEntry object) + streams: bool, include streams if True (True by default) - new in v0.26 + storages: bool, include storages if True (False by default) - new in v0.26 + (note: the root storage is never included) + """ prefix = prefix + [node.name] for entry in node.kids: if entry.kids: - self._list(files, prefix, entry) + # this is a storage + if storages: + # add it to the list + files.append(prefix[1:] + [entry.name]) + # check its kids + self._list(files, prefix, entry, streams, storages) else: - files.append(prefix[1:] + [entry.name]) + # this is a stream + if streams: + # add it to the list + files.append(prefix[1:] + [entry.name]) + + + def listdir(self, streams=True, storages=False): + """ + Return a list of streams stored in this file + + streams: bool, include streams if True (True by default) - new in v0.26 + storages: bool, include storages if True (False by default) - new in v0.26 + (note: the root storage is never included) + """ + files = [] + self._list(files, [], self.root, streams, storages) + return files + def _find(self, filename): - # openstream helper + """ + Returns directory entry of given filename. (openstream helper) + Note: this method is case-insensitive. + filename: path of stream in storage tree (except root entry), either: + - a string using Unix path syntax, for example: + 'storage_1/storage_1.2/stream' + - a list of storage filenames, path to the desired stream/storage. + Example: ['storage_1', 'storage_1.2', 'stream'] + return: sid of requested filename + raise IOError if file not found + """ + + # if filename is a string instead of a list, split it on slashes to + # convert to a list: + if isinstance(filename, basestring): + filename = filename.split('/') + # walk across storage tree, following given path: node = self.root for name in filename: for kid in node.kids: - if kid.name == name: + if kid.name.lower() == name.lower(): break else: raise IOError("file not found") node = kid return node.sid - def _open(self, start, size = 0x7FFFFFFF): - # openstream helper. - - if size < self.minisectorcutoff: - # ministream object - if not self.ministream: - self.loadminifat() - self.ministream = self._open(self.sidlist[0][2]) - return _OleStream(self.ministream, start, size, 0, - self.minisectorsize, self.minifat) - - # standard stream - return _OleStream(self.fp, start, size, 512, - self.sectorsize, self.fat) - - ## - # Returns a list of streams stored in this file. - - def listdir(self): - """Return a list of streams stored in this file""" - - files = [] - self._list(files, [], self.root) - return files - - ## - # Opens a stream as a read-only file object. def openstream(self, filename): - """Open a stream as a read-only file object""" + """ + Open a stream as a read-only file object (BytesIO). - slot = self._find(filename) - name, type, sect, size, sids, clsid = self.sidlist[slot] - if type != 2: + filename: path of stream in storage tree (except root entry), either: + - a string using Unix path syntax, for example: + 'storage_1/storage_1.2/stream' + - a list of storage filenames, path to the desired stream/storage. + Example: ['storage_1', 'storage_1.2', 'stream'] + return: file object (read-only) + raise IOError if filename not found, or if this is not a stream. + """ + sid = self._find(filename) + entry = self.direntries[sid] + if entry.entry_type != STGTY_STREAM: raise IOError("this file is not a stream") - return self._open(sect, size) + return self._open(entry.isectStart, entry.size) - ## - # Gets a list of properties described in substream. - def getproperties(self, filename): - """Return properties described in substream""" + def get_type(self, filename): + """ + Test if given filename exists as a stream or a storage in the OLE + container, and return its type. + + filename: path of stream in storage tree. (see openstream for syntax) + return: False if object does not exist, its entry type (>0) otherwise: + - STGTY_STREAM: a stream + - STGTY_STORAGE: a storage + - STGTY_ROOT: the root entry + """ + try: + sid = self._find(filename) + entry = self.direntries[sid] + return entry.entry_type + except: + return False + + + def getmtime(self, filename): + """ + Return modification time of a stream/storage. + + filename: path of stream/storage in storage tree. (see openstream for + syntax) + return: None if modification time is null, a python datetime object + otherwise (UTC timezone) + + new in version 0.26 + """ + sid = self._find(filename) + entry = self.direntries[sid] + return entry.getmtime() + + + def getctime(self, filename): + """ + Return creation time of a stream/storage. + + filename: path of stream/storage in storage tree. (see openstream for + syntax) + return: None if creation time is null, a python datetime object + otherwise (UTC timezone) + + new in version 0.26 + """ + sid = self._find(filename) + entry = self.direntries[sid] + return entry.getctime() + + + def exists(self, filename): + """ + Test if given filename exists as a stream or a storage in the OLE + container. + + filename: path of stream in storage tree. (see openstream for syntax) + return: True if object exist, else False. + """ + try: + sid = self._find(filename) + return True + except: + return False + + + def get_size(self, filename): + """ + Return size of a stream in the OLE container, in bytes. + + filename: path of stream in storage tree (see openstream for syntax) + return: size in bytes (long integer) + raise: IOError if file not found, TypeError if this is not a stream. + """ + sid = self._find(filename) + entry = self.direntries[sid] + if entry.entry_type != STGTY_STREAM: + #TODO: Should it return zero instead of raising an exception ? + raise TypeError('object is not an OLE stream') + return entry.size + + + def get_rootentry_name(self): + """ + Return root entry name. Should usually be 'Root Entry' or 'R' in most + implementations. + """ + return self.root.name + + + def getproperties(self, filename, convert_time=False, no_conversion=None): + """ + Return properties described in substream. + + filename: path of stream in storage tree (see openstream for syntax) + convert_time: bool, if True timestamps will be converted to Python datetime + no_conversion: None or list of int, timestamps not to be converted + (for example total editing time is not a real timestamp) + return: a dictionary of values indexed by id (integer) + """ + # make sure no_conversion is a list, just to simplify code below: + if no_conversion == None: + no_conversion = [] + # stream path as a string to report exceptions: + streampath = filename + if not isinstance(streampath, str): + streampath = '/'.join(streampath) fp = self.openstream(filename) data = {} - # header - s = fp.read(28) - clsid = self._clsid(s[8:24]) + try: + # header + s = fp.read(28) + clsid = _clsid(s[8:24]) - # format id - s = fp.read(20) - fmtid = self._clsid(s[:16]) - fp.seek(i32(s, 16)) + # format id + s = fp.read(20) + fmtid = _clsid(s[:16]) + fp.seek(i32(s, 16)) - # get section - s = "****" + fp.read(i32(fp.read(4))-4) + # get section + s = b"****" + fp.read(i32(fp.read(4))-4) + # number of properties: + num_props = i32(s, 4) + except BaseException as exc: + # catch exception while parsing property header, and only raise + # a DEFECT_INCORRECT then return an empty dict, because this is not + # a fatal error when parsing the whole file + msg = 'Error while parsing properties header in stream %s: %s' % ( + repr(streampath), exc) + self._raise_defect(DEFECT_INCORRECT, msg, type(exc)) + return data - for i in range(i32(s, 4)): + for i in range(num_props): + try: + id = 0 # just in case of an exception + id = i32(s, 8+i*8) + offset = i32(s, 12+i*8) + type = i32(s, offset) - id = i32(s, 8+i*8) - offset = i32(s, 12+i*8) - type = i32(s, offset) + debug ('property id=%d: type=%d offset=%X' % (id, type, offset)) - # test for common types first (should perhaps use - # a dictionary instead?) + # test for common types first (should perhaps use + # a dictionary instead?) - if type == VT_I2: - value = i16(s, offset+4) - if value >= 32768: - value = value - 65536 - elif type == VT_UI2: - value = i16(s, offset+4) - elif type in (VT_I4, VT_ERROR): - value = i32(s, offset+4) - elif type == VT_UI4: - value = i32(s, offset+4) # FIXME - elif type in (VT_BSTR, VT_LPSTR): - count = i32(s, offset+4) - value = s[offset+8:offset+8+count-1] - elif type == VT_BLOB: - count = i32(s, offset+4) - value = s[offset+8:offset+8+count] - elif type == VT_LPWSTR: - count = i32(s, offset+4) - value = self._unicode(s[offset+8:offset+8+count*2]) - elif type == VT_FILETIME: - value = long(i32(s, offset+4)) + (long(i32(s, offset+8))<<32) - # FIXME: this is a 64-bit int: "number of 100ns periods - # since Jan 1,1601". Should map this to Python time - value = value // 10000000 # seconds - elif type == VT_UI1: - value = i8(s[offset+4]) - elif type == VT_CLSID: - value = self._clsid(s[offset+4:offset+20]) - elif type == VT_CF: - count = i32(s, offset+4) - value = s[offset+8:offset+8+count] - else: - value = None # everything else yields "None" + if type == VT_I2: # 16-bit signed integer + value = i16(s, offset+4) + if value >= 32768: + value = value - 65536 + elif type == VT_UI2: # 2-byte unsigned integer + value = i16(s, offset+4) + elif type in (VT_I4, VT_INT, VT_ERROR): + # VT_I4: 32-bit signed integer + # VT_ERROR: HRESULT, similar to 32-bit signed integer, + # see http://msdn.microsoft.com/en-us/library/cc230330.aspx + value = i32(s, offset+4) + elif type in (VT_UI4, VT_UINT): # 4-byte unsigned integer + value = i32(s, offset+4) # FIXME + elif type in (VT_BSTR, VT_LPSTR): + # CodePageString, see http://msdn.microsoft.com/en-us/library/dd942354.aspx + # size is a 32 bits integer, including the null terminator, and + # possibly trailing or embedded null chars + #TODO: if codepage is unicode, the string should be converted as such + count = i32(s, offset+4) + value = s[offset+8:offset+8+count-1] + # remove all null chars: + value = value.replace(b'\x00', b'') + elif type == VT_BLOB: + # binary large object (BLOB) + # see http://msdn.microsoft.com/en-us/library/dd942282.aspx + count = i32(s, offset+4) + value = s[offset+8:offset+8+count] + elif type == VT_LPWSTR: + # UnicodeString + # see http://msdn.microsoft.com/en-us/library/dd942313.aspx + # "the string should NOT contain embedded or additional trailing + # null characters." + count = i32(s, offset+4) + value = _unicode(s[offset+8:offset+8+count*2]) + elif type == VT_FILETIME: + value = long(i32(s, offset+4)) + (long(i32(s, offset+8))<<32) + # FILETIME is a 64-bit int: "number of 100ns periods + # since Jan 1,1601". + if convert_time and id not in no_conversion: + debug('Converting property #%d to python datetime, value=%d=%fs' + %(id, value, float(value)/10000000)) + # convert FILETIME to Python datetime.datetime + # inspired from http://code.activestate.com/recipes/511425-filetime-to-datetime/ + _FILETIME_null_date = datetime.datetime(1601, 1, 1, 0, 0, 0) + debug('timedelta days=%d' % (value//(10*1000000*3600*24))) + value = _FILETIME_null_date + datetime.timedelta(microseconds=value//10) + else: + # legacy code kept for backward compatibility: returns a + # number of seconds since Jan 1,1601 + value = value // 10000000 # seconds + elif type == VT_UI1: # 1-byte unsigned integer + value = i8(s[offset+4]) + elif type == VT_CLSID: + value = _clsid(s[offset+4:offset+20]) + elif type == VT_CF: + # PropertyIdentifier or ClipboardData?? + # see http://msdn.microsoft.com/en-us/library/dd941945.aspx + count = i32(s, offset+4) + value = s[offset+8:offset+8+count] + elif type == VT_BOOL: + # VARIANT_BOOL, 16 bits bool, 0x0000=Fals, 0xFFFF=True + # see http://msdn.microsoft.com/en-us/library/cc237864.aspx + value = bool(i16(s, offset+4)) + else: + value = None # everything else yields "None" + debug ('property id=%d: type=%d not implemented in parser yet' % (id, type)) - # FIXME: add support for VT_VECTOR + # missing: VT_EMPTY, VT_NULL, VT_R4, VT_R8, VT_CY, VT_DATE, + # VT_DECIMAL, VT_I1, VT_I8, VT_UI8, + # see http://msdn.microsoft.com/en-us/library/dd942033.aspx - #print "%08x" % id, repr(value), - #print "(%s)" % VT[i32(s, offset) & 0xFFF] + # FIXME: add support for VT_VECTOR + # VT_VECTOR is a 32 uint giving the number of items, followed by + # the items in sequence. The VT_VECTOR value is combined with the + # type of items, e.g. VT_VECTOR|VT_BSTR + # see http://msdn.microsoft.com/en-us/library/dd942011.aspx - data[id] = value + #print("%08x" % id, repr(value), end=" ") + #print("(%s)" % VT[i32(s, offset) & 0xFFF]) + + data[id] = value + except BaseException as exc: + # catch exception while parsing each property, and only raise + # a DEFECT_INCORRECT, because parsing can go on + msg = 'Error while parsing property id %d in stream %s: %s' % ( + id, repr(streampath), exc) + self._raise_defect(DEFECT_INCORRECT, msg, type(exc)) return data + def get_metadata(self): + """ + Parse standard properties streams, return an OleMetadata object + containing all the available metadata. + (also stored in the metadata attribute of the OleFileIO object) + + new in version 0.25 + """ + self.metadata = OleMetadata() + self.metadata.parse_properties(self) + return self.metadata + # # -------------------------------------------------------------------- # This script can be used to dump the directory of any OLE2 structured @@ -517,19 +1918,105 @@ if __name__ == "__main__": import sys - for file in sys.argv[1:]: - try: - ole = OleFileIO(file) + # [PL] display quick usage info if launched from command-line + if len(sys.argv) <= 1: + print(__doc__) + print(""" +Launched from command line, this script parses OLE files and prints info. + +Usage: OleFileIO_PL.py [-d] [-c] [file2 ...] + +Options: +-d : debug mode (display a lot of debug information, for developers only) +-c : check all streams (for debugging purposes) +""") + sys.exit() + + check_streams = False + for filename in sys.argv[1:]: +## try: + # OPTIONS: + if filename == '-d': + # option to switch debug mode on: + set_debug_mode(True) + continue + if filename == '-c': + # option to switch check streams mode on: + check_streams = True + continue + + ole = OleFileIO(filename)#, raise_defects=DEFECT_INCORRECT) print("-" * 68) - print(file) + print(filename) print("-" * 68) ole.dumpdirectory() - for file in ole.listdir(): - if file[-1][0] == "\005": - print(file) - props = ole.getproperties(file) + for streamname in ole.listdir(): + if streamname[-1][0] == "\005": + print(streamname, ": properties") + props = ole.getproperties(streamname, convert_time=True) props = sorted(props.items()) for k, v in props: + #[PL]: avoid to display too large or binary values: + if isinstance(v, (basestring, bytes)): + if len(v) > 50: + v = v[:50] + if isinstance(v, bytes): + # quick and dirty binary check: + for c in (1,2,3,4,5,6,7,11,12,14,15,16,17,18,19,20, + 21,22,23,24,25,26,27,28,29,30,31): + if c in bytearray(v): + v = '(binary data)' + break print(" ", k, v) - except IOError as v: - print("***", "cannot read", file, "-", v) + + if check_streams: + # Read all streams to check if there are errors: + print('\nChecking streams...') + for streamname in ole.listdir(): + # print name using repr() to convert binary chars to \xNN: + print('-', repr('/'.join(streamname)),'-', end=' ') + st_type = ole.get_type(streamname) + if st_type == STGTY_STREAM: + print('size %d' % ole.get_size(streamname)) + # just try to read stream in memory: + ole.openstream(streamname) + else: + print('NOT a stream : type=%d' % st_type) + print() + +## for streamname in ole.listdir(): +## # print name using repr() to convert binary chars to \xNN: +## print('-', repr('/'.join(streamname)),'-', end=' ') +## print(ole.getmtime(streamname)) +## print() + + print('Modification/Creation times of all directory entries:') + for entry in ole.direntries: + if entry is not None: + print('- %s: mtime=%s ctime=%s' % (entry.name, + entry.getmtime(), entry.getctime())) + print() + + # parse and display metadata: + meta = ole.get_metadata() + meta.dump() + print() + #[PL] Test a few new methods: + root = ole.get_rootentry_name() + print('Root entry name: "%s"' % root) + if ole.exists('worddocument'): + print("This is a Word document.") + print("type of stream 'WordDocument':", ole.get_type('worddocument')) + print("size :", ole.get_size('worddocument')) + if ole.exists('macros/vba'): + print("This document may contain VBA macros.") + + # print parsing issues: + print('\nNon-fatal issues raised during parsing:') + if ole.parsing_issues: + for exctype, msg in ole.parsing_issues: + print('- %s: %s' % (exctype.__name__, msg)) + else: + print('None') +## except IOError as v: +## print("***", "cannot read", file, "-", v) diff --git a/PIL/PcxImagePlugin.py b/PIL/PcxImagePlugin.py index 42dd9be0b..2496af676 100644 --- a/PIL/PcxImagePlugin.py +++ b/PIL/PcxImagePlugin.py @@ -55,12 +55,18 @@ class PcxImageFile(ImageFile.ImageFile): bbox = i16(s,4), i16(s,6), i16(s,8)+1, i16(s,10)+1 if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]: raise SyntaxError("bad PCX image size") + if Image.DEBUG: + print ("BBox: %s %s %s %s" % bbox) + # format version = i8(s[1]) bits = i8(s[3]) planes = i8(s[65]) stride = i16(s,66) + if Image.DEBUG: + print ("PCX version %s, bits %s, planes %s, stride %s" % + (version, bits, planes, stride)) self.info["dpi"] = i16(s,12), i16(s,14) @@ -98,7 +104,9 @@ class PcxImageFile(ImageFile.ImageFile): self.size = bbox[2]-bbox[0], bbox[3]-bbox[1] bbox = (0, 0) + self.size - + if Image.DEBUG: + print ("size: %sx%s" % self.size) + self.tile = [("pcx", bbox, self.fp.tell(), (rawmode, planes * stride))] # -------------------------------------------------------------------- @@ -126,6 +134,16 @@ def _save(im, fp, filename, check=0): # bytes per plane stride = (im.size[0] * bits + 7) // 8 + # stride should be even + stride = stride + (stride % 2) + # Stride needs to be kept in sync with the PcxEncode.c version. + # Ideally it should be passed in in the state, but the bytes value + # gets overwritten. + + + if Image.DEBUG: + print ("PcxImagePlugin._save: xwidth: %d, bits: %d, stride: %d" % ( + im.size[0], bits, stride)) # under windows, we could determine the current screen size with # "Image.core.display_mode()[1]", but I think that's overkill... diff --git a/PIL/PngImagePlugin.py b/PIL/PngImagePlugin.py index 18abf27d3..2bdf74608 100644 --- a/PIL/PngImagePlugin.py +++ b/PIL/PngImagePlugin.py @@ -505,7 +505,7 @@ def _save(im, fp, filename, chunk=putchunk, check=0): else: # check palette contents if im.palette: - colors = len(im.palette.getdata()[1])//3 + colors = max(min(len(im.palette.getdata()[1])//3, 256), 2) else: colors = 256 diff --git a/PIL/TiffImagePlugin.py b/PIL/TiffImagePlugin.py index 68a77e9ca..18d5909dc 100644 --- a/PIL/TiffImagePlugin.py +++ b/PIL/TiffImagePlugin.py @@ -693,10 +693,11 @@ class TiffImageFile(ImageFile.ImageFile): if not len(self.tile) == 1: raise IOError("Not exactly one tile") - d, e, o, a = self.tile[0] - d = Image._getdecoder(self.mode, 'libtiff', a, self.decoderconfig) + # (self._compression, (extents tuple), 0, (rawmode, self._compression, fp)) + ignored, extents, ignored_2, args = self.tile[0] + decoder = Image._getdecoder(self.mode, 'libtiff', args, self.decoderconfig) try: - d.setimage(self.im, e) + decoder.setimage(self.im, extents) except ValueError: raise IOError("Couldn't set the image") @@ -712,27 +713,30 @@ class TiffImageFile(ImageFile.ImageFile): # with here by reordering. if Image.DEBUG: print ("have getvalue. just sending in a string from getvalue") - n,e = d.decode(self.fp.getvalue()) + n,err = decoder.decode(self.fp.getvalue()) elif hasattr(self.fp, "fileno"): # we've got a actual file on disk, pass in the fp. if Image.DEBUG: print ("have fileno, calling fileno version of the decoder.") self.fp.seek(0) - n,e = d.decode(b"fpfp") # 4 bytes, otherwise the trace might error out + n,err = decoder.decode(b"fpfp") # 4 bytes, otherwise the trace might error out else: # we have something else. if Image.DEBUG: print ("don't have fileno or getvalue. just reading") # UNDONE -- so much for that buffer size thing. - n, e = d.decode(self.fp.read()) + n,err = decoder.decode(self.fp.read()) self.tile = [] self.readonly = 0 + # libtiff closed the fp in a, we need to close self.fp, if possible + if hasattr(self.fp, 'close'): + self.fp.close() self.fp = None # might be shared - if e < 0: - raise IOError(e) + if err < 0: + raise IOError(err) self.load_end() diff --git a/PIL/__init__.py b/PIL/__init__.py index 18bd42a5f..c35f6207e 100644 --- a/PIL/__init__.py +++ b/PIL/__init__.py @@ -12,7 +12,7 @@ # ;-) VERSION = '1.1.7' # PIL version -PILLOW_VERSION = '2.3.0' # Pillow +PILLOW_VERSION = '2.4.0' # Pillow _plugins = ['ArgImagePlugin', 'BmpImagePlugin', @@ -33,6 +33,7 @@ _plugins = ['ArgImagePlugin', 'ImtImagePlugin', 'IptcImagePlugin', 'JpegImagePlugin', + 'Jpeg2KImagePlugin', 'McIdasImagePlugin', 'MicImagePlugin', 'MpegImagePlugin', diff --git a/PIL/_binary.py b/PIL/_binary.py index 37318a21a..71b2b78c9 100644 --- a/PIL/_binary.py +++ b/PIL/_binary.py @@ -25,10 +25,23 @@ else: return bytes((i&255,)) # Input, le = little endian, be = big endian +#TODO: replace with more readable struct.unpack equivalent def i16le(c, o=0): + """ + Converts a 2-bytes (16 bits) string to an integer. + + c: string containing bytes to convert + o: offset of bytes to convert in string + """ return i8(c[o]) | (i8(c[o+1])<<8) def i32le(c, o=0): + """ + Converts a 4-bytes (32 bits) string to an integer. + + c: string containing bytes to convert + o: offset of bytes to convert in string + """ return i8(c[o]) | (i8(c[o+1])<<8) | (i8(c[o+2])<<16) | (i8(c[o+3])<<24) def i16be(c, o=0): diff --git a/PIL/_util.py b/PIL/_util.py index 220ac6c52..761c258f1 100644 --- a/PIL/_util.py +++ b/PIL/_util.py @@ -14,3 +14,9 @@ else: # Checks if an object is a string, and that it points to a directory. def isDirectory(f): return isPath(f) and os.path.isdir(f) + +class import_err(object): + def __init__(self, ex): + self.ex = ex + def __getattr__(self, elt): + raise self.ex diff --git a/Sane/_sane.c b/Sane/_sane.c index 1c62610be..2ebcb1834 100644 --- a/Sane/_sane.c +++ b/Sane/_sane.c @@ -916,10 +916,13 @@ SaneDev_snap(SaneDevObject *self, PyObject *args) call which returns SANE_STATUS_EOF in order to start a new frame. */ - do { - st = sane_read(self->h, buffer, READSIZE, &len); - } - while (st == SANE_STATUS_GOOD); + if (st != SANE_STATUS_EOF) + { + do { + st = sane_read(self->h, buffer, READSIZE, &len); + } + while (st == SANE_STATUS_GOOD); + } if (st != SANE_STATUS_EOF) { Py_BLOCK_THREADS @@ -937,10 +940,13 @@ SaneDev_snap(SaneDevObject *self, PyObject *args) } } /* enforce SANE_STATUS_EOF. Can be necessary for ADF scans for some backends */ - do { - st = sane_read(self->h, buffer, READSIZE, &len); - } - while (st == SANE_STATUS_GOOD); + if (st != SANE_STATUS_EOF) + { + do { + st = sane_read(self->h, buffer, READSIZE, &len); + } + while (st == SANE_STATUS_GOOD); + } if (st != SANE_STATUS_EOF) { sane_cancel(self->h); diff --git a/Tests/images/high_ascii_chars.png b/Tests/images/high_ascii_chars.png new file mode 100644 index 000000000..fc9ab8401 Binary files /dev/null and b/Tests/images/high_ascii_chars.png differ diff --git a/Tests/images/pillow2.icns b/Tests/images/pillow2.icns new file mode 100644 index 000000000..5f0ff13b9 Binary files /dev/null and b/Tests/images/pillow2.icns differ diff --git a/Tests/images/pillow3.icns b/Tests/images/pillow3.icns new file mode 100644 index 000000000..ef9b89178 Binary files /dev/null and b/Tests/images/pillow3.icns differ diff --git a/Tests/images/python.ico b/Tests/images/python.ico new file mode 100644 index 000000000..c9efc5844 Binary files /dev/null and b/Tests/images/python.ico differ diff --git a/Tests/images/test-card-lossless.jp2 b/Tests/images/test-card-lossless.jp2 new file mode 100644 index 000000000..497b97b8d Binary files /dev/null and b/Tests/images/test-card-lossless.jp2 differ diff --git a/Tests/images/test-card-lossy-tiled.jp2 b/Tests/images/test-card-lossy-tiled.jp2 new file mode 100644 index 000000000..482773188 Binary files /dev/null and b/Tests/images/test-card-lossy-tiled.jp2 differ diff --git a/Tests/images/test-card.png b/Tests/images/test-card.png new file mode 100644 index 000000000..4b0e1f8de Binary files /dev/null and b/Tests/images/test-card.png differ diff --git a/Tests/images/test.colors.gif b/Tests/images/test.colors.gif new file mode 100644 index 000000000..0faf96760 Binary files /dev/null and b/Tests/images/test.colors.gif differ diff --git a/Tests/run.py b/Tests/run.py index 521e66af8..b2129db5f 100644 --- a/Tests/run.py +++ b/Tests/run.py @@ -2,7 +2,7 @@ from __future__ import print_function # minimal test runner -import glob, os, os.path, sys, tempfile +import glob, os, os.path, sys, tempfile, re from multiprocessing import Pool try: @@ -19,15 +19,41 @@ python_options = [] tester_options = [] include = [x for x in sys.argv[1:] if x[:2] != "--"] +ignore_re = re.compile('^ignore: (.*)$', re.MULTILINE) + def test_one(f): test, ext = os.path.splitext(os.path.basename(f)) + print("running", test, "...") # 2>&1 works on unix and on modern windowses. we might care about # very old Python versions, but not ancient microsoft products :-) out = os.popen("%s %s -u %s %s 2>&1" % ( sys.executable, python_options, f, tester_options )) - result = out.read().strip() + + result = out.read() + + # Extract any ignore patterns + ignore_pats = ignore_re.findall(result) + result = ignore_re.sub('', result) + + try: + def fix_re(p): + if not p.startswith('^'): + p = '^' + p + if not p.endswith('$'): + p = p + '$' + return p + + ignore_res = [re.compile(fix_re(p), re.MULTILINE) for p in ignore_pats] + except: + print('(bad ignore patterns %r)' % ignore_pats) + ignore_res = [] + + for r in ignore_res: + result = r.sub('', result) + + result = result.strip() status = out.close() return (result, status) diff --git a/Tests/test_bmp_reference.py b/Tests/test_bmp_reference.py index 0e03df79a..99818229f 100644 --- a/Tests/test_bmp_reference.py +++ b/Tests/test_bmp_reference.py @@ -3,7 +3,7 @@ from tester import * from PIL import Image import os -base = 'Tests/images/bmp/' +base = os.path.join('Tests', 'images', 'bmp') def get_files(d, ext='.bmp'): @@ -78,9 +78,9 @@ def test_good(): except Exception as msg: # there are three here that are unsupported: - unsupported = ('Tests/images/bmp/g/rgb32bf.bmp', - 'Tests/images/bmp/g/pal8rle.bmp', - 'Tests/images/bmp/g/pal4rle.bmp') + unsupported = (os.path.join(base, 'g', 'rgb32bf.bmp'), + os.path.join(base, 'g', 'pal8rle.bmp'), + os.path.join(base, 'g', 'pal4rle.bmp')) if f not in unsupported: assert_true(False, "Unsupported Image %s: %s" %(f,msg)) diff --git a/Tests/test_cffi.py b/Tests/test_cffi.py index 4065a9e53..1c0d8d31e 100644 --- a/Tests/test_cffi.py +++ b/Tests/test_cffi.py @@ -1,16 +1,15 @@ from tester import * -from PIL import Image, PyAccess - -import test_image_putpixel as put -import test_image_getpixel as get - - - try: import cffi except: skip() + +from PIL import Image, PyAccess + +import test_image_putpixel as put +import test_image_getpixel as get + Image.USE_CFFI_ACCESS = True diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 61faa637a..0041824b1 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -4,19 +4,9 @@ from PIL import Image, EpsImagePlugin import sys import io -if not EpsImagePlugin.gs_windows_binary: - # already checked. Not there. +if not EpsImagePlugin.has_ghostscript(): skip() -if not sys.platform.startswith('win'): - import subprocess - try: - gs = subprocess.Popen(['gs','--version'], stdout=subprocess.PIPE) - gs.stdout.read() - except OSError: - # no ghostscript - skip() - #Our two EPS test files (they are identical except for their bounding boxes) file1 = "Tests/images/zero_bb.eps" file2 = "Tests/images/non_zero_bb.eps" diff --git a/Tests/test_file_gif.py b/Tests/test_file_gif.py index 3a6478e2a..4318e178e 100644 --- a/Tests/test_file_gif.py +++ b/Tests/test_file_gif.py @@ -38,7 +38,7 @@ def test_roundtrip(): def test_roundtrip2(): #see https://github.com/python-imaging/Pillow/issues/403 - out = 'temp.gif'#tempfile('temp.gif') + out = tempfile('temp.gif') im = Image.open('Images/lena.gif') im2 = im.copy() im2.save(out) @@ -46,3 +46,42 @@ def test_roundtrip2(): assert_image_similar(reread.convert('RGB'), lena(), 50) + +def test_palette_handling(): + # see https://github.com/python-imaging/Pillow/issues/513 + + im = Image.open('Images/lena.gif') + im = im.convert('RGB') + + im = im.resize((100,100), Image.ANTIALIAS) + im2 = im.convert('P', palette=Image.ADAPTIVE, colors=256) + + f = tempfile('temp.gif') + im2.save(f, optimize=True) + + reloaded = Image.open(f) + + assert_image_similar(im, reloaded.convert('RGB'), 10) + +def test_palette_434(): + # see https://github.com/python-imaging/Pillow/issues/434 + + def roundtrip(im, *args, **kwargs): + out = tempfile('temp.gif') + im.save(out, *args, **kwargs) + reloaded = Image.open(out) + + return [im, reloaded] + + orig = "Tests/images/test.colors.gif" + im = Image.open(orig) + + assert_image_equal(*roundtrip(im)) + assert_image_equal(*roundtrip(im, optimize=True)) + + im = im.convert("RGB") + # check automatic P conversion + reloaded = roundtrip(im)[1].convert('RGB') + assert_image_equal(im, reloaded) + + diff --git a/Tests/test_file_icns.py b/Tests/test_file_icns.py new file mode 100644 index 000000000..3e31f8879 --- /dev/null +++ b/Tests/test_file_icns.py @@ -0,0 +1,66 @@ +from tester import * + +from PIL import Image + +# sample icon file +file = "Images/pillow.icns" +data = open(file, "rb").read() + +enable_jpeg2k = hasattr(Image.core, 'jp2klib_version') + +def test_sanity(): + # Loading this icon by default should result in the largest size + # (512x512@2x) being loaded + im = Image.open(file) + im.load() + assert_equal(im.mode, "RGBA") + assert_equal(im.size, (1024, 1024)) + assert_equal(im.format, "ICNS") + +def test_sizes(): + # Check that we can load all of the sizes, and that the final pixel + # dimensions are as expected + im = Image.open(file) + for w,h,r in im.info['sizes']: + wr = w * r + hr = h * r + im2 = Image.open(file) + im2.size = (w, h, r) + im2.load() + assert_equal(im2.mode, 'RGBA') + assert_equal(im2.size, (wr, hr)) + +def test_older_icon(): + # This icon was made with Icon Composer rather than iconutil; it still + # uses PNG rather than JP2, however (since it was made on 10.9). + im = Image.open('Tests/images/pillow2.icns') + for w,h,r in im.info['sizes']: + wr = w * r + hr = h * r + im2 = Image.open('Tests/images/pillow2.icns') + im2.size = (w, h, r) + im2.load() + assert_equal(im2.mode, 'RGBA') + assert_equal(im2.size, (wr, hr)) + +def test_jp2_icon(): + # This icon was made by using Uli Kusterer's oldiconutil to replace + # the PNG images with JPEG 2000 ones. The advantage of doing this is + # that OS X 10.5 supports JPEG 2000 but not PNG; some commercial + # software therefore does just this. + + # (oldiconutil is here: https://github.com/uliwitness/oldiconutil) + + if not enable_jpeg2k: + return + + im = Image.open('Tests/images/pillow3.icns') + for w,h,r in im.info['sizes']: + wr = w * r + hr = h * r + im2 = Image.open('Tests/images/pillow3.icns') + im2.size = (w, h, r) + im2.load() + assert_equal(im2.mode, 'RGBA') + assert_equal(im2.size, (wr, hr)) + diff --git a/Tests/test_file_jpeg.py b/Tests/test_file_jpeg.py index 07a7c9f96..095cad359 100644 --- a/Tests/test_file_jpeg.py +++ b/Tests/test_file_jpeg.py @@ -10,9 +10,7 @@ codecs = dir(Image.core) if "jpeg_encoder" not in codecs or "jpeg_decoder" not in codecs: skip("jpeg support not available") -# sample jpeg stream -file = "Images/lena.jpg" -data = open(file, "rb").read() +test_file = "Images/lena.jpg" def roundtrip(im, **options): out = BytesIO() @@ -30,7 +28,7 @@ def test_sanity(): # internal version number assert_match(Image.core.jpeglib_version, "\d+\.\d+$") - im = Image.open(file) + im = Image.open(test_file) im.load() assert_equal(im.mode, "RGB") assert_equal(im.size, (128, 128)) @@ -40,7 +38,7 @@ def test_sanity(): def test_app(): # Test APP/COM reader (@PIL135) - im = Image.open(file) + im = Image.open(test_file) assert_equal(im.applist[0], ("APP0", b"JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00")) assert_equal(im.applist[1], ("COM", b"Python Imaging Library")) @@ -49,8 +47,8 @@ def test_app(): def test_cmyk(): # Test CMYK handling. Thanks to Tim and Charlie for test data, # Michael for getting me to look one more time. - file = "Tests/images/pil_sample_cmyk.jpg" - im = Image.open(file) + f = "Tests/images/pil_sample_cmyk.jpg" + im = Image.open(f) # the source image has red pixels in the upper left corner. c, m, y, k = [x / 255.0 for x in im.getpixel((0, 0))] assert_true(c == 0.0 and m > 0.8 and y > 0.8 and k == 0.0) @@ -66,7 +64,7 @@ def test_cmyk(): def test_dpi(): def test(xdpi, ydpi=None): - im = Image.open(file) + im = Image.open(test_file) im = roundtrip(im, dpi=(xdpi, ydpi or xdpi)) return im.info.get("dpi") assert_equal(test(72), (72, 72)) @@ -80,9 +78,9 @@ def test_icc(): icc_profile = im1.info["icc_profile"] assert_equal(len(icc_profile), 3144) # Roundtrip via physical file. - file = tempfile("temp.jpg") - im1.save(file, icc_profile=icc_profile) - im2 = Image.open(file) + f = tempfile("temp.jpg") + im1.save(f, icc_profile=icc_profile) + im2 = Image.open(f) assert_equal(im2.info.get("icc_profile"), icc_profile) # Roundtrip via memory buffer. im1 = roundtrip(lena()) @@ -203,3 +201,9 @@ def test_exif(): im = Image.open("Tests/images/pil_sample_rgb.jpg") info = im._getexif() assert_equal(info[305], 'Adobe Photoshop CS Macintosh') + + +def test_quality_keep(): + im = Image.open("Images/lena.jpg") + f = tempfile('temp.jpg') + assert_no_exception(lambda: im.save(f, quality='keep')) diff --git a/Tests/test_file_jpeg2k.py b/Tests/test_file_jpeg2k.py new file mode 100644 index 000000000..b11e5e6ab --- /dev/null +++ b/Tests/test_file_jpeg2k.py @@ -0,0 +1,110 @@ +from tester import * + +from PIL import Image +from PIL import ImageFile + +codecs = dir(Image.core) + +if "jpeg2k_encoder" not in codecs or "jpeg2k_decoder" not in codecs: + skip('JPEG 2000 support not available') + +# OpenJPEG 2.0.0 outputs this debugging message sometimes; we should +# ignore it---it doesn't represent a test failure. +ignore('Not enough memory to handle tile data') + +test_card = Image.open('Tests/images/test-card.png') +test_card.load() + +def roundtrip(im, **options): + out = BytesIO() + im.save(out, "JPEG2000", **options) + bytes = out.tell() + out.seek(0) + im = Image.open(out) + im.bytes = bytes # for testing only + im.load() + return im + +# ---------------------------------------------------------------------- + +def test_sanity(): + # Internal version number + assert_match(Image.core.jp2klib_version, '\d+\.\d+\.\d+$') + + im = Image.open('Tests/images/test-card-lossless.jp2') + im.load() + assert_equal(im.mode, 'RGB') + assert_equal(im.size, (640, 480)) + assert_equal(im.format, 'JPEG2000') + +# ---------------------------------------------------------------------- + +# These two test pre-written JPEG 2000 files that were not written with +# PIL (they were made using Adobe Photoshop) + +def test_lossless(): + im = Image.open('Tests/images/test-card-lossless.jp2') + im.load() + im.save('/tmp/test-card.png') + assert_image_similar(im, test_card, 1.0e-3) + +def test_lossy_tiled(): + im = Image.open('Tests/images/test-card-lossy-tiled.jp2') + im.load() + assert_image_similar(im, test_card, 2.0) + +# ---------------------------------------------------------------------- + +def test_lossless_rt(): + im = roundtrip(test_card) + assert_image_equal(im, test_card) + +def test_lossy_rt(): + im = roundtrip(test_card, quality_layers=[20]) + assert_image_similar(im, test_card, 2.0) + +def test_tiled_rt(): + im = roundtrip(test_card, tile_size=(128, 128)) + assert_image_equal(im, test_card) + +def test_tiled_offset_rt(): + im = roundtrip(test_card, tile_size=(128, 128), tile_offset=(0, 0), + offset=(32, 32)) + assert_image_equal(im, test_card) + +def test_irreversible_rt(): + im = roundtrip(test_card, irreversible=True, quality_layers=[20]) + assert_image_similar(im, test_card, 2.0) + +def test_prog_qual_rt(): + im = roundtrip(test_card, quality_layers=[60, 40, 20], progression='LRCP') + assert_image_similar(im, test_card, 2.0) + +def test_prog_res_rt(): + im = roundtrip(test_card, num_resolutions=8, progression='RLCP') + assert_image_equal(im, test_card) + +# ---------------------------------------------------------------------- + +def test_reduce(): + im = Image.open('Tests/images/test-card-lossless.jp2') + im.reduce = 2 + im.load() + assert_equal(im.size, (160, 120)) + +def test_layers(): + out = BytesIO() + test_card.save(out, 'JPEG2000', quality_layers=[100, 50, 10], + progression='LRCP') + out.seek(0) + + im = Image.open(out) + im.layers = 1 + im.load() + assert_image_similar(im, test_card, 13) + + out.seek(0) + im = Image.open(out) + im.layers = 3 + im.load() + assert_image_similar(im, test_card, 0.4) diff --git a/Tests/test_file_libtiff.py b/Tests/test_file_libtiff.py index a94257bc0..a53593a9e 100644 --- a/Tests/test_file_libtiff.py +++ b/Tests/test_file_libtiff.py @@ -1,4 +1,5 @@ from tester import * +import os from PIL import Image, TiffImagePlugin @@ -295,3 +296,13 @@ def xtest_bw_compression_wRGB(): assert_exception(IOError, lambda: im.save(out, compression='group3')) assert_exception(IOError, lambda: im.save(out, compression='group4')) +def test_fp_leak(): + im = Image.open("Tests/images/lena_g4_500.tif") + fn = im.fp.fileno() + + assert_no_exception(lambda: os.fstat(fn)) + im.load() # this should close it. + assert_exception(OSError, lambda: os.fstat(fn)) + im = None # this should force even more closed. + assert_exception(OSError, lambda: os.fstat(fn)) + assert_exception(OSError, lambda: os.close(fn)) diff --git a/Tests/test_file_pcx.py b/Tests/test_file_pcx.py index 73d358229..cb785ec54 100644 --- a/Tests/test_file_pcx.py +++ b/Tests/test_file_pcx.py @@ -2,29 +2,30 @@ from tester import * from PIL import Image + +def _roundtrip(im): + f = tempfile("temp.pcx") + im.save(f) + im2 = Image.open(f) + + assert_equal(im2.mode, im.mode) + assert_equal(im2.size, im.size) + assert_equal(im2.format, "PCX") + assert_image_equal(im2, im) + def test_sanity(): + for mode in ('1', 'L', 'P', 'RGB'): + _roundtrip(lena(mode)) - file = tempfile("temp.pcx") - - lena("1").save(file) - - im = Image.open(file) - im.load() - assert_equal(im.mode, "1") - assert_equal(im.size, (128, 128)) - assert_equal(im.format, "PCX") - - lena("1").save(file) - im = Image.open(file) - - lena("L").save(file) - im = Image.open(file) - - lena("P").save(file) - im = Image.open(file) - - lena("RGB").save(file) - im = Image.open(file) +def test_odd(): + # see issue #523, odd sized images should have a stride that's even. + # not that imagemagick or gimp write pcx that way. + # we were not handling properly. + for mode in ('1', 'L', 'P', 'RGB'): + # larger, odd sized images are better here to ensure that + # we handle interrupted scan lines properly. + _roundtrip(lena(mode).resize((511,511))) + def test_pil184(): # Check reading of files where xmin/xmax is not zero. diff --git a/Tests/test_font_pcf.py b/Tests/test_font_pcf.py index 60e6e0e26..bae214e35 100644 --- a/Tests/test_font_pcf.py +++ b/Tests/test_font_pcf.py @@ -22,10 +22,28 @@ def test_sanity(): font.save(tempname) -def test_draw(): +def xtest_draw(): font = ImageFont.load(tempname) image = Image.new("L", font.getsize(message), "white") draw = ImageDraw.Draw(image) draw.text((0, 0), message, font=font) # assert_signature(image, "7216c60f988dea43a46bb68321e3c1b03ec62aee") + +def _test_high_characters(message): + + font = ImageFont.load(tempname) + image = Image.new("L", font.getsize(message), "white") + draw = ImageDraw.Draw(image) + draw.text((0, 0), message, font=font) + + compare = Image.open('Tests/images/high_ascii_chars.png') + assert_image_equal(image, compare) + +def test_high_characters(): + message = "".join([chr(i+1) for i in range(140,232)]) + _test_high_characters(message) + # accept bytes instances in Py3. + if bytes is not str: + _test_high_characters(message.encode('latin1')) + diff --git a/Tests/test_image_convert.py b/Tests/test_image_convert.py index fd3d39bc5..4d40e43b2 100644 --- a/Tests/test_image_convert.py +++ b/Tests/test_image_convert.py @@ -46,5 +46,67 @@ def test_16bit_workaround(): im = Image.open('Tests/images/16bit.cropped.tif') _test_float_conversion(im.convert('I')) +def test_rgba_p(): + im = lena('RGBA') + im.putalpha(lena('L')) + + converted = im.convert('P') + comparable = converted.convert('RGBA') + + assert_image_similar(im, comparable, 20) + +def test_trns_p(): + im = lena('P') + im.info['transparency']=0 + + f = tempfile('temp.png') + + l = im.convert('L') + assert_equal(l.info['transparency'], 0) # undone + assert_no_exception(lambda: l.save(f)) + + + rgb = im.convert('RGB') + assert_equal(rgb.info['transparency'], (0,0,0)) # undone + assert_no_exception(lambda: rgb.save(f)) + +def test_trns_l(): + im = lena('L') + im.info['transparency'] = 128 + + f = tempfile('temp.png') + + rgb = im.convert('RGB') + assert_equal(rgb.info['transparency'], (128,128,128)) # undone + assert_no_exception(lambda: rgb.save(f)) + + p = im.convert('P') + assert_true('transparency' in p.info) + assert_no_exception(lambda: p.save(f)) + + p = assert_warning(UserWarning, + lambda: im.convert('P', palette = Image.ADAPTIVE)) + assert_false('transparency' in p.info) + assert_no_exception(lambda: p.save(f)) +def test_trns_RGB(): + im = lena('RGB') + im.info['transparency'] = im.getpixel((0,0)) + + f = tempfile('temp.png') + + l = im.convert('L') + assert_equal(l.info['transparency'], l.getpixel((0,0))) # undone + assert_no_exception(lambda: l.save(f)) + + p = im.convert('P') + assert_true('transparency' in p.info) + assert_no_exception(lambda: p.save(f)) + + p = assert_warning(UserWarning, + lambda: im.convert('P', palette = Image.ADAPTIVE)) + assert_false('transparency' in p.info) + assert_no_exception(lambda: p.save(f)) + + diff --git a/Tests/test_image_point.py b/Tests/test_image_point.py index c70556f6a..34233f80e 100644 --- a/Tests/test_image_point.py +++ b/Tests/test_image_point.py @@ -2,6 +2,11 @@ from tester import * from PIL import Image +if hasattr(sys, 'pypy_version_info'): + # This takes _forever_ on pypy. Open Bug, + # see https://github.com/python-imaging/Pillow/issues/484 + skip() + def test_sanity(): im = lena() diff --git a/Tests/test_image_quantize.py b/Tests/test_image_quantize.py index 70b5eb503..dbf68a25e 100644 --- a/Tests/test_image_quantize.py +++ b/Tests/test_image_quantize.py @@ -19,4 +19,9 @@ def test_octree_quantize(): im = im.quantize(100, Image.FASTOCTREE) assert_image(im, "P", im.size) - assert len(im.getcolors()) == 100 \ No newline at end of file + assert len(im.getcolors()) == 100 + +def test_rgba_quantize(): + im = lena('RGBA') + assert_no_exception(lambda: im.quantize()) + assert_exception(Exception, lambda: im.quantize(method=0)) diff --git a/Tests/test_imagecms.py b/Tests/test_imagecms.py index d18132598..dcb445c9f 100644 --- a/Tests/test_imagecms.py +++ b/Tests/test_imagecms.py @@ -3,6 +3,7 @@ from tester import * from PIL import Image try: from PIL import ImageCms + ImageCms.core.profile_open except ImportError: skip() diff --git a/Tests/test_imagecolor.py b/Tests/test_imagecolor.py index 23f21744a..c67c20255 100644 --- a/Tests/test_imagecolor.py +++ b/Tests/test_imagecolor.py @@ -26,9 +26,29 @@ for color in list(ImageColor.colormap.keys()): assert_equal((0, 0, 0), ImageColor.getcolor("black", "RGB")) assert_equal((255, 255, 255), ImageColor.getcolor("white", "RGB")) +assert_equal((0, 255, 115), ImageColor.getcolor("rgba(0, 255, 115, 33)", "RGB")) +Image.new("RGB", (1, 1), "white") + +assert_equal((0, 0, 0, 255), ImageColor.getcolor("black", "RGBA")) +assert_equal((255, 255, 255, 255), ImageColor.getcolor("white", "RGBA")) +assert_equal((0, 255, 115, 33), ImageColor.getcolor("rgba(0, 255, 115, 33)", "RGBA")) +Image.new("RGBA", (1, 1), "white") assert_equal(0, ImageColor.getcolor("black", "L")) assert_equal(255, ImageColor.getcolor("white", "L")) +assert_equal(162, ImageColor.getcolor("rgba(0, 255, 115, 33)", "L")) +Image.new("L", (1, 1), "white") assert_equal(0, ImageColor.getcolor("black", "1")) assert_equal(255, ImageColor.getcolor("white", "1")) +# The following test is wrong, but is current behavior +# The correct result should be 255 due to the mode 1 +assert_equal(162, ImageColor.getcolor("rgba(0, 255, 115, 33)", "1")) +# Correct behavior +# assert_equal(255, ImageColor.getcolor("rgba(0, 255, 115, 33)", "1")) +Image.new("1", (1, 1), "white") + +assert_equal((0, 255), ImageColor.getcolor("black", "LA")) +assert_equal((255, 255), ImageColor.getcolor("white", "LA")) +assert_equal((162, 33), ImageColor.getcolor("rgba(0, 255, 115, 33)", "LA")) +Image.new("LA", (1, 1), "white") diff --git a/Tests/test_imagefile.py b/Tests/test_imagefile.py index d602935e7..adf282b03 100644 --- a/Tests/test_imagefile.py +++ b/Tests/test_imagefile.py @@ -2,6 +2,7 @@ from tester import * from PIL import Image from PIL import ImageFile +from PIL import EpsImagePlugin codecs = dir(Image.core) @@ -46,8 +47,9 @@ def test_parser(): assert_image_equal(*roundtrip("TGA")) assert_image_equal(*roundtrip("PCX")) - im1, im2 = roundtrip("EPS") - assert_image_similar(im1, im2.convert('L'),20) # EPS comes back in RGB + if EpsImagePlugin.has_ghostscript(): + im1, im2 = roundtrip("EPS") + assert_image_similar(im1, im2.convert('L'),20) # EPS comes back in RGB if "jpeg_encoder" in codecs: im1, im2 = roundtrip("JPEG") # lossy compression @@ -57,6 +59,13 @@ def test_parser(): # https://github.com/python-imaging/Pillow/issues/78 #assert_exception(IOError, lambda: roundtrip("PDF")) +def test_ico(): + with open('Tests/images/python.ico', 'rb') as f: + data = f.read() + p = ImageFile.Parser() + p.feed(data) + assert_equal((48,48), p.image.size) + def test_safeblock(): im1 = lena() diff --git a/Tests/test_imagetk.py b/Tests/test_imagetk.py index 5c39c9283..b30971e8f 100644 --- a/Tests/test_imagetk.py +++ b/Tests/test_imagetk.py @@ -3,7 +3,7 @@ from tester import * from PIL import Image try: from PIL import ImageTk -except ImportError as v: +except (OSError, ImportError) as v: skip(v) success() diff --git a/Tests/tester.py b/Tests/tester.py index 2c6fa071c..5900a7f3a 100644 --- a/Tests/tester.py +++ b/Tests/tester.py @@ -6,15 +6,13 @@ warnings.simplefilter('default') # temporarily turn off resource warnings that warn about unclosed # files in the test scripts. try: - warnings.filterwarnings("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning) except NameError: - # we expect a NameError on py2.x, since it doesn't have ResourceWarnings. - pass - - + # we expect a NameError on py2.x, since it doesn't have ResourceWarnings. + pass import sys -py3 = (sys.version_info >= (3,0)) +py3 = (sys.version_info >= (3, 0)) # some test helpers @@ -22,6 +20,7 @@ _target = None _tempfiles = [] _logfile = None + def success(): import sys success.count += 1 @@ -29,8 +28,10 @@ def success(): print(sys.argv[0], success.count, failure.count, file=_logfile) return True + def failure(msg=None, frame=None): - import sys, linecache + import sys + import linecache failure.count += 1 if _target: if frame is None: @@ -49,6 +50,7 @@ def failure(msg=None, frame=None): success.count = failure.count = 0 + # predicates def assert_true(v, msg=None): @@ -57,35 +59,39 @@ def assert_true(v, msg=None): else: failure(msg or "got %r, expected true value" % v) + def assert_false(v, msg=None): if v: failure(msg or "got %r, expected false value" % v) else: success() + def assert_equal(a, b, msg=None): if a == b: success() else: failure(msg or "got %r, expected %r" % (a, b)) + def assert_almost_equal(a, b, msg=None, eps=1e-6): if abs(a-b) < eps: success() else: failure(msg or "got %r, expected %r" % (a, b)) + def assert_deep_equal(a, b, msg=None): try: if len(a) == len(b): - if all([x==y for x,y in zip(a,b)]): + if all([x == y for x, y in zip(a, b)]): success() else: - failure(msg or "got %s, expected %s" % (a,b)) + failure(msg or "got %s, expected %s" % (a, b)) else: - failure(msg or "got length %s, expected %s" % (len(a), len(b))) + failure(msg or "got length %s, expected %s" % (len(a), len(b))) except: - assert_equal(a,b,msg) + assert_equal(a, b, msg) def assert_match(v, pattern, msg=None): @@ -95,8 +101,10 @@ def assert_match(v, pattern, msg=None): else: failure(msg or "got %r, doesn't match pattern %r" % (v, pattern)) + def assert_exception(exc_class, func): - import sys, traceback + import sys + import traceback try: func() except exc_class: @@ -108,8 +116,10 @@ def assert_exception(exc_class, func): else: failure("expected %r exception, got no exception" % exc_class.__name__) + def assert_no_exception(func): - import sys, traceback + import sys + import traceback try: func() except: @@ -118,12 +128,15 @@ def assert_no_exception(func): else: success() + def assert_warning(warn_class, func): # note: this assert calls func three times! import warnings - def warn_error(message, category, **options): + + def warn_error(message, category=UserWarning, **options): raise category(message) - def warn_ignore(message, category, **options): + + def warn_ignore(message, category=UserWarning, **options): pass warn = warnings.warn result = None @@ -134,22 +147,25 @@ def assert_warning(warn_class, func): warnings.warn = warn_error assert_exception(warn_class, func) finally: - warnings.warn = warn # restore + warnings.warn = warn # restore return result # helpers from io import BytesIO + def fromstring(data): from PIL import Image return Image.open(BytesIO(data)) + def tostring(im, format, **options): out = BytesIO() im.save(out, format, **options) return out.getvalue() + def lena(mode="RGB", cache={}): from PIL import Image im = cache.get(mode) @@ -165,6 +181,7 @@ def lena(mode="RGB", cache={}): cache[mode] = im return im + def assert_image(im, mode, size, msg=None): if mode is not None and im.mode != mode: failure(msg or "got mode %r, expected %r" % (im.mode, mode)) @@ -173,6 +190,7 @@ def assert_image(im, mode, size, msg=None): else: success() + def assert_image_equal(a, b, msg=None): if a.mode != b.mode: failure(msg or "got mode %r, expected %r" % (a.mode, b.mode)) @@ -184,6 +202,7 @@ def assert_image_equal(a, b, msg=None): else: success() + def assert_image_similar(a, b, epsilon, msg=None): epsilon = float(epsilon) if a.mode != b.mode: @@ -193,19 +212,25 @@ def assert_image_similar(a, b, epsilon, msg=None): diff = 0 try: ord(b'0') - for abyte,bbyte in zip(a.tobytes(),b.tobytes()): + for abyte, bbyte in zip(a.tobytes(), b.tobytes()): diff += abs(ord(abyte)-ord(bbyte)) except: - for abyte,bbyte in zip(a.tobytes(),b.tobytes()): + for abyte, bbyte in zip(a.tobytes(), b.tobytes()): diff += abs(abyte-bbyte) ave_diff = float(diff)/(a.size[0]*a.size[1]) if epsilon < ave_diff: - return failure(msg or "average pixel value difference %.4f > epsilon %.4f" %(ave_diff, epsilon)) + return failure( + msg or "average pixel value difference %.4f > epsilon %.4f" % ( + ave_diff, epsilon)) else: return success() + def tempfile(template, *extra): - import os, os.path, sys, tempfile + import os + import os.path + import sys + import tempfile files = [] root = os.path.join(tempfile.gettempdir(), 'pillow-tests') try: @@ -222,18 +247,20 @@ def tempfile(template, *extra): _tempfiles.extend(files) return files[0] + # test runner def run(): global _target, _tests, run - import sys, traceback + import sys + import traceback _target = sys.modules["__main__"] - run = None # no need to run twice + run = None # no need to run twice tests = [] for name, value in list(vars(_target).items()): if name[:5] == "test_" and type(value) is type(success): tests.append((value.__code__.co_firstlineno, name, value)) - tests.sort() # sort by line + tests.sort() # sort by line for lineno, name, func in tests: try: _tests = [] @@ -251,47 +278,56 @@ def run(): sys.argv[0], lineno, v)) failure.count += 1 + def yield_test(function, *args): # collect delayed/generated tests _tests.append((function, args)) + def skip(msg=None): import os print("skip") - os._exit(0) # don't run exit handlers + os._exit(0) # don't run exit handlers + + +def ignore(pattern): + """Tells the driver to ignore messages matching the pattern, for the + duration of the current test.""" + print('ignore: %s' % pattern) + def _setup(): global _logfile + + import sys + if "--coverage" in sys.argv: + import coverage + cov = coverage.coverage(auto_data=True, include="PIL/*") + cov.start() + def report(): if run: run() if success.count and not failure.count: print("ok") # only clean out tempfiles if test passed - import os, os.path, tempfile + import os + import os.path + import tempfile for file in _tempfiles: try: os.remove(file) except OSError: - pass # report? + pass # report? temp_root = os.path.join(tempfile.gettempdir(), 'pillow-tests') try: os.rmdir(temp_root) except OSError: pass - if "--coverage" in sys.argv: - import coverage - coverage.stop() - # The coverage module messes up when used from inside an - # atexit handler. Do an explicit save to make sure that - # we actually flush the coverage cache. - coverage.the_coverage.save() - import atexit, sys + import atexit atexit.register(report) - if "--coverage" in sys.argv: - import coverage - coverage.start() + if "--log" in sys.argv: _logfile = open("test.log", "a") diff --git a/_imaging.c b/_imaging.c index 078961da4..c47868b81 100644 --- a/_imaging.c +++ b/_imaging.c @@ -71,7 +71,7 @@ * See the README file for information on usage and redistribution. */ -#define PILLOW_VERSION "2.3.0" +#define PILLOW_VERSION "2.4.0" #include "Python.h" @@ -846,7 +846,7 @@ _crop(ImagingObject* self, PyObject* args) } static PyObject* -_expand(ImagingObject* self, PyObject* args) +_expand_image(ImagingObject* self, PyObject* args) { int x, y; int mode = 0; @@ -2239,30 +2239,66 @@ textwidth(ImagingFontObject* self, const unsigned char* text) return xsize; } +void _font_text_asBytes(PyObject* encoded_string, unsigned char** text){ + PyObject* bytes = NULL; + + *text = NULL; + + if (PyUnicode_CheckExact(encoded_string)){ + bytes = PyUnicode_AsLatin1String(encoded_string); + } else if (PyBytes_Check(encoded_string)) { + bytes = encoded_string; + } + if (bytes) { + *text = (unsigned char*)PyBytes_AsString(bytes); + return; + } + +#if PY_VERSION_HEX < 0x03000000 + /* likely case here is py2.x with an ordinary string. + but this isn't defined in Py3.x */ + if (PyString_Check(encoded_string)) { + *text = (unsigned char *)PyString_AsString(encoded_string); + } +#endif +} + + static PyObject* _font_getmask(ImagingFontObject* self, PyObject* args) { Imaging im; Imaging bitmap; int x, b; + int i=0; int status; Glyph* glyph; + PyObject* encoded_string; + unsigned char* text; char* mode = ""; - if (!PyArg_ParseTuple(args, "s|s:getmask", &text, &mode)) + + if (!PyArg_ParseTuple(args, "O|s:getmask", &encoded_string, &mode)){ return NULL; + } + + _font_text_asBytes(encoded_string, &text); + if (!text) { + return NULL; + } im = ImagingNew(self->bitmap->mode, textwidth(self, text), self->ysize); - if (!im) + if (!im) { return NULL; + } b = 0; (void) ImagingFill(im, &b); b = self->baseline; - for (x = 0; *text; text++) { - glyph = &self->glyphs[*text]; + for (x = 0; text[i]; i++) { + glyph = &self->glyphs[text[i]]; bitmap = ImagingCrop( self->bitmap, glyph->sx0, glyph->sy0, glyph->sx1, glyph->sy1 @@ -2279,7 +2315,6 @@ _font_getmask(ImagingFontObject* self, PyObject* args) x = x + glyph->dx; b = b + glyph->dy; } - return PyImagingNew(im); failed: @@ -2291,10 +2326,17 @@ static PyObject* _font_getsize(ImagingFontObject* self, PyObject* args) { unsigned char* text; - if (!PyArg_ParseTuple(args, "s:getsize", &text)) + PyObject* encoded_string; + + if (!PyArg_ParseTuple(args, "O:getsize", &encoded_string)) return NULL; - return Py_BuildValue("ii", textwidth(self, text), self->ysize); + _font_text_asBytes(encoded_string, &text); + if (!text) { + return NULL; + } + + return Py_BuildValue("ii", textwidth(self, text), self->ysize); } static struct PyMethodDef _font_methods[] = { @@ -2954,7 +2996,7 @@ static struct PyMethodDef methods[] = { {"crackcode", (PyCFunction)_crackcode, 1}, #endif {"crop", (PyCFunction)_crop, 1}, - {"expand", (PyCFunction)_expand, 1}, + {"expand", (PyCFunction)_expand_image, 1}, {"filter", (PyCFunction)_filter, 1}, {"histogram", (PyCFunction)_histogram, 1}, #ifdef WITH_MODEFILTER @@ -3283,6 +3325,7 @@ extern PyObject* PyImaging_FliDecoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_GifDecoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_HexDecoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_JpegDecoderNew(PyObject* self, PyObject* args); +extern PyObject* PyImaging_Jpeg2KDecoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_TiffLzwDecoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_LibTiffDecoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_MspDecoderNew(PyObject* self, PyObject* args); @@ -3299,6 +3342,7 @@ extern PyObject* PyImaging_ZipDecoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_EpsEncoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_GifEncoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_JpegEncoderNew(PyObject* self, PyObject* args); +extern PyObject* PyImaging_Jpeg2KEncoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_PcxEncoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_RawEncoderNew(PyObject* self, PyObject* args); extern PyObject* PyImaging_XbmEncoderNew(PyObject* self, PyObject* args); @@ -3351,6 +3395,10 @@ static PyMethodDef functions[] = { #ifdef HAVE_LIBJPEG {"jpeg_decoder", (PyCFunction)PyImaging_JpegDecoderNew, 1}, {"jpeg_encoder", (PyCFunction)PyImaging_JpegEncoderNew, 1}, +#endif +#ifdef HAVE_OPENJPEG + {"jpeg2k_decoder", (PyCFunction)PyImaging_Jpeg2KDecoderNew, 1}, + {"jpeg2k_encoder", (PyCFunction)PyImaging_Jpeg2KEncoderNew, 1}, #endif {"tiff_lzw_decoder", (PyCFunction)PyImaging_TiffLzwDecoderNew, 1}, #ifdef HAVE_LIBTIFF @@ -3455,6 +3503,13 @@ setup_module(PyObject* m) { } #endif +#ifdef HAVE_OPENJPEG + { + extern const char *ImagingJpeg2KVersion(void); + PyDict_SetItemString(d, "jp2klib_version", PyUnicode_FromString(ImagingJpeg2KVersion())); + } +#endif + #ifdef HAVE_LIBZ /* zip encoding strategies */ PyModule_AddIntConstant(m, "DEFAULT_STRATEGY", Z_DEFAULT_STRATEGY); diff --git a/decode.c b/decode.c index f3ac60e51..77038cc2c 100644 --- a/decode.c +++ b/decode.c @@ -52,6 +52,7 @@ typedef struct { struct ImagingCodecStateInstance state; Imaging im; PyObject* lock; + int handles_eof; } ImagingDecoderObject; static PyTypeObject ImagingDecoderType; @@ -93,6 +94,9 @@ PyImaging_DecoderNew(int contextsize) /* Initialize the cleanup function pointer */ decoder->cleanup = NULL; + /* Most decoders don't want to handle EOF themselves */ + decoder->handles_eof = 0; + return decoder; } @@ -194,6 +198,12 @@ _setimage(ImagingDecoderObject* decoder, PyObject* args) return Py_None; } +static PyObject * +_get_handles_eof(ImagingDecoderObject *decoder) +{ + return PyBool_FromLong(decoder->handles_eof); +} + static struct PyMethodDef methods[] = { {"decode", (PyCFunction)_decode, 1}, {"cleanup", (PyCFunction)_decode_cleanup, 1}, @@ -201,6 +211,13 @@ static struct PyMethodDef methods[] = { {NULL, NULL} /* sentinel */ }; +static struct PyGetSetDef getseters[] = { + {"handles_eof", (getter)_get_handles_eof, NULL, + "True if this decoder expects to handle EOF itself.", + NULL}, + {NULL, NULL, NULL, NULL, NULL} /* sentinel */ +}; + static PyTypeObject ImagingDecoderType = { PyVarObject_HEAD_INIT(NULL, 0) "ImagingDecoder", /*tp_name*/ @@ -232,7 +249,7 @@ static PyTypeObject ImagingDecoderType = { 0, /*tp_iternext*/ methods, /*tp_methods*/ 0, /*tp_members*/ - 0, /*tp_getset*/ + getseters, /*tp_getset*/ }; /* -------------------------------------------------------------------- */ @@ -762,3 +779,55 @@ PyImaging_JpegDecoderNew(PyObject* self, PyObject* args) return (PyObject*) decoder; } #endif + +/* -------------------------------------------------------------------- */ +/* JPEG 2000 */ +/* -------------------------------------------------------------------- */ + +#ifdef HAVE_OPENJPEG + +#include "Jpeg2K.h" + +PyObject* +PyImaging_Jpeg2KDecoderNew(PyObject* self, PyObject* args) +{ + ImagingDecoderObject* decoder; + JPEG2KDECODESTATE *context; + + char* mode; + char* format; + OPJ_CODEC_FORMAT codec_format; + int reduce = 0; + int layers = 0; + int fd = -1; + if (!PyArg_ParseTuple(args, "ss|iii", &mode, &format, + &reduce, &layers, &fd)) + return NULL; + + if (strcmp(format, "j2k") == 0) + codec_format = OPJ_CODEC_J2K; + else if (strcmp(format, "jpt") == 0) + codec_format = OPJ_CODEC_JPT; + else if (strcmp(format, "jp2") == 0) + codec_format = OPJ_CODEC_JP2; + else + return NULL; + + decoder = PyImaging_DecoderNew(sizeof(JPEG2KDECODESTATE)); + if (decoder == NULL) + return NULL; + + decoder->handles_eof = 1; + decoder->decode = ImagingJpeg2KDecode; + decoder->cleanup = ImagingJpeg2KDecodeCleanup; + + context = (JPEG2KDECODESTATE *)decoder->state.context; + + context->fd = fd; + context->format = codec_format; + context->reduce = reduce; + context->layers = layers; + + return (PyObject*) decoder; +} +#endif /* HAVE_OPENJPEG */ diff --git a/depends/install_openjpeg.sh b/depends/install_openjpeg.sh new file mode 100755 index 000000000..bd6b83e3b --- /dev/null +++ b/depends/install_openjpeg.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# install openjpeg + + +if [ ! -f openjpeg-2.0.0.tar.gz ]; then + wget 'https://openjpeg.googlecode.com/files/openjpeg-2.0.0.tar.gz' +fi + +rm -r openjpeg-2.0.0 +tar -xvzf openjpeg-2.0.0.tar.gz + + +pushd openjpeg-2.0.0 + +cmake -DCMAKE_INSTALL_PREFIX=/usr . && make && sudo make install + +popd + diff --git a/depends/install_webp.sh b/depends/install_webp.sh new file mode 100755 index 000000000..5f5963712 --- /dev/null +++ b/depends/install_webp.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# install webp + + +if [ ! -f libwebp-0.4.0.tar.gz ]; then + wget 'https://webp.googlecode.com/files/libwebp-0.4.0.tar.gz' +fi + +rm -r libwebp-0.4.0 +tar -xvzf libwebp-0.4.0.tar.gz + + +pushd libwebp-0.4.0 + +./configure --prefix=/usr --enable-libwebpmux --enable-libwebpdemux && make && sudo make install + +popd + diff --git a/docs/handbook/image-file-formats.rst b/docs/handbook/image-file-formats.rst index 21cd615fc..fddc134e6 100644 --- a/docs/handbook/image-file-formats.rst +++ b/docs/handbook/image-file-formats.rst @@ -153,6 +153,94 @@ The :py:meth:`~PIL.Image.Image.save` method supports the following options: before building the Python Imaging Library. See the distribution README for details. +JPEG 2000 +^^^^^^^^^ + +.. versionadded:: 2.4.0 + +PIL reads and writes JPEG 2000 files containing ``L``, ``LA``, ``RGB`` or +``RGBA`` data. It can also read files containing ``YCbCr`` data, which it +converts on read into ``RGB`` or ``RGBA`` depending on whether or not there is +an alpha channel. PIL supports JPEG 2000 raw codestreams (``.j2k`` files), as +well as boxed JPEG 2000 files (``.j2p`` or ``.jpx`` files). PIL does *not* +support files whose components have different sampling frequencies. + +When loading, if you set the ``mode`` on the image prior to the +:py:meth:`~PIL.Image.Image.load` method being invoked, you can ask PIL to +convert the image to either ``RGB`` or ``RGBA`` rather than choosing for +itself. It is also possible to set ``reduce`` to the number of resolutions to +discard (each one reduces the size of the resulting image by a factor of 2), +and ``layers`` to specify the number of quality layers to load. + +The :py:meth:`~PIL.Image.Image.save` method supports the following options: + +**offset** + The image offset, as a tuple of integers, e.g. (16, 16) + +**tile_offset** + The tile offset, again as a 2-tuple of integers. + +**tile_size** + The tile size as a 2-tuple. If not specified, or if set to None, the + image will be saved without tiling. + +**quality_mode** + Either `"rates"` or `"dB"` depending on the units you want to use to + specify image quality. + +**quality_layers** + A sequence of numbers, each of which represents either an approximate size + reduction (if quality mode is `"rates"`) or a signal to noise ratio value + in decibels. If not specified, defaults to a single layer of full quality. + +**num_resolutions** + The number of different image resolutions to be stored (which corresponds + to the number of Discrete Wavelet Transform decompositions plus one). + +**codeblock_size** + The code-block size as a 2-tuple. Minimum size is 4 x 4, maximum is 1024 x + 1024, with the additional restriction that no code-block may have more + than 4096 coefficients (i.e. the product of the two numbers must be no + greater than 4096). + +**precinct_size** + The precinct size as a 2-tuple. Must be a power of two along both axes, + and must be greater than the code-block size. + +**irreversible** + If ``True``, use the lossy Irreversible Color Transformation + followed by DWT 9-7. Defaults to ``False``, which means to use the + Reversible Color Transformation with DWT 5-3. + +**progression** + Controls the progression order; must be one of ``"LRCP"``, ``"RLCP"``, + ``"RPCL"``, ``"PCRL"``, ``"CPRL"``. The letters stand for Component, + Position, Resolution and Layer respectively and control the order of + encoding, the idea being that e.g. an image encoded using LRCP mode can + have its quality layers decoded as they arrive at the decoder, while one + encoded using RLCP mode will have increasing resolutions decoded as they + arrive, and so on. + +**cinema_mode** + Set the encoder to produce output compliant with the digital cinema + specifications. The options here are ``"no"`` (the default), + ``"cinema2k-24"`` for 24fps 2K, ``"cinema2k-48"`` for 48fps 2K, and + ``"cinema4k-24"`` for 24fps 4K. Note that for compliant 2K files, + *at least one* of your image dimensions must match 2048 x 1080, while + for compliant 4K files, *at least one* of the dimensions must match + 4096 x 2160. + +.. note:: + + To enable JPEG 2000 support, you need to build and install the OpenJPEG + library, version 2.0.0 or higher, before building the Python Imaging + Library. + + Windows users can install the OpenJPEG binaries available on the + OpenJPEG website, but must add them to their PATH in order to use PIL (if + you fail to do this, you will get errors about not being able to load the + ``_imaging`` DLL). + MSP ^^^ @@ -437,6 +525,25 @@ ICO ICO is used to store icons on Windows. The largest available icon is read. +ICNS +^^^^ + +PIL reads Mac OS X ``.icns`` files. By default, the largest available icon is +read, though you can override this by setting the :py:attr:`~PIL.Image.Image.size` +property before calling :py:meth:`~PIL.Image.Image.load`. The +:py:meth:`~PIL.Image.Image.open` method sets the following +:py:attr:`~PIL.Image.Image.info` property: + +**sizes** + A list of supported sizes found in this icon file; these are a + 3-tuple, ``(width, height, scale)``, where ``scale`` is 2 for a retina + icon and 1 for a standard icon. You *are* permitted to use this 3-tuple + format for the :py:attr:`~PIL.Image.Image.size` property if you set it + before calling :py:meth:`~PIL.Image.Image.load`; after loading, the size + will be reset to a 2-tuple containing pixel dimensions (so, e.g. if you + ask for ``(512, 512, 2)``, the final value of + :py:attr:`~PIL.Image.Image.size` will be ``(1024, 1024)``). + IMT ^^^ diff --git a/docs/handbook/tutorial.rst b/docs/handbook/tutorial.rst index d7bb98386..9ce50da7d 100644 --- a/docs/handbook/tutorial.rst +++ b/docs/handbook/tutorial.rst @@ -105,7 +105,6 @@ Create JPEG thumbnails except IOError: print("cannot create thumbnail for", infile) - It is important to note that the library doesn’t decode or load the raster data unless it really has to. When you open a file, the file header is read to determine the file format and extract things like mode, size, and other diff --git a/docs/index.rst b/docs/index.rst index 52a054e22..a59cda115 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ -Pillow: a modern fork of PIL -============================ +Pillow +====== -Pillow is the "friendly" PIL fork by Alex Clark and Contributors. PIL is the +Pillow is the 'friendly' PIL fork by Alex Clark and Contributors. PIL is the Python Imaging Library by Fredrik Lundh and Contributors. .. image:: https://travis-ci.org/python-imaging/Pillow.png @@ -15,15 +15,12 @@ Python Imaging Library by Fredrik Lundh and Contributors. :target: https://pypi.python.org/pypi/Pillow/ :alt: Number of PyPI downloads -To start using Pillow, read the :doc:`installation +To start using Pillow, please read the :doc:`installation instructions `. -If you can't find the information you need, try the old `PIL Handbook`_, but be -aware that it was last updated for PIL 1.1.5. You can download archives and old -versions from `PyPI `_. You can get the -source and contribute at https://github.com/python-imaging/Pillow. - -.. _PIL Handbook: http://effbot.org/imagingbook/pil-index.htm +You can get the source and contribute at +https://github.com/python-imaging/Pillow. You can download archives +and old versions from `PyPI `_. .. toctree:: :maxdepth: 2 diff --git a/docs/installation.rst b/docs/installation.rst index 1eae2208c..715a6ac30 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -66,10 +66,15 @@ Many of Pillow's features require external libraries: * **libwebp** provides the Webp format. * Pillow has been tested with version **0.1.3**, which does not read - transparent webp files. Version **0.3.0** supports transparency. + transparent webp files. Versions **0.3.0** and **0.4.0** support + transparency. * **tcl/tk** provides support for tkinter bitmap and photo images. +* **openjpeg** provides JPEG 2000 functionality. + + * Pillow has been tested with openjpeg **2.0.0**. + If the prerequisites are installed in the standard library locations for your machine (e.g. :file:`/usr` or :file:`/usr/local`), no additional configuration should be required. If they are installed in a non-standard location, you may @@ -109,13 +114,13 @@ In Fedora, the command is:: Prerequisites are installed on **Ubuntu 10.04 LTS** with:: $ sudo apt-get install libtiff4-dev libjpeg62-dev zlib1g-dev \ - libfreetype6-dev tcl8.5-dev tk8.5-dev + libfreetype6-dev tcl8.5-dev tk8.5-dev python-tk Prerequisites are installed with on **Ubuntu 12.04 LTS** or **Raspian Wheezy 7.0** with:: $ sudo apt-get install libtiff4-dev libjpeg8-dev zlib1g-dev \ - libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev + libfreetype6-dev liblcms2-dev libwebp-dev tcl8.5-dev tk8.5-dev python-tk Prerequisites are installed on **Fedora 20** with:: @@ -172,6 +177,15 @@ Python Wheels $ pip install --use-wheel Pillow +If the above does not work, it's likely because we haven't uploaded a +wheel for the latest version of Pillow. In that case, try pinning it +to a specific version: + +:: + + $ pip install --use-wheel Pillow==2.3.0 + + Platform support ---------------- @@ -216,4 +230,6 @@ current versions of Linux, OS X, and Windows. +----------------------------------+-------------+------------------------------+------------------------------+-----------------------+ | Windows 8 Pro |Yes | 2.6,2.7,3.2,3.3,3.4a3 | 2.2.0 |x86,x86-64 | +----------------------------------+-------------+------------------------------+------------------------------+-----------------------+ +| Windows 8.1 Pro |Yes | 2.6,2.7,3.2,3.3,3.4 | 2.3.0, 2.4.0 |x86,x86-64 | ++----------------------------------+-------------+------------------------------+------------------------------+-----------------------+ diff --git a/docs/plugins.rst b/docs/plugins.rst index b92b500c1..001cee949 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -153,6 +153,14 @@ Plugin reference :undoc-members: :show-inheritance: +:mod:`Jpeg2KImagePlugin` Module +----------------------------- + +.. automodule:: PIL.Jpeg2KImagePlugin + :members: + :undoc-members: + :show-inheritance: + :mod:`McIdasImagePlugin` Module ------------------------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index d825a5fcc..7611ea463 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # requirements for working on docs # install pillow from master if you're into that, but RtD needs this -pillow>=2.2.1 +pillow>=2.4.0 Jinja2==2.7.1 MarkupSafe==0.18 diff --git a/encode.c b/encode.c index 8be44d8ec..d403ab072 100644 --- a/encode.c +++ b/encode.c @@ -33,13 +33,14 @@ #endif /* -------------------------------------------------------------------- */ -/* Common */ +/* Common */ /* -------------------------------------------------------------------- */ typedef struct { PyObject_HEAD int (*encode)(Imaging im, ImagingCodecState state, - UINT8* buffer, int bytes); + UINT8* buffer, int bytes); + int (*cleanup)(ImagingCodecState state); struct ImagingCodecStateInstance state; Imaging im; PyObject* lock; @@ -58,25 +59,28 @@ PyImaging_EncoderNew(int contextsize) encoder = PyObject_New(ImagingEncoderObject, &ImagingEncoderType); if (encoder == NULL) - return NULL; + return NULL; /* Clear the encoder state */ memset(&encoder->state, 0, sizeof(encoder->state)); /* Allocate encoder context */ if (contextsize > 0) { - context = (void*) calloc(1, contextsize); - if (!context) { - Py_DECREF(encoder); - (void) PyErr_NoMemory(); - return NULL; - } + context = (void*) calloc(1, contextsize); + if (!context) { + Py_DECREF(encoder); + (void) PyErr_NoMemory(); + return NULL; + } } else - context = 0; + context = 0; /* Initialize encoder context */ encoder->state.context = context; + /* Most encoders don't need this */ + encoder->cleanup = NULL; + /* Target image */ encoder->lock = NULL; encoder->im = NULL; @@ -87,6 +91,8 @@ PyImaging_EncoderNew(int contextsize) static void _dealloc(ImagingEncoderObject* encoder) { + if (encoder->cleanup) + encoder->cleanup(&encoder->state); free(encoder->state.buffer); free(encoder->state.context); Py_XDECREF(encoder->lock); @@ -105,14 +111,14 @@ _encode(ImagingEncoderObject* encoder, PyObject* args) int bufsize = 16384; if (!PyArg_ParseTuple(args, "|i", &bufsize)) - return NULL; + return NULL; buf = PyBytes_FromStringAndSize(NULL, bufsize); if (!buf) - return NULL; + return NULL; status = encoder->encode(encoder->im, &encoder->state, - (UINT8*) PyBytes_AsString(buf), bufsize); + (UINT8*) PyBytes_AsString(buf), bufsize); /* adjust string length to avoid slicing in encoder */ if (_PyBytes_Resize(&buf, (status > 0) ? status : 0) < 0) @@ -138,28 +144,28 @@ _encode_to_file(ImagingEncoderObject* encoder, PyObject* args) int bufsize = 16384; if (!PyArg_ParseTuple(args, "i|i", &fh, &bufsize)) - return NULL; + return NULL; /* Allocate an encoder buffer */ buf = (UINT8*) malloc(bufsize); if (!buf) - return PyErr_NoMemory(); + return PyErr_NoMemory(); ImagingSectionEnter(&cookie); do { - /* This replaces the inner loop in the ImageFile _save - function. */ + /* This replaces the inner loop in the ImageFile _save + function. */ - status = encoder->encode(encoder->im, &encoder->state, buf, bufsize); + status = encoder->encode(encoder->im, &encoder->state, buf, bufsize); - if (status > 0) - if (write(fh, buf, status) < 0) { + if (status > 0) + if (write(fh, buf, status) < 0) { ImagingSectionLeave(&cookie); - free(buf); - return PyErr_SetFromErrno(PyExc_IOError); - } + free(buf); + return PyErr_SetFromErrno(PyExc_IOError); + } } while (encoder->state.errcode == 0); @@ -186,39 +192,39 @@ _setimage(ImagingEncoderObject* encoder, PyObject* args) /* FIXME: should publish the ImagingType descriptor */ if (!PyArg_ParseTuple(args, "O|(iiii)", &op, &x0, &y0, &x1, &y1)) - return NULL; + return NULL; im = PyImaging_AsImaging(op); if (!im) - return NULL; + return NULL; encoder->im = im; state = &encoder->state; if (x0 == 0 && x1 == 0) { - state->xsize = im->xsize; - state->ysize = im->ysize; + state->xsize = im->xsize; + state->ysize = im->ysize; } else { - state->xoff = x0; - state->yoff = y0; - state->xsize = x1 - x0; - state->ysize = y1 - y0; + state->xoff = x0; + state->yoff = y0; + state->xsize = x1 - x0; + state->ysize = y1 - y0; } if (state->xsize <= 0 || - state->xsize + state->xoff > im->xsize || - state->ysize <= 0 || - state->ysize + state->yoff > im->ysize) { - PyErr_SetString(PyExc_SystemError, "tile cannot extend outside image"); - return NULL; + state->xsize + state->xoff > im->xsize || + state->ysize <= 0 || + state->ysize + state->yoff > im->ysize) { + PyErr_SetString(PyExc_SystemError, "tile cannot extend outside image"); + return NULL; } /* Allocate memory buffer (if bits field is set) */ if (state->bits > 0) { - state->bytes = (state->bits * state->xsize+7)/8; - state->buffer = (UINT8*) malloc(state->bytes); - if (!state->buffer) - return PyErr_NoMemory(); + state->bytes = (state->bits * state->xsize+7)/8; + state->buffer = (UINT8*) malloc(state->bytes); + if (!state->buffer) + return PyErr_NoMemory(); } /* Keep a reference to the image object, to make sure it doesn't @@ -239,13 +245,13 @@ static struct PyMethodDef methods[] = { }; static PyTypeObject ImagingEncoderType = { - PyVarObject_HEAD_INIT(NULL, 0) - "ImagingEncoder", /*tp_name*/ - sizeof(ImagingEncoderObject), /*tp_size*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ + PyVarObject_HEAD_INIT(NULL, 0) + "ImagingEncoder", /*tp_name*/ + sizeof(ImagingEncoderObject), /*tp_size*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ 0, /*tp_compare*/ @@ -283,9 +289,9 @@ get_packer(ImagingEncoderObject* encoder, const char* mode, pack = ImagingFindPacker(mode, rawmode, &bits); if (!pack) { - Py_DECREF(encoder); - PyErr_SetString(PyExc_SystemError, "unknown raw mode"); - return -1; + Py_DECREF(encoder); + PyErr_SetString(PyExc_SystemError, "unknown raw mode"); + return -1; } encoder->state.shuffle = pack; @@ -296,7 +302,7 @@ get_packer(ImagingEncoderObject* encoder, const char* mode, /* -------------------------------------------------------------------- */ -/* EPS */ +/* EPS */ /* -------------------------------------------------------------------- */ PyObject* @@ -306,7 +312,7 @@ PyImaging_EpsEncoderNew(PyObject* self, PyObject* args) encoder = PyImaging_EncoderNew(0); if (encoder == NULL) - return NULL; + return NULL; encoder->encode = ImagingEpsEncode; @@ -315,7 +321,7 @@ PyImaging_EpsEncoderNew(PyObject* self, PyObject* args) /* -------------------------------------------------------------------- */ -/* GIF */ +/* GIF */ /* -------------------------------------------------------------------- */ PyObject* @@ -328,14 +334,14 @@ PyImaging_GifEncoderNew(PyObject* self, PyObject* args) int bits = 8; int interlace = 0; if (!PyArg_ParseTuple(args, "ss|ii", &mode, &rawmode, &bits, &interlace)) - return NULL; + return NULL; encoder = PyImaging_EncoderNew(sizeof(GIFENCODERSTATE)); if (encoder == NULL) - return NULL; + return NULL; if (get_packer(encoder, mode, rawmode) < 0) - return NULL; + return NULL; encoder->encode = ImagingGifEncode; @@ -347,7 +353,7 @@ PyImaging_GifEncoderNew(PyObject* self, PyObject* args) /* -------------------------------------------------------------------- */ -/* PCX */ +/* PCX */ /* -------------------------------------------------------------------- */ PyObject* @@ -358,15 +364,19 @@ PyImaging_PcxEncoderNew(PyObject* self, PyObject* args) char *mode; char *rawmode; int bits = 8; - if (!PyArg_ParseTuple(args, "ss|ii", &mode, &rawmode, &bits)) - return NULL; + + if (!PyArg_ParseTuple(args, "ss|ii", &mode, &rawmode, &bits)) { + return NULL; + } encoder = PyImaging_EncoderNew(0); - if (encoder == NULL) - return NULL; + if (encoder == NULL) { + return NULL; + } - if (get_packer(encoder, mode, rawmode) < 0) - return NULL; + if (get_packer(encoder, mode, rawmode) < 0) { + return NULL; + } encoder->encode = ImagingPcxEncode; @@ -375,7 +385,7 @@ PyImaging_PcxEncoderNew(PyObject* self, PyObject* args) /* -------------------------------------------------------------------- */ -/* RAW */ +/* RAW */ /* -------------------------------------------------------------------- */ PyObject* @@ -389,14 +399,14 @@ PyImaging_RawEncoderNew(PyObject* self, PyObject* args) int ystep = 1; if (!PyArg_ParseTuple(args, "ss|ii", &mode, &rawmode, &stride, &ystep)) - return NULL; + return NULL; encoder = PyImaging_EncoderNew(0); if (encoder == NULL) - return NULL; + return NULL; if (get_packer(encoder, mode, rawmode) < 0) - return NULL; + return NULL; encoder->encode = ImagingRawEncode; @@ -408,7 +418,7 @@ PyImaging_RawEncoderNew(PyObject* self, PyObject* args) /* -------------------------------------------------------------------- */ -/* XBM */ +/* XBM */ /* -------------------------------------------------------------------- */ PyObject* @@ -418,10 +428,10 @@ PyImaging_XbmEncoderNew(PyObject* self, PyObject* args) encoder = PyImaging_EncoderNew(0); if (encoder == NULL) - return NULL; + return NULL; if (get_packer(encoder, "1", "1;R") < 0) - return NULL; + return NULL; encoder->encode = ImagingXbmEncode; @@ -430,7 +440,7 @@ PyImaging_XbmEncoderNew(PyObject* self, PyObject* args) /* -------------------------------------------------------------------- */ -/* ZIP */ +/* ZIP */ /* -------------------------------------------------------------------- */ #ifdef HAVE_LIBZ @@ -469,16 +479,16 @@ PyImaging_ZipEncoderNew(PyObject* self, PyObject* args) encoder = PyImaging_EncoderNew(sizeof(ZIPSTATE)); if (encoder == NULL) - return NULL; + return NULL; if (get_packer(encoder, mode, rawmode) < 0) - return NULL; + return NULL; encoder->encode = ImagingZipEncode; if (rawmode[0] == 'P') - /* disable filtering */ - ((ZIPSTATE*)encoder->state.context)->mode = ZIP_PNG_PALETTE; + /* disable filtering */ + ((ZIPSTATE*)encoder->state.context)->mode = ZIP_PNG_PALETTE; ((ZIPSTATE*)encoder->state.context)->optimize = optimize; ((ZIPSTATE*)encoder->state.context)->compress_level = compress_level; @@ -492,7 +502,7 @@ PyImaging_ZipEncoderNew(PyObject* self, PyObject* args) /* -------------------------------------------------------------------- */ -/* JPEG */ +/* JPEG */ /* -------------------------------------------------------------------- */ #ifdef HAVE_LIBJPEG @@ -500,15 +510,15 @@ PyImaging_ZipEncoderNew(PyObject* self, PyObject* args) /* We better define this encoder last in this file, so the following undef's won't mess things up for the Imaging library proper. */ -#undef HAVE_PROTOTYPES -#undef HAVE_STDDEF_H -#undef HAVE_STDLIB_H -#undef UINT8 -#undef UINT16 -#undef UINT32 -#undef INT8 -#undef INT16 -#undef INT32 +#undef HAVE_PROTOTYPES +#undef HAVE_STDDEF_H +#undef HAVE_STDLIB_H +#undef UINT8 +#undef UINT16 +#undef UINT32 +#undef INT8 +#undef INT16 +#undef INT32 #include "Jpeg.h" @@ -601,14 +611,14 @@ PyImaging_JpegEncoderNew(PyObject* self, PyObject* args) &progressive, &smooth, &optimize, &streamtype, &xdpi, &ydpi, &subsampling, &qtables, &extra, &extra_size, &rawExif, &rawExifLen)) - return NULL; + return NULL; encoder = PyImaging_EncoderNew(sizeof(JPEGENCODERSTATE)); if (encoder == NULL) - return NULL; + return NULL; if (get_packer(encoder, mode, rawmode) < 0) - return NULL; + return NULL; qarrays = get_qtables_arrays(qtables); @@ -718,11 +728,11 @@ PyImaging_LibTiffEncoderNew(PyObject* self, PyObject* args) return NULL; } - // While failes on 64 bit machines, complains that pos is an int instead of a Py_ssize_t - // while (PyDict_Next(dir, &pos, &key, &value)) { - for (pos=0;posencode = ImagingJpeg2KEncode; + encoder->cleanup = ImagingJpeg2KEncodeCleanup; + + context = (JPEG2KENCODESTATE *)encoder->state.context; + + context->fd = fd; + context->format = codec_format; + context->offset_x = context->offset_y = 0; + + j2k_decode_coord_tuple(offset, &context->offset_x, &context->offset_y); + j2k_decode_coord_tuple(tile_offset, + &context->tile_offset_x, + &context->tile_offset_y); + j2k_decode_coord_tuple(tile_size, + &context->tile_size_x, + &context->tile_size_y); + + /* Error on illegal tile offsets */ + if (context->tile_size_x && context->tile_size_y) { + if (context->tile_offset_x <= context->offset_x - context->tile_size_x + || context->tile_offset_y <= context->offset_y - context->tile_size_y) { + PyErr_SetString(PyExc_ValueError, + "JPEG 2000 tile offset too small; top left tile must " + "intersect image area"); + } + + if (context->tile_offset_x > context->offset_x + || context->tile_offset_y > context->offset_y) { + PyErr_SetString(PyExc_ValueError, + "JPEG 2000 tile offset too large to cover image area"); + Py_DECREF(encoder); + return NULL; + } + } + + if (quality_layers && PySequence_Check(quality_layers)) { + context->quality_is_in_db = strcmp (quality_mode, "dB") == 0; + context->quality_layers = quality_layers; + Py_INCREF(quality_layers); + } + + context->num_resolutions = num_resolutions; + + j2k_decode_coord_tuple(cblk_size, + &context->cblk_width, + &context->cblk_height); + j2k_decode_coord_tuple(precinct_size, + &context->precinct_width, + &context->precinct_height); + + context->irreversible = PyObject_IsTrue(irreversible); + context->progression = prog_order; + context->cinema_mode = cine_mode; + + return (PyObject *)encoder; +} + +#endif + +/* + * Local Variables: + * c-basic-offset: 4 + * End: + * + */ diff --git a/libImaging/Imaging.h b/libImaging/Imaging.h index b45dcbe23..26207d121 100644 --- a/libImaging/Imaging.h +++ b/libImaging/Imaging.h @@ -424,6 +424,14 @@ extern int ImagingJpegDecodeCleanup(ImagingCodecState state); extern int ImagingJpegEncode(Imaging im, ImagingCodecState state, UINT8* buffer, int bytes); #endif +#ifdef HAVE_OPENJPEG +extern int ImagingJpeg2KDecode(Imaging im, ImagingCodecState state, + UINT8* buffer, int bytes); +extern int ImagingJpeg2KDecodeCleanup(ImagingCodecState state); +extern int ImagingJpeg2KEncode(Imaging im, ImagingCodecState state, + UINT8* buffer, int bytes); +extern int ImagingJpeg2KEncodeCleanup(ImagingCodecState state); +#endif extern int ImagingLzwDecode(Imaging im, ImagingCodecState state, UINT8* buffer, int bytes); #ifdef HAVE_LIBTIFF @@ -497,6 +505,32 @@ struct ImagingCodecStateInstance { void *context; }; +/* Incremental encoding/decoding support */ +typedef struct ImagingIncrementalCodecStruct *ImagingIncrementalCodec; + +typedef int (*ImagingIncrementalCodecEntry)(Imaging im, + ImagingCodecState state, + ImagingIncrementalCodec codec); + +enum { + INCREMENTAL_CODEC_READ = 1, + INCREMENTAL_CODEC_WRITE = 2 +}; + +enum { + INCREMENTAL_CODEC_NOT_SEEKABLE = 0, + INCREMENTAL_CODEC_SEEKABLE = 1 +}; + +extern ImagingIncrementalCodec ImagingIncrementalCodecCreate(ImagingIncrementalCodecEntry codec_entry, Imaging im, ImagingCodecState state, int read_or_write, int seekable, int fd); +extern void ImagingIncrementalCodecDestroy(ImagingIncrementalCodec codec); +extern int ImagingIncrementalCodecPushBuffer(ImagingIncrementalCodec codec, UINT8 *buf, int bytes); +extern ssize_t ImagingIncrementalCodecRead(ImagingIncrementalCodec codec, void *buffer, size_t bytes); +extern off_t ImagingIncrementalCodecSkip(ImagingIncrementalCodec codec, off_t bytes); +extern ssize_t ImagingIncrementalCodecWrite(ImagingIncrementalCodec codec, const void *buffer, size_t bytes); +extern off_t ImagingIncrementalCodecSeek(ImagingIncrementalCodec codec, off_t bytes); +extern size_t ImagingIncrementalCodecBytesInBuffer(ImagingIncrementalCodec codec); + /* Errcodes */ #define IMAGING_CODEC_END 1 #define IMAGING_CODEC_OVERRUN -1 diff --git a/libImaging/Incremental.c b/libImaging/Incremental.c new file mode 100644 index 000000000..9e7fb38ec --- /dev/null +++ b/libImaging/Incremental.c @@ -0,0 +1,677 @@ +/* + * The Python Imaging Library + * $Id$ + * + * incremental decoding adaptor. + * + * Copyright (c) 2014 Coriolis Systems Limited + * Copyright (c) 2014 Alastair Houghton + * + */ + +#include "Imaging.h" + +/* The idea behind this interface is simple: the actual decoding proceeds in + a thread, which is run in lock step with the main thread. Whenever the + ImagingIncrementalCodecRead() call runs short on data, it suspends the + decoding thread and wakes the main thread. Conversely, the + ImagingIncrementalCodecPushBuffer() call suspends the main thread and wakes + the decoding thread, providing a buffer of data. + + The two threads are never running simultaneously, so there is no need for + any addition synchronisation measures outside of this file. + + Note also that we start the thread suspended (on Windows), or make it + immediately wait (other platforms), so that it's possible to initialise + things before the thread starts running. + + This interface is useful to allow PIL to interact efficiently with any + third-party imaging library that does not support suspendable reads; + one example is OpenJPEG (which is used for J2K support). The TIFF library + might also benefit from using this code. + + Note that if using this module, you want to set handles_eof on your + decoder to true. Why? Because otherwise ImageFile.load() will abort, + thinking that the image is truncated, whereas generally you want it to + pass the EOF condition (0 bytes to read) through to your code. */ + +/* Additional complication: *Some* codecs need to seek; this is fine if + there is a file descriptor, but if we're buffering data it becomes + awkward. The incremental adaptor now contains code to handle these + two cases. */ + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#define DEBUG_INCREMENTAL 0 + +#if DEBUG_INCREMENTAL +#define DEBUG(...) printf(__VA_ARGS__) +#else +#define DEBUG(...) +#endif + +struct ImagingIncrementalCodecStruct { +#ifdef _WIN32 + HANDLE hCodecEvent; + HANDLE hDataEvent; + HANDLE hThread; +#else + pthread_mutex_t start_mutex; + pthread_cond_t start_cond; + pthread_mutex_t codec_mutex; + pthread_cond_t codec_cond; + pthread_mutex_t data_mutex; + pthread_cond_t data_cond; + pthread_t thread; +#endif + ImagingIncrementalCodecEntry entry; + Imaging im; + ImagingCodecState state; + struct { + int fd; + UINT8 *buffer; /* Base of buffer */ + UINT8 *ptr; /* Current pointer in buffer */ + UINT8 *top; /* Highest point in buffer we've used */ + UINT8 *end; /* End of buffer */ + } stream; + int read_or_write; + int seekable; + int started; + int result; +}; + +static void flush_stream(ImagingIncrementalCodec codec); + +#if _WIN32 +static unsigned int __stdcall +codec_thread(void *ptr) +{ + ImagingIncrementalCodec codec = (ImagingIncrementalCodec)ptr; + + DEBUG("Entering thread\n"); + + codec->result = codec->entry(codec->im, codec->state, codec); + + DEBUG("Leaving thread (%d)\n", codec->result); + + flush_stream(codec); + + SetEvent(codec->hCodecEvent); + + return 0; +} +#else +static void * +codec_thread(void *ptr) +{ + ImagingIncrementalCodec codec = (ImagingIncrementalCodec)ptr; + + DEBUG("Entering thread\n"); + + codec->result = codec->entry(codec->im, codec->state, codec); + + DEBUG("Leaving thread (%d)\n", codec->result); + + flush_stream(codec); + + pthread_mutex_lock(&codec->codec_mutex); + pthread_cond_signal(&codec->codec_cond); + pthread_mutex_unlock(&codec->codec_mutex); + + return NULL; +} +#endif + +static void +flush_stream(ImagingIncrementalCodec codec) +{ + UINT8 *buffer; + size_t bytes; + + /* This is to flush data from the write buffer for a seekable write + codec. */ + if (codec->read_or_write != INCREMENTAL_CODEC_WRITE + || codec->state->errcode != IMAGING_CODEC_END + || !codec->seekable + || codec->stream.fd >= 0) + return; + + DEBUG("flushing data\n"); + + buffer = codec->stream.buffer; + bytes = codec->stream.ptr - codec->stream.buffer; + + codec->state->errcode = 0; + codec->seekable = INCREMENTAL_CODEC_NOT_SEEKABLE; + codec->stream.buffer = codec->stream.ptr = codec->stream.end + = codec->stream.top = NULL; + + ImagingIncrementalCodecWrite(codec, buffer, bytes); + + codec->state->errcode = IMAGING_CODEC_END; + codec->result = (int)ImagingIncrementalCodecBytesInBuffer(codec); + + free(buffer); +} + +/** + * Create a new incremental codec */ +ImagingIncrementalCodec +ImagingIncrementalCodecCreate(ImagingIncrementalCodecEntry codec_entry, + Imaging im, + ImagingCodecState state, + int read_or_write, + int seekable, + int fd) +{ + ImagingIncrementalCodec codec = (ImagingIncrementalCodec)malloc(sizeof(struct ImagingIncrementalCodecStruct)); + + codec->entry = codec_entry; + codec->im = im; + codec->state = state; + codec->result = 0; + codec->stream.fd = fd; + codec->stream.buffer = codec->stream.ptr = codec->stream.end + = codec->stream.top = NULL; + codec->started = 0; + codec->seekable = seekable; + codec->read_or_write = read_or_write; + + if (fd >= 0) + lseek(fd, 0, SEEK_SET); + + /* System specific set-up */ +#if _WIN32 + codec->hCodecEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + + if (!codec->hCodecEvent) { + free(codec); + return NULL; + } + + codec->hDataEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + + if (!codec->hDataEvent) { + CloseHandle(codec->hCodecEvent); + free(codec); + return NULL; + } + + codec->hThread = _beginthreadex(NULL, 0, codec_thread, codec, + CREATE_SUSPENDED, NULL); + + if (!codec->hThread) { + CloseHandle(codec->hCodecEvent); + CloseHandle(codec->hDataEvent); + free(codec); + return NULL; + } +#else + if (pthread_mutex_init(&codec->start_mutex, NULL)) { + free (codec); + return NULL; + } + + if (pthread_mutex_init(&codec->codec_mutex, NULL)) { + pthread_mutex_destroy(&codec->start_mutex); + free(codec); + return NULL; + } + + if (pthread_mutex_init(&codec->data_mutex, NULL)) { + pthread_mutex_destroy(&codec->start_mutex); + pthread_mutex_destroy(&codec->codec_mutex); + free(codec); + return NULL; + } + + if (pthread_cond_init(&codec->start_cond, NULL)) { + pthread_mutex_destroy(&codec->start_mutex); + pthread_mutex_destroy(&codec->codec_mutex); + pthread_mutex_destroy(&codec->data_mutex); + free(codec); + return NULL; + } + + if (pthread_cond_init(&codec->codec_cond, NULL)) { + pthread_mutex_destroy(&codec->start_mutex); + pthread_mutex_destroy(&codec->codec_mutex); + pthread_mutex_destroy(&codec->data_mutex); + pthread_cond_destroy(&codec->start_cond); + free(codec); + return NULL; + } + + if (pthread_cond_init(&codec->data_cond, NULL)) { + pthread_mutex_destroy(&codec->start_mutex); + pthread_mutex_destroy(&codec->codec_mutex); + pthread_mutex_destroy(&codec->data_mutex); + pthread_cond_destroy(&codec->start_cond); + pthread_cond_destroy(&codec->codec_cond); + free(codec); + return NULL; + } + + if (pthread_create(&codec->thread, NULL, codec_thread, codec)) { + pthread_mutex_destroy(&codec->start_mutex); + pthread_mutex_destroy(&codec->codec_mutex); + pthread_mutex_destroy(&codec->data_mutex); + pthread_cond_destroy(&codec->start_cond); + pthread_cond_destroy(&codec->codec_cond); + pthread_cond_destroy(&codec->data_cond); + free(codec); + return NULL; + } +#endif + + return codec; +} + +/** + * Destroy an incremental codec */ +void +ImagingIncrementalCodecDestroy(ImagingIncrementalCodec codec) +{ + DEBUG("destroying\n"); + + if (!codec->started) { +#ifdef _WIN32 + ResumeThread(codec->hThread); +#else + pthread_cond_signal(&codec->start_cond); +#endif + codec->started = 1; + } + +#ifndef _WIN32 + pthread_mutex_lock(&codec->data_mutex); +#endif + + if (codec->seekable && codec->stream.fd < 0) + free (codec->stream.buffer); + + codec->stream.buffer = codec->stream.ptr = codec->stream.end + = codec->stream.top = NULL; + +#ifdef _WIN32 + SetEvent(codec->hDataEvent); + + WaitForSingleObject(codec->hThread, INFINITE); + + CloseHandle(codec->hThread); + CloseHandle(codec->hCodecEvent); + CloseHandle(codec->hDataEvent); +#else + pthread_cond_signal(&codec->data_cond); + pthread_mutex_unlock(&codec->data_mutex); + + pthread_join(codec->thread, NULL); + + pthread_mutex_destroy(&codec->start_mutex); + pthread_mutex_destroy(&codec->codec_mutex); + pthread_mutex_destroy(&codec->data_mutex); + pthread_cond_destroy(&codec->start_cond); + pthread_cond_destroy(&codec->codec_cond); + pthread_cond_destroy(&codec->data_cond); +#endif + free (codec); +} + +/** + * Push a data buffer for an incremental codec */ +int +ImagingIncrementalCodecPushBuffer(ImagingIncrementalCodec codec, + UINT8 *buf, int bytes) +{ + if (!codec->started) { + DEBUG("starting\n"); + +#ifdef _WIN32 + ResumeThread(codec->hThread); +#else + pthread_cond_signal(&codec->start_cond); +#endif + codec->started = 1; + + /* Wait for the thread to ask for data */ +#ifdef _WIN32 + WaitForSingleObject(codec->hCodecEvent, INFINITE); +#else + pthread_mutex_lock(&codec->codec_mutex); + pthread_cond_wait(&codec->codec_cond, &codec->codec_mutex); + pthread_mutex_unlock(&codec->codec_mutex); +#endif + if (codec->result < 0) { + DEBUG("got result %d\n", codec->result); + + return codec->result; + } + } + + /* Codecs using an fd don't need data, so when we get here, we're done */ + if (codec->stream.fd >= 0) { + DEBUG("got result %d\n", codec->result); + + return codec->result; + } + + DEBUG("providing %p, %d\n", buf, bytes); + +#ifndef _WIN32 + pthread_mutex_lock(&codec->data_mutex); +#endif + + if (codec->read_or_write == INCREMENTAL_CODEC_READ + && codec->seekable && codec->stream.fd < 0) { + /* In this specific case, we append to a buffer we allocate ourselves */ + size_t old_size = codec->stream.end - codec->stream.buffer; + size_t new_size = codec->stream.end - codec->stream.buffer + bytes; + UINT8 *new = (UINT8 *)realloc (codec->stream.buffer, new_size); + + if (!new) { + codec->state->errcode = IMAGING_CODEC_MEMORY; +#ifndef _WIN32 + pthread_mutex_unlock(&codec->data_mutex); +#endif + return -1; + } + + codec->stream.ptr = codec->stream.ptr - codec->stream.buffer + new; + codec->stream.end = new + new_size; + codec->stream.buffer = new; + + memcpy(new + old_size, buf, bytes); + } else { + codec->stream.buffer = codec->stream.ptr = buf; + codec->stream.end = buf + bytes; + } + +#ifdef _WIN32 + SetEvent(codec->hDataEvent); + WaitForSingleObject(codec->hCodecEvent, INFINITE); +#else + pthread_cond_signal(&codec->data_cond); + pthread_mutex_unlock(&codec->data_mutex); + + pthread_mutex_lock(&codec->codec_mutex); + pthread_cond_wait(&codec->codec_cond, &codec->codec_mutex); + pthread_mutex_unlock(&codec->codec_mutex); +#endif + + DEBUG("got result %d\n", codec->result); + + return codec->result; +} + +size_t +ImagingIncrementalCodecBytesInBuffer(ImagingIncrementalCodec codec) +{ + return codec->stream.ptr - codec->stream.buffer; +} + +ssize_t +ImagingIncrementalCodecRead(ImagingIncrementalCodec codec, + void *buffer, size_t bytes) +{ + UINT8 *ptr = (UINT8 *)buffer; + size_t done = 0; + + if (codec->read_or_write == INCREMENTAL_CODEC_WRITE) { + DEBUG("attempt to read from write codec\n"); + return -1; + } + + DEBUG("reading (want %llu bytes)\n", (unsigned long long)bytes); + + if (codec->stream.fd >= 0) { + ssize_t ret = read(codec->stream.fd, buffer, bytes); + DEBUG("read %lld bytes from fd\n", (long long)ret); + return ret; + } + +#ifndef _WIN32 + pthread_mutex_lock(&codec->data_mutex); +#endif + while (bytes) { + size_t todo = bytes; + size_t remaining = codec->stream.end - codec->stream.ptr; + + if (!remaining) { + DEBUG("waiting for data\n"); + +#ifndef _WIN32 + pthread_mutex_lock(&codec->codec_mutex); +#endif + codec->result = (int)(codec->stream.ptr - codec->stream.buffer); +#if _WIN32 + SetEvent(codec->hCodecEvent); + WaitForSingleObject(codec->hDataEvent, INFINITE); +#else + pthread_cond_signal(&codec->codec_cond); + pthread_mutex_unlock(&codec->codec_mutex); + pthread_cond_wait(&codec->data_cond, &codec->data_mutex); +#endif + + remaining = codec->stream.end - codec->stream.ptr; + codec->stream.top = codec->stream.end; + + DEBUG("got %llu bytes\n", (unsigned long long)remaining); + } + + if (todo > remaining) + todo = remaining; + + if (!todo) + break; + + memcpy (ptr, codec->stream.ptr, todo); + codec->stream.ptr += todo; + bytes -= todo; + done += todo; + ptr += todo; + } +#ifndef _WIN32 + pthread_mutex_unlock(&codec->data_mutex); +#endif + + DEBUG("read total %llu bytes\n", (unsigned long long)done); + + return done; +} + +off_t +ImagingIncrementalCodecSkip(ImagingIncrementalCodec codec, + off_t bytes) +{ + off_t done = 0; + + DEBUG("skipping (want %llu bytes)\n", (unsigned long long)bytes); + + /* In write mode, explicitly fill with zeroes */ + if (codec->read_or_write == INCREMENTAL_CODEC_WRITE) { + static const UINT8 zeroes[256] = { 0 }; + off_t done = 0; + while (bytes) { + size_t todo = (size_t)(bytes > 256 ? 256 : bytes); + ssize_t written = ImagingIncrementalCodecWrite(codec, zeroes, todo); + if (written <= 0) + break; + done += written; + bytes -= written; + } + return done; + } + + if (codec->stream.fd >= 0) + return lseek(codec->stream.fd, bytes, SEEK_CUR); + +#ifndef _WIN32 + pthread_mutex_lock(&codec->data_mutex); +#endif + while (bytes) { + off_t todo = bytes; + off_t remaining = codec->stream.end - codec->stream.ptr; + + if (!remaining) { + DEBUG("waiting for data\n"); + +#ifndef _WIN32 + pthread_mutex_lock(&codec->codec_mutex); +#endif + codec->result = (int)(codec->stream.ptr - codec->stream.buffer); +#if _WIN32 + SetEvent(codec->hCodecEvent); + WaitForSingleObject(codec->hDataEvent, INFINITE); +#else + pthread_cond_signal(&codec->codec_cond); + pthread_mutex_unlock(&codec->codec_mutex); + pthread_cond_wait(&codec->data_cond, &codec->data_mutex); +#endif + + remaining = codec->stream.end - codec->stream.ptr; + } + + if (todo > remaining) + todo = remaining; + + if (!todo) + break; + + codec->stream.ptr += todo; + bytes -= todo; + done += todo; + } +#ifndef _WIN32 + pthread_mutex_unlock(&codec->data_mutex); +#endif + + DEBUG("skipped total %llu bytes\n", (unsigned long long)done); + + return done; +} + +ssize_t +ImagingIncrementalCodecWrite(ImagingIncrementalCodec codec, + const void *buffer, size_t bytes) +{ + const UINT8 *ptr = (const UINT8 *)buffer; + size_t done = 0; + + if (codec->read_or_write == INCREMENTAL_CODEC_READ) { + DEBUG("attempt to write from read codec\n"); + return -1; + } + + DEBUG("write (have %llu bytes)\n", (unsigned long long)bytes); + + if (codec->stream.fd >= 0) + return write(codec->stream.fd, buffer, bytes); + +#ifndef _WIN32 + pthread_mutex_lock(&codec->data_mutex); +#endif + while (bytes) { + size_t todo = bytes; + size_t remaining = codec->stream.end - codec->stream.ptr; + + if (!remaining) { + if (codec->seekable && codec->stream.fd < 0) { + /* In this case, we maintain the stream buffer ourselves */ + size_t old_size = codec->stream.top - codec->stream.buffer; + size_t new_size = (old_size + bytes + 65535) & ~65535; + UINT8 *new = (UINT8 *)realloc(codec->stream.buffer, new_size); + + if (!new) { + codec->state->errcode = IMAGING_CODEC_MEMORY; +#ifndef _WIN32 + pthread_mutex_unlock(&codec->data_mutex); +#endif + return done == 0 ? -1 : done; + } + + codec->stream.ptr = codec->stream.ptr - codec->stream.buffer + new; + codec->stream.buffer = new; + codec->stream.end = new + new_size; + codec->stream.top = new + old_size; + } else { + DEBUG("waiting for space\n"); + +#ifndef _WIN32 + pthread_mutex_lock(&codec->codec_mutex); +#endif + codec->result = (int)(codec->stream.ptr - codec->stream.buffer); +#if _WIN32 + SetEvent(codec->hCodecEvent); + WaitForSingleObject(codec->hDataEvent, INFINITE); +#else + pthread_cond_signal(&codec->codec_cond); + pthread_mutex_unlock(&codec->codec_mutex); + pthread_cond_wait(&codec->data_cond, &codec->data_mutex); +#endif + } + + remaining = codec->stream.end - codec->stream.ptr; + + DEBUG("got %llu bytes\n", (unsigned long long)remaining); + } + if (todo > remaining) + todo = remaining; + + if (!todo) + break; + + memcpy (codec->stream.ptr, ptr, todo); + codec->stream.ptr += todo; + bytes -= todo; + done += todo; + ptr += todo; + } + + if (codec->stream.ptr > codec->stream.top) + codec->stream.top = codec->stream.ptr; + +#ifndef _WIN32 + pthread_mutex_unlock(&codec->data_mutex); +#endif + + DEBUG("wrote total %llu bytes\n", (unsigned long long)done); + + return done; +} + +off_t +ImagingIncrementalCodecSeek(ImagingIncrementalCodec codec, + off_t bytes) +{ + off_t buffered; + + DEBUG("seeking (going to %llu bytes)\n", (unsigned long long)bytes); + + if (codec->stream.fd >= 0) + return lseek(codec->stream.fd, bytes, SEEK_SET); + + if (!codec->seekable) { + DEBUG("attempt to seek non-seekable stream\n"); + return -1; + } + + if (bytes < 0) { + DEBUG("attempt to seek before stream start\n"); + return -1; + } + + buffered = codec->stream.top - codec->stream.buffer; + + if (bytes <= buffered) { + DEBUG("seek within buffer\n"); + codec->stream.ptr = codec->stream.buffer + bytes; + return bytes; + } + + return buffered + ImagingIncrementalCodecSkip(codec, bytes - buffered); +} diff --git a/libImaging/Jpeg2K.h b/libImaging/Jpeg2K.h new file mode 100644 index 000000000..be6e0770b --- /dev/null +++ b/libImaging/Jpeg2K.h @@ -0,0 +1,91 @@ +/* + * The Python Imaging Library + * $Id$ + * + * declarations for the OpenJPEG codec interface. + * + * Copyright (c) 2014 by Coriolis Systems Limited + * Copyright (c) 2014 by Alastair Houghton + */ + +#include + +/* -------------------------------------------------------------------- */ +/* Decoder */ +/* -------------------------------------------------------------------- */ + +typedef struct { + /* CONFIGURATION */ + + /* File descriptor, if available; otherwise, -1 */ + int fd; + + /* Specify the desired format */ + OPJ_CODEC_FORMAT format; + + /* Set to divide image resolution by 2**reduce. */ + int reduce; + + /* Set to limit the number of quality layers to decode (0 = all layers) */ + int layers; + + /* PRIVATE CONTEXT (set by decoder) */ + const char *error_msg; + + ImagingIncrementalCodec decoder; +} JPEG2KDECODESTATE; + +/* -------------------------------------------------------------------- */ +/* Encoder */ +/* -------------------------------------------------------------------- */ + +typedef struct { + /* CONFIGURATION */ + + /* File descriptor, if available; otherwise, -1 */ + int fd; + + /* Specify the desired format */ + OPJ_CODEC_FORMAT format; + + /* Image offset */ + int offset_x, offset_y; + + /* Tile information */ + int tile_offset_x, tile_offset_y; + int tile_size_x, tile_size_y; + + /* Quality layers (a sequence of numbers giving *either* rates or dB) */ + int quality_is_in_db; + PyObject *quality_layers; + + /* Number of resolutions (DWT decompositions + 1 */ + int num_resolutions; + + /* Code block size */ + int cblk_width, cblk_height; + + /* Precinct size */ + int precinct_width, precinct_height; + + /* Compression style */ + int irreversible; + + /* Progression order (LRCP/RLCP/RPCL/PCRL/CPRL) */ + OPJ_PROG_ORDER progression; + + /* Cinema mode */ + OPJ_CINEMA_MODE cinema_mode; + + /* PRIVATE CONTEXT (set by decoder) */ + const char *error_msg; + + ImagingIncrementalCodec encoder; +} JPEG2KENCODESTATE; + +/* + * Local Variables: + * c-basic-offset: 4 + * End: + * + */ diff --git a/libImaging/Jpeg2KDecode.c b/libImaging/Jpeg2KDecode.c new file mode 100644 index 000000000..6b6176c78 --- /dev/null +++ b/libImaging/Jpeg2KDecode.c @@ -0,0 +1,753 @@ +/* + * The Python Imaging Library. + * $Id$ + * + * decoder for JPEG2000 image data. + * + * history: + * 2014-03-12 ajh Created + * + * Copyright (c) 2014 Coriolis Systems Limited + * Copyright (c) 2014 Alastair Houghton + * + * See the README file for details on usage and redistribution. + */ + +#include "Imaging.h" + +#ifdef HAVE_OPENJPEG + +#include +#include "Jpeg2K.h" + +typedef struct { + OPJ_UINT32 tile_index; + OPJ_UINT32 data_size; + OPJ_INT32 x0, y0, x1, y1; + OPJ_UINT32 nb_comps; +} JPEG2KTILEINFO; + +/* -------------------------------------------------------------------- */ +/* Error handler */ +/* -------------------------------------------------------------------- */ + +static void +j2k_error(const char *msg, void *client_data) +{ + JPEG2KDECODESTATE *state = (JPEG2KDECODESTATE *) client_data; + free((void *)state->error_msg); + state->error_msg = strdup(msg); +} + +/* -------------------------------------------------------------------- */ +/* Buffer input stream */ +/* -------------------------------------------------------------------- */ + +static OPJ_SIZE_T +j2k_read(void *p_buffer, OPJ_SIZE_T p_nb_bytes, void *p_user_data) +{ + ImagingIncrementalCodec decoder = (ImagingIncrementalCodec)p_user_data; + + size_t len = ImagingIncrementalCodecRead(decoder, p_buffer, p_nb_bytes); + + return len ? len : (OPJ_SIZE_T)-1; +} + +static OPJ_OFF_T +j2k_skip(OPJ_OFF_T p_nb_bytes, void *p_user_data) +{ + ImagingIncrementalCodec decoder = (ImagingIncrementalCodec)p_user_data; + off_t pos = ImagingIncrementalCodecSkip(decoder, p_nb_bytes); + + return pos ? pos : (OPJ_OFF_T)-1; +} + +/* -------------------------------------------------------------------- */ +/* Unpackers */ +/* -------------------------------------------------------------------- */ + +typedef void (*j2k_unpacker_t)(opj_image_t *in, + const JPEG2KTILEINFO *tileInfo, + const UINT8 *data, + Imaging im); + +struct j2k_decode_unpacker { + const char *mode; + OPJ_COLOR_SPACE color_space; + unsigned components; + j2k_unpacker_t unpacker; +}; + +static inline +unsigned j2ku_shift(unsigned x, int n) +{ + if (n < 0) + return x >> -n; + else + return x << n; +} + +static void +j2ku_gray_l(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, + const UINT8 *tiledata, Imaging im) +{ + unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0; + unsigned w = tileinfo->x1 - tileinfo->x0; + unsigned h = tileinfo->y1 - tileinfo->y0; + + int shift = 8 - in->comps[0].prec; + int offset = in->comps[0].sgnd ? 1 << (in->comps[0].prec - 1) : 0; + int csiz = (in->comps[0].prec + 7) >> 3; + + unsigned x, y; + + if (csiz == 3) + csiz = 4; + + if (shift < 0) + offset += 1 << (-shift - 1); + + switch (csiz) { + case 1: + for (y = 0; y < h; ++y) { + const UINT8 *data = &tiledata[y * w]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0; + for (x = 0; x < w; ++x) + *row++ = j2ku_shift(offset + *data++, shift); + } + break; + case 2: + for (y = 0; y < h; ++y) { + const UINT16 *data = (const UINT16 *)&tiledata[2 * y * w]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0; + for (x = 0; x < w; ++x) + *row++ = j2ku_shift(offset + *data++, shift); + } + break; + case 4: + for (y = 0; y < h; ++y) { + const UINT32 *data = (const UINT32 *)&tiledata[4 * y * w]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0; + for (x = 0; x < w; ++x) + *row++ = j2ku_shift(offset + *data++, shift); + } + break; + } +} + +static void +j2ku_gray_rgb(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, + const UINT8 *tiledata, Imaging im) +{ + unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0; + unsigned w = tileinfo->x1 - tileinfo->x0; + unsigned h = tileinfo->y1 - tileinfo->y0; + + int shift = 8 - in->comps[0].prec; + int offset = in->comps[0].sgnd ? 1 << (in->comps[0].prec - 1) : 0; + int csiz = (in->comps[0].prec + 7) >> 3; + + unsigned x, y; + + if (shift < 0) + offset += 1 << (-shift - 1); + + if (csiz == 3) + csiz = 4; + + switch (csiz) { + case 1: + for (y = 0; y < h; ++y) { + const UINT8 *data = &tiledata[y * w]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0; + for (x = 0; x < w; ++x) { + UINT8 byte = j2ku_shift(offset + *data++, shift); + row[0] = row[1] = row[2] = byte; + row[3] = 0xff; + row += 4; + } + } + break; + case 2: + for (y = 0; y < h; ++y) { + const UINT16 *data = (UINT16 *)&tiledata[2 * y * w]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0; + for (x = 0; x < w; ++x) { + UINT8 byte = j2ku_shift(offset + *data++, shift); + row[0] = row[1] = row[2] = byte; + row[3] = 0xff; + row += 4; + } + } + break; + case 4: + for (y = 0; y < h; ++y) { + const UINT32 *data = (UINT32 *)&tiledata[4 * y * w]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0; + for (x = 0; x < w; ++x) { + UINT8 byte = j2ku_shift(offset + *data++, shift); + row[0] = row[1] = row[2] = byte; + row[3] = 0xff; + row += 4; + } + } + break; + } +} + +static void +j2ku_graya_la(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, + const UINT8 *tiledata, Imaging im) +{ + unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0; + unsigned w = tileinfo->x1 - tileinfo->x0; + unsigned h = tileinfo->y1 - tileinfo->y0; + + int shift = 8 - in->comps[0].prec; + int offset = in->comps[0].sgnd ? 1 << (in->comps[0].prec - 1) : 0; + int csiz = (in->comps[0].prec + 7) >> 3; + int ashift = 8 - in->comps[1].prec; + int aoffset = in->comps[1].sgnd ? 1 << (in->comps[1].prec - 1) : 0; + int acsiz = (in->comps[1].prec + 7) >> 3; + const UINT8 *atiledata; + + unsigned x, y; + + if (csiz == 3) + csiz = 4; + if (acsiz == 3) + acsiz = 4; + + if (shift < 0) + offset += 1 << (-shift - 1); + if (ashift < 0) + aoffset += 1 << (-ashift - 1); + + atiledata = tiledata + csiz * w * h; + + for (y = 0; y < h; ++y) { + const UINT8 *data = &tiledata[csiz * y * w]; + const UINT8 *adata = &atiledata[acsiz * y * w]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; + for (x = 0; x < w; ++x) { + UINT32 word = 0, aword = 0, byte; + + switch (csiz) { + case 1: word = *data++; break; + case 2: word = *(const UINT16 *)data; data += 2; break; + case 4: word = *(const UINT32 *)data; data += 4; break; + } + + switch (acsiz) { + case 1: aword = *adata++; break; + case 2: aword = *(const UINT16 *)adata; adata += 2; break; + case 4: aword = *(const UINT32 *)adata; adata += 4; break; + } + + byte = j2ku_shift(offset + word, shift); + row[0] = row[1] = row[2] = byte; + row[3] = j2ku_shift(aoffset + aword, ashift); + row += 4; + } + } +} + +static void +j2ku_srgb_rgb(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, + const UINT8 *tiledata, Imaging im) +{ + unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0; + unsigned w = tileinfo->x1 - tileinfo->x0; + unsigned h = tileinfo->y1 - tileinfo->y0; + + int shifts[3], offsets[3], csiz[3]; + const UINT8 *cdata[3]; + const UINT8 *cptr = tiledata; + unsigned n, x, y; + + for (n = 0; n < 3; ++n) { + cdata[n] = cptr; + shifts[n] = 8 - in->comps[n].prec; + offsets[n] = in->comps[n].sgnd ? 1 << (in->comps[n].prec - 1) : 0; + csiz[n] = (in->comps[n].prec + 7) >> 3; + + if (csiz[n] == 3) + csiz[n] = 4; + + if (shifts[n] < 0) + offsets[n] += 1 << (-shifts[n] - 1); + + cptr += csiz[n] * w * h; + } + + for (y = 0; y < h; ++y) { + const UINT8 *data[3]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; + for (n = 0; n < 3; ++n) + data[n] = &cdata[n][csiz[n] * y * w]; + + for (x = 0; x < w; ++x) { + for (n = 0; n < 3; ++n) { + UINT32 word = 0; + + switch (csiz[n]) { + case 1: word = *data[n]++; break; + case 2: word = *(const UINT16 *)data[n]; data[n] += 2; break; + case 4: word = *(const UINT32 *)data[n]; data[n] += 4; break; + } + + row[n] = j2ku_shift(offsets[n] + word, shifts[n]); + } + row[3] = 0xff; + row += 4; + } + } +} + +static void +j2ku_sycc_rgb(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, + const UINT8 *tiledata, Imaging im) +{ + unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0; + unsigned w = tileinfo->x1 - tileinfo->x0; + unsigned h = tileinfo->y1 - tileinfo->y0; + + int shifts[3], offsets[3], csiz[3]; + const UINT8 *cdata[3]; + const UINT8 *cptr = tiledata; + unsigned n, x, y; + + for (n = 0; n < 3; ++n) { + cdata[n] = cptr; + shifts[n] = 8 - in->comps[n].prec; + offsets[n] = in->comps[n].sgnd ? 1 << (in->comps[n].prec - 1) : 0; + csiz[n] = (in->comps[n].prec + 7) >> 3; + + if (csiz[n] == 3) + csiz[n] = 4; + + if (shifts[n] < 0) + offsets[n] += 1 << (-shifts[n] - 1); + + cptr += csiz[n] * w * h; + } + + for (y = 0; y < h; ++y) { + const UINT8 *data[3]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; + UINT8 *row_start = row; + for (n = 0; n < 3; ++n) + data[n] = &cdata[n][csiz[n] * y * w]; + + for (x = 0; x < w; ++x) { + for (n = 0; n < 3; ++n) { + UINT32 word = 0; + + switch (csiz[n]) { + case 1: word = *data[n]++; break; + case 2: word = *(const UINT16 *)data[n]; data[n] += 2; break; + case 4: word = *(const UINT32 *)data[n]; data[n] += 4; break; + } + + row[n] = j2ku_shift(offsets[n] + word, shifts[n]); + } + row[3] = 0xff; + row += 4; + } + + ImagingConvertYCbCr2RGB(row_start, row_start, w); + } +} + +static void +j2ku_srgba_rgba(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, + const UINT8 *tiledata, Imaging im) +{ + unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0; + unsigned w = tileinfo->x1 - tileinfo->x0; + unsigned h = tileinfo->y1 - tileinfo->y0; + + int shifts[4], offsets[4], csiz[4]; + const UINT8 *cdata[4]; + const UINT8 *cptr = tiledata; + unsigned n, x, y; + + for (n = 0; n < 4; ++n) { + cdata[n] = cptr; + shifts[n] = 8 - in->comps[n].prec; + offsets[n] = in->comps[n].sgnd ? 1 << (in->comps[n].prec - 1) : 0; + csiz[n] = (in->comps[n].prec + 7) >> 3; + + if (csiz[n] == 3) + csiz[n] = 4; + + if (shifts[n] < 0) + offsets[n] += 1 << (-shifts[n] - 1); + + cptr += csiz[n] * w * h; + } + + for (y = 0; y < h; ++y) { + const UINT8 *data[4]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; + for (n = 0; n < 4; ++n) + data[n] = &cdata[n][csiz[n] * y * w]; + + for (x = 0; x < w; ++x) { + for (n = 0; n < 4; ++n) { + UINT32 word = 0; + + switch (csiz[n]) { + case 1: word = *data[n]++; break; + case 2: word = *(const UINT16 *)data[n]; data[n] += 2; break; + case 4: word = *(const UINT32 *)data[n]; data[n] += 4; break; + } + + row[n] = j2ku_shift(offsets[n] + word, shifts[n]); + } + row += 4; + } + } +} + +static void +j2ku_sycca_rgba(opj_image_t *in, const JPEG2KTILEINFO *tileinfo, + const UINT8 *tiledata, Imaging im) +{ + unsigned x0 = tileinfo->x0 - in->x0, y0 = tileinfo->y0 - in->y0; + unsigned w = tileinfo->x1 - tileinfo->x0; + unsigned h = tileinfo->y1 - tileinfo->y0; + + int shifts[4], offsets[4], csiz[4]; + const UINT8 *cdata[4]; + const UINT8 *cptr = tiledata; + unsigned n, x, y; + + for (n = 0; n < 4; ++n) { + cdata[n] = cptr; + shifts[n] = 8 - in->comps[n].prec; + offsets[n] = in->comps[n].sgnd ? 1 << (in->comps[n].prec - 1) : 0; + csiz[n] = (in->comps[n].prec + 7) >> 3; + + if (csiz[n] == 3) + csiz[n] = 4; + + if (shifts[n] < 0) + offsets[n] += 1 << (-shifts[n] - 1); + + cptr += csiz[n] * w * h; + } + + for (y = 0; y < h; ++y) { + const UINT8 *data[4]; + UINT8 *row = (UINT8 *)im->image[y0 + y] + x0 * 4; + UINT8 *row_start = row; + for (n = 0; n < 4; ++n) + data[n] = &cdata[n][csiz[n] * y * w]; + + for (x = 0; x < w; ++x) { + for (n = 0; n < 4; ++n) { + UINT32 word = 0; + + switch (csiz[n]) { + case 1: word = *data[n]++; break; + case 2: word = *(const UINT16 *)data[n]; data[n] += 2; break; + case 4: word = *(const UINT32 *)data[n]; data[n] += 4; break; + } + + row[n] = j2ku_shift(offsets[n] + word, shifts[n]); + } + row += 4; + } + + ImagingConvertYCbCr2RGB(row_start, row_start, w); + } +} + +static const struct j2k_decode_unpacker j2k_unpackers[] = { + { "L", OPJ_CLRSPC_GRAY, 1, j2ku_gray_l }, + { "LA", OPJ_CLRSPC_GRAY, 2, j2ku_graya_la }, + { "RGB", OPJ_CLRSPC_GRAY, 1, j2ku_gray_rgb }, + { "RGB", OPJ_CLRSPC_GRAY, 2, j2ku_gray_rgb }, + { "RGB", OPJ_CLRSPC_SRGB, 3, j2ku_srgb_rgb }, + { "RGB", OPJ_CLRSPC_SYCC, 3, j2ku_sycc_rgb }, + { "RGB", OPJ_CLRSPC_SRGB, 4, j2ku_srgb_rgb }, + { "RGB", OPJ_CLRSPC_SYCC, 4, j2ku_sycc_rgb }, + { "RGBA", OPJ_CLRSPC_GRAY, 1, j2ku_gray_rgb }, + { "RGBA", OPJ_CLRSPC_GRAY, 2, j2ku_graya_la }, + { "RGBA", OPJ_CLRSPC_SRGB, 3, j2ku_srgb_rgb }, + { "RGBA", OPJ_CLRSPC_SYCC, 3, j2ku_sycc_rgb }, + { "RGBA", OPJ_CLRSPC_SRGB, 4, j2ku_srgba_rgba }, + { "RGBA", OPJ_CLRSPC_SYCC, 4, j2ku_sycca_rgba }, +}; + +/* -------------------------------------------------------------------- */ +/* Decoder */ +/* -------------------------------------------------------------------- */ + +enum { + J2K_STATE_START = 0, + J2K_STATE_DECODING = 1, + J2K_STATE_DONE = 2, + J2K_STATE_FAILED = 3, +}; + +static int +j2k_decode_entry(Imaging im, ImagingCodecState state, + ImagingIncrementalCodec decoder) +{ + JPEG2KDECODESTATE *context = (JPEG2KDECODESTATE *) state->context; + opj_stream_t *stream = NULL; + opj_image_t *image = NULL; + opj_codec_t *codec = NULL; + opj_dparameters_t params; + OPJ_COLOR_SPACE color_space; + j2k_unpacker_t unpack = NULL; + size_t buffer_size = 0; + unsigned n; + + stream = opj_stream_default_create(OPJ_TRUE); + + if (!stream) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + opj_stream_set_read_function(stream, j2k_read); + opj_stream_set_skip_function(stream, j2k_skip); + + opj_stream_set_user_data(stream, decoder); + + /* Setup decompression context */ + context->error_msg = NULL; + + opj_set_default_decoder_parameters(¶ms); + params.cp_reduce = context->reduce; + params.cp_layer = context->layers; + + codec = opj_create_decompress(context->format); + + if (!codec) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + opj_set_error_handler(codec, j2k_error, context); + opj_setup_decoder(codec, ¶ms); + + if (!opj_read_header(stream, codec, &image)) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + /* Check that this image is something we can handle */ + if (image->numcomps < 1 || image->numcomps > 4 + || image->color_space == OPJ_CLRSPC_UNKNOWN) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + for (n = 1; n < image->numcomps; ++n) { + if (image->comps[n].dx != 1 || image->comps[n].dy != 1) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + } + + /* + Colorspace Number of components PIL mode + ------------------------------------------------------ + sRGB 3 RGB + sRGB 4 RGBA + gray 1 L or I + gray 2 LA + YCC 3 YCbCr + + + If colorspace is unspecified, we assume: + + Number of components Colorspace + ----------------------------------------- + 1 gray + 2 gray (+ alpha) + 3 sRGB + 4 sRGB (+ alpha) + + */ + + /* Find the correct unpacker */ + color_space = image->color_space; + + if (color_space == OPJ_CLRSPC_UNSPECIFIED) { + switch (image->numcomps) { + case 1: case 2: color_space = OPJ_CLRSPC_GRAY; break; + case 3: case 4: color_space = OPJ_CLRSPC_SRGB; break; + } + } + + for (n = 0; n < sizeof(j2k_unpackers) / sizeof (j2k_unpackers[0]); ++n) { + if (color_space == j2k_unpackers[n].color_space + && image->numcomps == j2k_unpackers[n].components + && strcmp (im->mode, j2k_unpackers[n].mode) == 0) { + unpack = j2k_unpackers[n].unpacker; + break; + } + } + + if (!unpack) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + /* Decode the image tile-by-tile; this means we only need use as much + memory as is required for one tile's worth of components. */ + for (;;) { + JPEG2KTILEINFO tile_info; + OPJ_BOOL should_continue; + unsigned correction = (1 << params.cp_reduce) - 1; + + if (!opj_read_tile_header(codec, + stream, + &tile_info.tile_index, + &tile_info.data_size, + &tile_info.x0, &tile_info.y0, + &tile_info.x1, &tile_info.y1, + &tile_info.nb_comps, + &should_continue)) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + if (!should_continue) + break; + + /* Adjust the tile co-ordinates based on the reduction (OpenJPEG + doesn't do this for us) */ + tile_info.x0 = (tile_info.x0 + correction) >> context->reduce; + tile_info.y0 = (tile_info.y0 + correction) >> context->reduce; + tile_info.x1 = (tile_info.x1 + correction) >> context->reduce; + tile_info.y1 = (tile_info.y1 + correction) >> context->reduce; + + if (buffer_size < tile_info.data_size) { + UINT8 *new = realloc (state->buffer, tile_info.data_size); + if (!new) { + state->errcode = IMAGING_CODEC_MEMORY; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + state->buffer = new; + buffer_size = tile_info.data_size; + } + + if (!opj_decode_tile_data(codec, + tile_info.tile_index, + (OPJ_BYTE *)state->buffer, + tile_info.data_size, + stream)) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + /* Check the tile bounds; if the tile is outside the image area, + or if it has a negative width or height (i.e. the coordinates are + swapped), bail. */ + if (tile_info.x0 >= tile_info.x1 + || tile_info.y0 >= tile_info.y1 + || tile_info.x0 < image->x0 + || tile_info.y0 < image->y0 + || tile_info.x1 - image->x0 > im->xsize + || tile_info.y1 - image->y0 > im->ysize) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + unpack(image, &tile_info, state->buffer, im); + } + + if (!opj_end_decompress(codec, stream)) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + state->state = J2K_STATE_DONE; + state->errcode = IMAGING_CODEC_END; + + quick_exit: + if (codec) + opj_destroy_codec(codec); + if (image) + opj_image_destroy(image); + if (stream) + opj_stream_destroy(stream); + + return -1; +} + +int +ImagingJpeg2KDecode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes) +{ + JPEG2KDECODESTATE *context = (JPEG2KDECODESTATE *) state->context; + + if (state->state == J2K_STATE_DONE || state->state == J2K_STATE_FAILED) + return -1; + + if (state->state == J2K_STATE_START) { + context->decoder = ImagingIncrementalCodecCreate(j2k_decode_entry, + im, state, + INCREMENTAL_CODEC_READ, + INCREMENTAL_CODEC_NOT_SEEKABLE, + context->fd); + + if (!context->decoder) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + return -1; + } + + state->state = J2K_STATE_DECODING; + } + + return ImagingIncrementalCodecPushBuffer(context->decoder, buf, bytes); +} + +/* -------------------------------------------------------------------- */ +/* Cleanup */ +/* -------------------------------------------------------------------- */ + +int +ImagingJpeg2KDecodeCleanup(ImagingCodecState state) { + JPEG2KDECODESTATE *context = (JPEG2KDECODESTATE *)state->context; + + if (context->error_msg) + free ((void *)context->error_msg); + + if (context->decoder) + ImagingIncrementalCodecDestroy(context->decoder); + + return -1; +} + +const char * +ImagingJpeg2KVersion(void) +{ + return opj_version(); +} + +#endif /* HAVE_OPENJPEG */ + +/* + * Local Variables: + * c-basic-offset: 4 + * End: + * + */ diff --git a/libImaging/Jpeg2KEncode.c b/libImaging/Jpeg2KEncode.c new file mode 100644 index 000000000..c1e16e97f --- /dev/null +++ b/libImaging/Jpeg2KEncode.c @@ -0,0 +1,562 @@ +/* + * The Python Imaging Library. + * $Id$ + * + * decoder for JPEG2000 image data. + * + * history: + * 2014-03-12 ajh Created + * + * Copyright (c) 2014 Coriolis Systems Limited + * Copyright (c) 2014 Alastair Houghton + * + * See the README file for details on usage and redistribution. + */ + +#include "Imaging.h" + +#ifdef HAVE_OPENJPEG + +#include "Jpeg2K.h" + +#define CINEMA_24_CS_LENGTH 1302083 +#define CINEMA_48_CS_LENGTH 651041 +#define COMP_24_CS_MAX_LENGTH 1041666 +#define COMP_48_CS_MAX_LENGTH 520833 + +/* -------------------------------------------------------------------- */ +/* Error handler */ +/* -------------------------------------------------------------------- */ + +static void +j2k_error(const char *msg, void *client_data) +{ + JPEG2KENCODESTATE *state = (JPEG2KENCODESTATE *) client_data; + free((void *)state->error_msg); + state->error_msg = strdup(msg); +} + +/* -------------------------------------------------------------------- */ +/* Buffer output stream */ +/* -------------------------------------------------------------------- */ + +static OPJ_SIZE_T +j2k_write(void *p_buffer, OPJ_SIZE_T p_nb_bytes, void *p_user_data) +{ + ImagingIncrementalCodec encoder = (ImagingIncrementalCodec)p_user_data; + size_t len = ImagingIncrementalCodecWrite(encoder, p_buffer, p_nb_bytes); + + return len ? len : (OPJ_SIZE_T)-1; +} + +static OPJ_OFF_T +j2k_skip(OPJ_OFF_T p_nb_bytes, void *p_user_data) +{ + ImagingIncrementalCodec encoder = (ImagingIncrementalCodec)p_user_data; + off_t pos = ImagingIncrementalCodecSkip(encoder, p_nb_bytes); + + return pos ? pos : (OPJ_OFF_T)-1; +} + +static OPJ_BOOL +j2k_seek(OPJ_OFF_T p_nb_bytes, void *p_user_data) +{ + ImagingIncrementalCodec encoder = (ImagingIncrementalCodec)p_user_data; + off_t pos = ImagingIncrementalCodecSeek(encoder, p_nb_bytes); + + return pos == p_nb_bytes; +} + +/* -------------------------------------------------------------------- */ +/* Encoder */ +/* -------------------------------------------------------------------- */ + +typedef void (*j2k_pack_tile_t)(Imaging im, UINT8 *buf, + unsigned x0, unsigned y0, + unsigned w, unsigned h); + +static void +j2k_pack_l(Imaging im, UINT8 *buf, + unsigned x0, unsigned y0, unsigned w, unsigned h) +{ + UINT8 *ptr = buf; + unsigned x,y; + for (y = 0; y < h; ++y) { + UINT8 *data = (UINT8 *)(im->image[y + y0] + x0); + for (x = 0; x < w; ++x) + *ptr++ = *data++; + } +} + +static void +j2k_pack_la(Imaging im, UINT8 *buf, + unsigned x0, unsigned y0, unsigned w, unsigned h) +{ + UINT8 *ptr = buf; + UINT8 *ptra = buf + w * h; + unsigned x,y; + for (y = 0; y < h; ++y) { + UINT8 *data = (UINT8 *)(im->image[y + y0] + 4 * x0); + for (x = 0; x < w; ++x) { + *ptr++ = data[0]; + *ptra++ = data[3]; + data += 4; + } + } +} + +static void +j2k_pack_rgb(Imaging im, UINT8 *buf, + unsigned x0, unsigned y0, unsigned w, unsigned h) +{ + UINT8 *pr = buf; + UINT8 *pg = pr + w * h; + UINT8 *pb = pg + w * h; + unsigned x,y; + for (y = 0; y < h; ++y) { + UINT8 *data = (UINT8 *)(im->image[y + y0] + 4 * x0); + for (x = 0; x < w; ++x) { + *pr++ = data[0]; + *pg++ = data[1]; + *pb++ = data[2]; + data += 4; + } + } +} + +static void +j2k_pack_rgba(Imaging im, UINT8 *buf, + unsigned x0, unsigned y0, unsigned w, unsigned h) +{ + UINT8 *pr = buf; + UINT8 *pg = pr + w * h; + UINT8 *pb = pg + w * h; + UINT8 *pa = pb + w * h; + unsigned x,y; + for (y = 0; y < h; ++y) { + UINT8 *data = (UINT8 *)(im->image[y + y0] + 4 * x0); + for (x = 0; x < w; ++x) { + *pr++ = *data++; + *pg++ = *data++; + *pb++ = *data++; + *pa++ = *data++; + } + } +} + +enum { + J2K_STATE_START = 0, + J2K_STATE_ENCODING = 1, + J2K_STATE_DONE = 2, + J2K_STATE_FAILED = 3, +}; + +static void +j2k_set_cinema_params(Imaging im, int components, opj_cparameters_t *params) +{ + float rate; + unsigned n; + + /* These settings have been copied from opj_compress in the OpenJPEG + sources. */ + + params->tile_size_on = OPJ_FALSE; + params->cp_tdx = params->cp_tdy = 1; + params->tp_flag = 'C'; + params->tp_on = 1; + params->cp_tx0 = params->cp_ty0 = 0; + params->image_offset_x0 = params->image_offset_y0 = 0; + params->cblockw_init = 32; + params->cblockh_init = 32; + params->csty |= 0x01; + params->prog_order = OPJ_CPRL; + params->roi_compno = -1; + params->subsampling_dx = params->subsampling_dy = 1; + params->irreversible = 1; + + if (params->cp_cinema == OPJ_CINEMA4K_24) { + float max_rate = ((float)(components * im->xsize * im->ysize * 8) + / (CINEMA_24_CS_LENGTH * 8)); + + params->POC[0].tile = 1; + params->POC[0].resno0 = 0; + params->POC[0].compno0 = 0; + params->POC[0].layno1 = 1; + params->POC[0].resno1 = params->numresolution - 1; + params->POC[0].compno1 = 3; + params->POC[0].prg1 = OPJ_CPRL; + params->POC[1].tile = 1; + params->POC[1].resno0 = 0; + params->POC[1].compno0 = 0; + params->POC[1].layno1 = 1; + params->POC[1].resno1 = params->numresolution - 1; + params->POC[1].compno1 = 3; + params->POC[1].prg1 = OPJ_CPRL; + params->numpocs = 2; + + for (n = 0; n < params->tcp_numlayers; ++n) { + rate = 0; + if (params->tcp_rates[0] == 0) { + params->tcp_rates[n] = max_rate; + } else { + rate = ((float)(components * im->xsize * im->ysize * 8) + / (params->tcp_rates[n] * 8)); + if (rate > CINEMA_24_CS_LENGTH) + params->tcp_rates[n] = max_rate; + } + } + + params->max_comp_size = COMP_24_CS_MAX_LENGTH; + } else { + float max_rate = ((float)(components * im->xsize * im->ysize * 8) + / (CINEMA_48_CS_LENGTH * 8)); + + for (n = 0; n < params->tcp_numlayers; ++n) { + rate = 0; + if (params->tcp_rates[0] == 0) { + params->tcp_rates[n] = max_rate; + } else { + rate = ((float)(components * im->xsize * im->ysize * 8) + / (params->tcp_rates[n] * 8)); + if (rate > CINEMA_48_CS_LENGTH) + params->tcp_rates[n] = max_rate; + } + } + + params->max_comp_size = COMP_48_CS_MAX_LENGTH; + } +} + +static int +j2k_encode_entry(Imaging im, ImagingCodecState state, + ImagingIncrementalCodec encoder) +{ + JPEG2KENCODESTATE *context = (JPEG2KENCODESTATE *)state->context; + opj_stream_t *stream = NULL; + opj_image_t *image = NULL; + opj_codec_t *codec = NULL; + opj_cparameters_t params; + unsigned components; + OPJ_COLOR_SPACE color_space; + opj_image_cmptparm_t image_params[4]; + unsigned xsiz, ysiz; + unsigned tile_width, tile_height; + unsigned tiles_x, tiles_y, num_tiles; + unsigned x, y, tile_ndx; + unsigned n; + j2k_pack_tile_t pack; + int ret = -1; + + stream = opj_stream_default_create(OPJ_FALSE); + + if (!stream) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + opj_stream_set_write_function(stream, j2k_write); + opj_stream_set_skip_function(stream, j2k_skip); + opj_stream_set_seek_function(stream, j2k_seek); + + opj_stream_set_user_data(stream, encoder); + + /* Setup an opj_image */ + if (strcmp (im->mode, "L") == 0) { + components = 1; + color_space = OPJ_CLRSPC_GRAY; + pack = j2k_pack_l; + } else if (strcmp (im->mode, "LA") == 0) { + components = 2; + color_space = OPJ_CLRSPC_GRAY; + pack = j2k_pack_la; + } else if (strcmp (im->mode, "RGB") == 0) { + components = 3; + color_space = OPJ_CLRSPC_SRGB; + pack = j2k_pack_rgb; + } else if (strcmp (im->mode, "YCbCr") == 0) { + components = 3; + color_space = OPJ_CLRSPC_SYCC; + pack = j2k_pack_rgb; + } else if (strcmp (im->mode, "RGBA") == 0) { + components = 4; + color_space = OPJ_CLRSPC_SRGB; + pack = j2k_pack_rgba; + } else { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + for (n = 0; n < components; ++n) { + image_params[n].dx = image_params[n].dy = 1; + image_params[n].w = im->xsize; + image_params[n].h = im->ysize; + image_params[n].x0 = image_params[n].y0 = 0; + image_params[n].prec = 8; + image_params[n].bpp = 8; + image_params[n].sgnd = 0; + } + + image = opj_image_create(components, image_params, color_space); + + /* Setup compression context */ + context->error_msg = NULL; + + opj_set_default_encoder_parameters(¶ms); + + params.image_offset_x0 = context->offset_x; + params.image_offset_y0 = context->offset_y; + + if (context->tile_size_x && context->tile_size_y) { + params.tile_size_on = OPJ_TRUE; + params.cp_tx0 = context->tile_offset_x; + params.cp_ty0 = context->tile_offset_y; + params.cp_tdx = context->tile_size_x; + params.cp_tdy = context->tile_size_y; + + tile_width = params.cp_tdx; + tile_height = params.cp_tdy; + } else { + params.cp_tx0 = 0; + params.cp_ty0 = 0; + params.cp_tdx = 1; + params.cp_tdy = 1; + + tile_width = im->xsize; + tile_height = im->ysize; + } + + if (context->quality_layers && PySequence_Check(context->quality_layers)) { + Py_ssize_t len = PySequence_Length(context->quality_layers); + Py_ssize_t n; + float *pq; + + if (len) { + if (len > sizeof(params.tcp_rates) / sizeof(params.tcp_rates[0])) + len = sizeof(params.tcp_rates)/sizeof(params.tcp_rates[0]); + + params.tcp_numlayers = (int)len; + + if (context->quality_is_in_db) { + params.cp_disto_alloc = params.cp_fixed_alloc = 0; + params.cp_fixed_quality = 1; + pq = params.tcp_distoratio; + } else { + params.cp_disto_alloc = 1; + params.cp_fixed_alloc = params.cp_fixed_quality = 0; + pq = params.tcp_rates; + } + + for (n = 0; n < len; ++n) { + PyObject *obj = PySequence_ITEM(context->quality_layers, n); + pq[n] = PyFloat_AsDouble(obj); + } + } + } else { + params.tcp_numlayers = 1; + params.tcp_rates[0] = 0; + params.cp_disto_alloc = 1; + } + + if (context->num_resolutions) + params.numresolution = context->num_resolutions; + + if (context->cblk_width >= 4 && context->cblk_width <= 1024 + && context->cblk_height >= 4 && context->cblk_height <= 1024 + && context->cblk_width * context->cblk_height <= 4096) { + params.cblockw_init = context->cblk_width; + params.cblockh_init = context->cblk_height; + } + + if (context->precinct_width >= 4 && context->precinct_height >= 4 + && context->precinct_width >= context->cblk_width + && context->precinct_height > context->cblk_height) { + params.prcw_init[0] = context->precinct_width; + params.prch_init[0] = context->precinct_height; + params.res_spec = 1; + params.csty |= 0x01; + } + + params.irreversible = context->irreversible; + + params.prog_order = context->progression; + + params.cp_cinema = context->cinema_mode; + + switch (params.cp_cinema) { + case OPJ_OFF: + params.cp_rsiz = OPJ_STD_RSIZ; + break; + case OPJ_CINEMA2K_24: + case OPJ_CINEMA2K_48: + params.cp_rsiz = OPJ_CINEMA2K; + if (params.numresolution > 6) + params.numresolution = 6; + break; + case OPJ_CINEMA4K_24: + params.cp_rsiz = OPJ_CINEMA4K; + if (params.numresolution > 7) + params.numresolution = 7; + break; + } + + if (context->cinema_mode != OPJ_OFF) + j2k_set_cinema_params(im, components, ¶ms); + + /* Set up the reference grid in the image */ + image->x0 = params.image_offset_x0; + image->y0 = params.image_offset_y0; + image->x1 = xsiz = im->xsize + params.image_offset_x0; + image->y1 = ysiz = im->ysize + params.image_offset_y0; + + /* Create the compressor */ + codec = opj_create_compress(context->format); + + if (!codec) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + opj_set_error_handler(codec, j2k_error, context); + opj_setup_encoder(codec, ¶ms, image); + + /* Start encoding */ + if (!opj_start_compress(codec, image, stream)) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + /* Write each tile */ + tiles_x = (im->xsize + (params.image_offset_x0 - params.cp_tx0) + + tile_width - 1) / tile_width; + tiles_y = (im->ysize + (params.image_offset_y0 - params.cp_ty0) + + tile_height - 1) / tile_height; + + num_tiles = tiles_x * tiles_y; + + state->buffer = malloc (tile_width * tile_height * components); + + tile_ndx = 0; + for (y = 0; y < tiles_y; ++y) { + unsigned ty0 = params.cp_ty0 + y * tile_height; + unsigned ty1 = ty0 + tile_height; + unsigned pixy, pixh; + + if (ty0 < params.image_offset_y0) + ty0 = params.image_offset_y0; + if (ty1 > ysiz) + ty1 = ysiz; + + pixy = ty0 - params.image_offset_y0; + pixh = ty1 - ty0; + + for (x = 0; x < tiles_x; ++x) { + unsigned tx0 = params.cp_tx0 + x * tile_width; + unsigned tx1 = tx0 + tile_width; + unsigned pixx, pixw; + unsigned data_size; + + if (tx0 < params.image_offset_x0) + tx0 = params.image_offset_x0; + if (tx1 > xsiz) + tx1 = xsiz; + + pixx = tx0 - params.image_offset_x0; + pixw = tx1 - tx0; + + pack(im, state->buffer, pixx, pixy, pixw, pixh); + + data_size = pixw * pixh * components; + + if (!opj_write_tile(codec, tile_ndx++, state->buffer, + data_size, stream)) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + } + } + + if (!opj_end_compress(codec, stream)) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + goto quick_exit; + } + + state->errcode = IMAGING_CODEC_END; + state->state = J2K_STATE_DONE; + ret = (int)ImagingIncrementalCodecBytesInBuffer(encoder); + + quick_exit: + if (codec) + opj_destroy_codec(codec); + if (image) + opj_image_destroy(image); + if (stream) + opj_stream_destroy(stream); + + return ret; +} + +int +ImagingJpeg2KEncode(Imaging im, ImagingCodecState state, UINT8 *buf, int bytes) +{ + JPEG2KENCODESTATE *context = (JPEG2KENCODESTATE *)state->context; + + if (state->state == J2K_STATE_FAILED) + return -1; + + if (state->state == J2K_STATE_START) { + int seekable = (context->format != OPJ_CODEC_J2K + ? INCREMENTAL_CODEC_SEEKABLE + : INCREMENTAL_CODEC_NOT_SEEKABLE); + + context->encoder = ImagingIncrementalCodecCreate(j2k_encode_entry, + im, state, + INCREMENTAL_CODEC_WRITE, + seekable, + context->fd); + + if (!context->encoder) { + state->errcode = IMAGING_CODEC_BROKEN; + state->state = J2K_STATE_FAILED; + return -1; + } + + state->state = J2K_STATE_ENCODING; + } + + return ImagingIncrementalCodecPushBuffer(context->encoder, buf, bytes); +} + +/* -------------------------------------------------------------------- */ +/* Cleanup */ +/* -------------------------------------------------------------------- */ + +int +ImagingJpeg2KEncodeCleanup(ImagingCodecState state) { + JPEG2KENCODESTATE *context = (JPEG2KENCODESTATE *)state->context; + + if (context->quality_layers) + Py_DECREF(context->quality_layers); + + if (context->error_msg) + free ((void *)context->error_msg); + + if (context->encoder) + ImagingIncrementalCodecDestroy(context->encoder); + + return -1; +} + +#endif /* HAVE_OPENJPEG */ + +/* + * Local Variables: + * c-basic-offset: 4 + * End: + * + */ diff --git a/libImaging/PcxDecode.c b/libImaging/PcxDecode.c index 04c86cb35..af282cfe4 100644 --- a/libImaging/PcxDecode.c +++ b/libImaging/PcxDecode.c @@ -57,7 +57,16 @@ ImagingPcxDecode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes) } if (state->x >= state->bytes) { - + if (state->bytes % state->xsize && state->bytes > state->xsize) { + int bands = state->bytes / state->xsize; + int stride = state->bytes / bands; + int i; + for (i=1; i< bands; i++) { // note -- skipping first band + memmove(&state->buffer[i*state->xsize], + &state->buffer[i*stride], + state->xsize); + } + } /* Got a full line, unpack it */ state->shuffle((UINT8*) im->image[state->y + state->yoff] + state->xoff * im->pixelsize, state->buffer, diff --git a/libImaging/PcxEncode.c b/libImaging/PcxEncode.c index 87d599463..c1f64a33d 100644 --- a/libImaging/PcxEncode.c +++ b/libImaging/PcxEncode.c @@ -5,7 +5,7 @@ * encoder for PCX data * * history: - * 99-02-07 fl created + * 99-02-07 fl created * * Copyright (c) Fredrik Lundh 1999. * Copyright (c) Secret Labs AB 1999. @@ -26,23 +26,41 @@ ImagingPcxEncode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes) { UINT8* ptr; int this; + int bytes_per_line = 0; + int padding = 0; + int stride = 0; + int bpp = 0; + int planes = 1; + int i; ptr = buf; if (!state->state) { - /* sanity check */ if (state->xsize <= 0 || state->ysize <= 0) { state->errcode = IMAGING_CODEC_END; return 0; } - - state->bytes = (state->xsize*state->bits + 7) / 8; state->state = FETCH; - } - for (;;) + bpp = state->bits; + if (state->bits == 24){ + planes = 3; + bpp = 8; + } + + bytes_per_line = (state->xsize*bpp + 7) / 8; + /* The stride here needs to be kept in sync with the version in + PcxImagePlugin.py. If it's not, the header and the body of the + image will be out of sync and bad things will happen on decode. + */ + stride = bytes_per_line + (bytes_per_line % 2); + + padding = stride - bytes_per_line; + + + for (;;) { switch (state->state) { case FETCH: @@ -68,81 +86,103 @@ ImagingPcxEncode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes) /* fall through */ case ENCODE: - /* compress this line */ /* when we arrive here, "count" contains the number of bytes having the value of "LAST" that we've already seen */ + while (state->x < planes * bytes_per_line) { + /* If we're encoding an odd width file, and we've + got more than one plane, we need to pad each + color row with padding bytes at the end. Since + The pixels are stored RRRRRGGGGGBBBBB, so we need + to have the padding be RRRRRPGGGGGPBBBBBP. Hence + the double loop + */ + while (state->x % bytes_per_line) { - while (state->x < state->bytes) { - - if (state->count == 63) { - - /* this run is full; flush it */ - if (bytes < 2) - return ptr - buf; - *ptr++ = 0xff; - *ptr++ = state->LAST; - bytes -= 2; - - state->count = 0; - - } - - this = state->buffer[state->x]; - - if (this == state->LAST) { - - /* extend the current run */ - state->x++; - state->count++; - - } else { - - /* start a new run */ - if (state->count == 1 && (state->LAST < 0xc0)) { - if (bytes < 1) + if (state->count == 63) { + /* this run is full; flush it */ + if (bytes < 2) return ptr - buf; + *ptr++ = 0xff; *ptr++ = state->LAST; - bytes--; - } else { - if (state->count > 0) { - if (bytes < 2) - return ptr - buf; - *ptr++ = 0xc0 | state->count; - *ptr++ = state->LAST; - bytes -= 2; - } + bytes -= 2; + + state->count = 0; + } - state->LAST = this; - state->count = 1; + this = state->buffer[state->x]; - state->x++; + if (this == state->LAST) { + /* extend the current run */ + state->x++; + state->count++; + } else { + /* start a new run */ + if (state->count == 1 && (state->LAST < 0xc0)) { + if (bytes < 1) { + return ptr - buf; + } + *ptr++ = state->LAST; + bytes--; + } else { + if (state->count > 0) { + if (bytes < 2) { + return ptr - buf; + } + *ptr++ = 0xc0 | state->count; + *ptr++ = state->LAST; + bytes -= 2; + } + } + + state->LAST = this; + state->count = 1; + + state->x++; + + } } - } - /* end of line; flush the current run */ - if (state->count == 1 && (state->LAST < 0xc0)) { - if (bytes < 1) - return ptr - buf; - *ptr++ = state->LAST; - bytes--; - } else { - if (state->count > 0) { - if (bytes < 2) + /* end of line; flush the current run */ + if (state->count == 1 && (state->LAST < 0xc0)) { + if (bytes < 1 + padding) { return ptr - buf; - *ptr++ = 0xc0 | state->count; + } *ptr++ = state->LAST; - bytes -= 2; + bytes--; + } else { + if (state->count > 0) { + if (bytes < 2 + padding) { + return ptr - buf; + } + *ptr++ = 0xc0 | state->count; + *ptr++ = state->LAST; + bytes -= 2; + } + } + if (bytes < padding) { + return ptr - buf; + } + /* add the padding */ + for (i=0;ix < planes * bytes_per_line) { + state->count = 1; + state->LAST = state->buffer[state->x]; + state->x++; } } - /* read next line */ state->state = FETCH; break; - } + } } + diff --git a/selftest.py b/selftest.py index 1f905b9a7..248cb3937 100644 --- a/selftest.py +++ b/selftest.py @@ -192,6 +192,7 @@ if __name__ == "__main__": check_module("PIL CORE", "PIL._imaging") check_module("TKINTER", "PIL._imagingtk") check_codec("JPEG", "jpeg") + check_codec("JPEG 2000", "jpeg2k") check_codec("ZLIB (PNG/ZIP)", "zip") check_codec("LIBTIFF", "libtiff") check_module("FREETYPE2", "PIL._imagingft") diff --git a/setup.py b/setup.py index 784ef4915..b3d6195cd 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,8 @@ _LIB_IMAGING = ( "QuantHeap", "PcdDecode", "PcxDecode", "PcxEncode", "Point", "RankFilter", "RawDecode", "RawEncode", "Storage", "SunRleDecode", "TgaRleDecode", "Unpack", "UnpackYCC", "UnsharpMask", "XbmDecode", - "XbmEncode", "ZipDecode", "ZipEncode", "TiffDecode") + "XbmEncode", "ZipDecode", "ZipEncode", "TiffDecode", "Incremental", + "Jpeg2KDecode", "Jpeg2KEncode") def _add_directory(path, dir, where=None): @@ -82,14 +83,16 @@ def _read(file): try: import _tkinter -except ImportError: +except (ImportError, OSError): + # pypy emits an oserror _tkinter = None NAME = 'Pillow' -VERSION = '2.3.0' +VERSION = '2.4.0' TCL_ROOT = None JPEG_ROOT = None +JPEG2K_ROOT = None ZLIB_ROOT = None TIFF_ROOT = None FREETYPE_ROOT = None @@ -100,6 +103,7 @@ class pil_build_ext(build_ext): class feature: zlib = jpeg = tiff = freetype = tcl = tk = lcms = webp = webpmux = None + jpeg2000 = None required = [] def require(self, feat): @@ -152,7 +156,7 @@ class pil_build_ext(build_ext): # # add configured kits - for root in (TCL_ROOT, JPEG_ROOT, TIFF_ROOT, ZLIB_ROOT, + for root in (TCL_ROOT, JPEG_ROOT, JPEG2K_ROOT, TIFF_ROOT, ZLIB_ROOT, FREETYPE_ROOT, LCMS_ROOT): if isinstance(root, type(())): lib_root, include_root = root @@ -224,45 +228,50 @@ class pil_build_ext(build_ext): _add_directory(include_dirs, "/usr/X11/include") elif sys.platform.startswith("linux"): - for platform_ in (plat.processor(), plat.architecture()[0]): - - if not platform_: - continue - - if platform_ in ["x86_64", "64bit"]: - _add_directory(library_dirs, "/lib64") - _add_directory(library_dirs, "/usr/lib64") - _add_directory(library_dirs, "/usr/lib/x86_64-linux-gnu") - break - elif platform_ in ["i386", "i686", "32bit"]: - _add_directory(library_dirs, "/usr/lib/i386-linux-gnu") - break - elif platform_ in ["aarch64"]: - _add_directory(library_dirs, "/usr/lib64") - _add_directory(library_dirs, "/usr/lib/aarch64-linux-gnu") - break - elif platform_ in ["arm", "armv7l"]: - _add_directory(library_dirs, "/usr/lib/arm-linux-gnueabi") - break - elif platform_ in ["ppc64"]: - _add_directory(library_dirs, "/usr/lib64") - _add_directory(library_dirs, "/usr/lib/ppc64-linux-gnu") - _add_directory(library_dirs, "/usr/lib/powerpc64-linux-gnu") - break - elif platform_ in ["ppc"]: - _add_directory(library_dirs, "/usr/lib/ppc-linux-gnu") - _add_directory(library_dirs, "/usr/lib/powerpc-linux-gnu") - break - elif platform_ in ["s390x"]: - _add_directory(library_dirs, "/usr/lib64") - _add_directory(library_dirs, "/usr/lib/s390x-linux-gnu") - break - elif platform_ in ["s390"]: - _add_directory(library_dirs, "/usr/lib/s390-linux-gnu") - break + arch_tp = (plat.processor(), plat.architecture()[0]) + if arch_tp == ("x86_64","32bit"): + # 32 bit build on 64 bit machine. + _add_directory(library_dirs, "/usr/lib/i386-linux-gnu") else: - raise ValueError( - "Unable to identify Linux platform: `%s`" % platform_) + for platform_ in arch_tp: + + if not platform_: + continue + + if platform_ in ["x86_64", "64bit"]: + _add_directory(library_dirs, "/lib64") + _add_directory(library_dirs, "/usr/lib64") + _add_directory(library_dirs, "/usr/lib/x86_64-linux-gnu") + break + elif platform_ in ["i386", "i686", "32bit"]: + _add_directory(library_dirs, "/usr/lib/i386-linux-gnu") + break + elif platform_ in ["aarch64"]: + _add_directory(library_dirs, "/usr/lib64") + _add_directory(library_dirs, "/usr/lib/aarch64-linux-gnu") + break + elif platform_ in ["arm", "armv7l"]: + _add_directory(library_dirs, "/usr/lib/arm-linux-gnueabi") + break + elif platform_ in ["ppc64"]: + _add_directory(library_dirs, "/usr/lib64") + _add_directory(library_dirs, "/usr/lib/ppc64-linux-gnu") + _add_directory(library_dirs, "/usr/lib/powerpc64-linux-gnu") + break + elif platform_ in ["ppc"]: + _add_directory(library_dirs, "/usr/lib/ppc-linux-gnu") + _add_directory(library_dirs, "/usr/lib/powerpc-linux-gnu") + break + elif platform_ in ["s390x"]: + _add_directory(library_dirs, "/usr/lib64") + _add_directory(library_dirs, "/usr/lib/s390x-linux-gnu") + break + elif platform_ in ["s390"]: + _add_directory(library_dirs, "/usr/lib/s390-linux-gnu") + break + else: + raise ValueError( + "Unable to identify Linux platform: `%s`" % platform_) # XXX Kludge. Above /\ we brute force support multiarch. Here we # try Barry's more general approach. Afterward, something should @@ -323,6 +332,16 @@ class pil_build_ext(build_ext): _add_directory(library_dirs, "/usr/lib") _add_directory(include_dirs, "/usr/include") + # on Windows, look for the OpenJPEG libraries in the location that + # the official installed puts them + if sys.platform == "win32": + _add_directory(library_dirs, + os.path.join(os.environ.get("ProgramFiles", ""), + "OpenJPEG 2.0", "lib")) + _add_directory(include_dirs, + os.path.join(os.environ.get("ProgramFiles", ""), + "OpenJPEG 2.0", "include")) + # # insert new dirs *before* default libs, to avoid conflicts # between Python PYD stub libs and real libraries @@ -351,6 +370,11 @@ class pil_build_ext(build_ext): _find_library_file(self, "libjpeg")): feature.jpeg = "libjpeg" # alternative name + if feature.want('jpeg2000'): + if _find_include_file(self, "openjpeg-2.0/openjpeg.h"): + if _find_library_file(self, "openjp2"): + feature.jpeg2000 = "openjp2" + if feature.want('tiff'): if _find_library_file(self, "tiff"): feature.tiff = "tiff" @@ -432,6 +456,11 @@ class pil_build_ext(build_ext): if feature.jpeg: libs.append(feature.jpeg) defs.append(("HAVE_LIBJPEG", None)) + if feature.jpeg2000: + libs.append(feature.jpeg2000) + defs.append(("HAVE_OPENJPEG", None)) + if sys.platform == "win32": + defs.append(("OPJ_STATIC", None)) if feature.zlib: libs.append(feature.zlib) defs.append(("HAVE_LIBZ", None)) @@ -539,6 +568,7 @@ class pil_build_ext(build_ext): options = [ (feature.tcl and feature.tk, "TKINTER"), (feature.jpeg, "JPEG"), + (feature.jpeg2000, "OPENJPEG (JPEG2000)"), (feature.zlib, "ZLIB (PNG/ZIP)"), (feature.tiff, "LIBTIFF"), (feature.freetype, "FREETYPE2"),