Merge pull request #1938 from wiredfool/python-decoder

Pure Python Decoders - fix for MSP decoding

PIL/Image.py

@@ -208,6 +208,8 @@ MIME = {}
 SAVE = {}
 SAVE_ALL = {}
 EXTENSION = {}
+DECODERS = {}
+ENCODERS = {}
 
 # --------------------------------------------------------------------
 # Modes supported by this version
@@ -413,6 +415,11 @@ def _getdecoder(mode, decoder_name, args, extra=()):
     elif not isinstance(args, tuple):
         args = (args,)
 
+    try:
+        decoder = DECODERS[decoder_name]
+        return decoder(mode, *args + extra)
+    except KeyError:
+        pass
     try:
         # get decoder
         decoder = getattr(core, decoder_name + "_decoder")
@@ -430,6 +437,11 @@ def _getencoder(mode, encoder_name, args, extra=()):
     elif not isinstance(args, tuple):
         args = (args,)
 
+    try:
+        encoder = ENCODERS[encoder_name]
+        return encoder(mode, *args + extra)
+    except KeyError:
+        pass
     try:
         # get encoder
         encoder = getattr(core, encoder_name + "_encoder")
@@ -2603,6 +2615,33 @@ def registered_extensions():
         init()
     return EXTENSION
 
+def register_decoder(name, decoder):
+    """
+    Registers an image decoder.  This function should not be
+    used in application code.
+
+    :param name: The name of the decoder
+    :param decoder: A callable(mode, args) that returns an
+                    ImageFile.PyDecoder object
+
+    .. versionadded:: 4.1.0                
+    """
+    DECODERS[name] = decoder
+
+
+def register_encoder(name, encoder):
+    """
+    Registers an image encoder.  This function should not be
+    used in application code.
+
+    :param name: The name of the encoder
+    :param encoder: A callable(mode, args) that returns an
+                    ImageFile.PyEncoder object
+
+    .. versionadded:: 4.1.0
+    """
+    ENCODERS[name] = encoder
+
 
 # --------------------------------------------------------------------
 # Simple display support.  User code may override this.

PIL/ImageFile.py

@@ -193,10 +193,7 @@ class ImageFile(Image.Image):
                 decoder = Image._getdecoder(self.mode, decoder_name,
                                       args, self.decoderconfig)
                 seek(offset)
-                try:
+                decoder.setimage(self.im, extents)
-                    decoder.setimage(self.im, extents)
-                except ValueError:
-                    continue
                 if decoder.pulls_fd:
                     decoder.setfd(self.fp)
                     status, err_code = decoder.decode(b"")
@@ -520,3 +517,128 @@ def _safe_read(fp, size):
         data.append(block)
         size -= len(block)
     return b"".join(data)
+
+
+class PyCodecState(object):
+    def __init__(self):
+        self.xsize = 0
+        self.ysize = 0
+        self.xoff = 0
+        self.yoff = 0
+
+    def extents(self):
+        return (self.xoff, self.yoff,
+                self.xoff+self.xsize, self.yoff+self.ysize)
+
+class PyDecoder(object):
+    """
+    Python implementation of a format decoder. Override this class and
+    add the decoding logic in the `decode` method.
+
+    See :ref:`Writing Your Own File Decoder in Python<file-decoders-py>`
+    """
+
+    _pulls_fd = False
+
+    def __init__(self, mode, *args):
+        self.im = None
+        self.state = PyCodecState()
+        self.fd = None
+        self.mode = mode
+        self.init(args)
+
+    def init(self, args):
+        """
+        Override to perform decoder specific initialization
+
+        :param args: Array of args items from the tile entry
+        :returns: None
+        """
+        self.args = args
+    
+    @property
+    def pulls_fd(self):
+        return self._pulls_fd
+
+    def decode(self, buffer):
+        """
+        Override to perform the decoding process.
+
+        :param buffer: A bytes object with the data to be decoded.  If `handles_eof`
+             is set, then `buffer` will be empty and `self.fd` will be set.
+        :returns: A tuple of (bytes consumed, errcode). If finished with decoding
+             return <0 for the bytes consumed. Err codes are from `ERRORS`
+        """
+        raise NotImplementedError()
+
+    def cleanup(self):
+        """
+        Override to perform decoder specific cleanup
+
+        :returns: None
+        """
+        pass
+
+    def setfd(self, fd):
+        """
+        Called from ImageFile to set the python file-like object
+
+        :param fd: A python file-like object
+        :returns: None
+        """
+        self.fd = fd
+        
+    def setimage(self, im, extents=None):
+        """
+        Called from ImageFile to set the core output image for the decoder
+
+        :param im: A core image object
+        :param extents: a 4 tuple of (x0, y0, x1, y1) defining the rectangle
+            for this tile
+        :returns: None
+        """
+        
+        # following c code
+        self.im = im
+
+        if extents:
+            (x0, y0, x1, y1) = extents
+        else:
+            (x0, y0, x1, y1) = (0, 0, 0, 0)
+
+        
+        if x0 == 0 and x1 == 0:
+            self.state.xsize, self.state.ysize = self.im.size
+        else:
+            self.state.xoff = x0
+            self.state.yoff = y0
+            self.state.xsize = x1 - x0
+            self.state.ysize = y1 - y0
+
+        if self.state.xsize <= 0 or self.state.ysize <= 0:
+            raise ValueError("Size cannot be negative")
+        
+        if (self.state.xsize + self.state.xoff > self.im.size[0] or
+            self.state.ysize + self.state.yoff > self.im.size[1]):
+            raise ValueError("Tile cannot extend outside image")
+           
+    def set_as_raw(self, data, rawmode=None):
+        """
+        Convenience method to set the internal image from a stream of raw data
+
+        :param data: Bytes to be set
+        :param rawmode: The rawmode to be used for the decoder. If not specified,
+             it will default to the mode of the image
+        :returns: None
+        """
+        
+        if not rawmode:
+            rawmode = self.mode
+        d = Image._getdecoder(self.mode, 'raw', (rawmode))
+        d.setimage(self.im, self.state.extents())
+        s = d.decode(data)
+                
+        if s[0] >= 0:
+            raise ValueError("not enough image data")
+        if s[1] != 0:
+            raise ValueError("cannot decode image data")

PIL/MspImagePlugin.py

@@ -1,6 +1,5 @@
 #
 # The Python Imaging Library.
-# $Id$
 #
 # MSP file handling
 #
@@ -9,16 +8,24 @@
 # History:
 #       95-09-05 fl     Created
 #       97-01-03 fl     Read/write MSP images
+#       17-02-21 es     Fixed RLE interpretation
 #
 # Copyright (c) Secret Labs AB 1997.
 # Copyright (c) Fredrik Lundh 1995-97.
+# Copyright (c) Eric Soroos 2017.
 #
 # See the README file for information on usage and redistribution.
 #
-
+# More info on this format: https://archive.org/details/gg243631
+# Page 313:
+# Figure 205. Windows Paint Version 1: "DanM" Format
+# Figure 206. Windows Paint Version 2: "LinS" Format. Used in Windows V2.03
+#
+# See also: http://www.fileformat.info/format/mspaint/egff.htm
 
 from . import Image, ImageFile
-from ._binary import i16le as i16, o16le as o16
+from ._binary import i16le as i16, o16le as o16, i8
+import struct, io
 
 __version__ = "0.1"
 
@@ -60,7 +67,90 @@ class MspImageFile(ImageFile.ImageFile):
         if s[:4] == b"DanM":
             self.tile = [("raw", (0, 0)+self.size, 32, ("1", 0, 1))]
         else:
-            self.tile = [("msp", (0, 0)+self.size, 32+2*self.size[1], None)]
+            self.tile = [("MSP", (0, 0)+self.size, 32, None)]
+
+
+class MspDecoder(ImageFile.PyDecoder):
+    # The algo for the MSP decoder is from
+    # http://www.fileformat.info/format/mspaint/egff.htm
+    # cc-by-attribution -- That page references is taken from the
+    # Encyclopedia of Graphics File Formats and is licensed by
+    # O'Reilly under the Creative Common/Attribution license
+    #
+    # For RLE encoded files, the 32byte header is followed by a scan
+    # line map, encoded as one 16bit word of encoded byte length per
+    # line.
+    #
+    # NOTE: the encoded length of the line can be 0. This was not
+    # handled in the previous version of this encoder, and there's no
+    # mention of how to handle it in the documentation. From the few
+    # examples I've seen, I've assumed that it is a fill of the
+    # background color, in this case, white.
+    #
+    #
+    # Pseudocode of the decoder:
+    # Read a BYTE value as the RunType
+    #  If the RunType value is zero
+    #   Read next byte as the RunCount
+    #   Read the next byte as the RunValue
+    #   Write the RunValue byte RunCount times
+    #  If the RunType value is non-zero
+    #   Use this value as the RunCount
+    #   Read and write the next RunCount bytes literally
+    # 
+    #  e.g.:
+    #  0x00 03 ff 05 00 01 02 03 04
+    #  would yield the bytes:
+    #  0xff ff ff 00 01 02 03 04
+    #
+    # which are then interpreted as a bit packed mode '1' image
+
+    
+    _pulls_fd = True
+
+    def decode(self, buffer):
+        
+        img = io.BytesIO()
+        blank_line = bytearray((0xff,)*((self.state.xsize+7)//8))
+        try:
+            self.fd.seek(32)
+            rowmap = struct.unpack_from("<%dH" % (self.state.ysize),
+                                        self.fd.read(self.state.ysize*2))
+        except struct.error:
+            raise IOError("Truncated MSP file in row map")
+
+        for x, rowlen in enumerate(rowmap):
+            try:
+                if rowlen == 0:
+                    img.write(blank_line)
+                    continue
+                row = self.fd.read(rowlen)
+                if len(row) != rowlen:
+                    raise IOError("Truncated MSP file, expected %d bytes on row %s",
+                                  (rowlen, x))
+                idx = 0
+                while idx < rowlen:
+                    runtype = i8(row[idx])
+                    idx += 1
+                    if runtype == 0:
+                        (runcount, runval) = struct.unpack("Bc", row[idx:idx+2])
+                        img.write(runval * runcount)
+                        idx += 2
+                    else:
+                        runcount = runtype
+                        img.write(row[idx:idx+runcount])
+                        idx += runcount
+                        
+            except struct.error:
+                raise IOError("Corrupted MSP file in row %d" %x)
+            
+        self.set_as_raw(img.getvalue(), ("1", 0, 1))
+                
+        return 0,0
+
+
+Image.register_decoder('MSP', MspDecoder)
+   
 
 #
 # write MSP files (uncompressed only)
@@ -92,6 +182,7 @@ def _save(im, fp, filename):
     # image body
     ImageFile._save(im, fp, [("raw", (0, 0)+im.size, 32, ("1", 0, 1))])
 
+
 #
 # registry
 

Tests/test_file_msp.py

@@ -2,7 +2,11 @@ from helper import unittest, PillowTestCase, hopper
 
 from PIL import Image, MspImagePlugin
 
+import os
+
 TEST_FILE = "Tests/images/hopper.msp"
+EXTRA_DIR = "Tests/images/picins"
+YA_EXTRA_DIR = "Tests/images/msp"
 
 
 class TestFileMsp(PillowTestCase):
@@ -24,14 +28,48 @@ class TestFileMsp(PillowTestCase):
         self.assertRaises(SyntaxError,
                           lambda: MspImagePlugin.MspImageFile(invalid_file))
 
-    def test_open(self):
+    def test_bad_checksum(self):
+        # Arrange
+        # This was created by forcing Pillow to save with checksum=0
+        bad_checksum = "Tests/images/hopper_bad_checksum.msp"
+
+        # Act / Assert
+        self.assertRaises(SyntaxError,
+                          lambda: MspImagePlugin.MspImageFile(bad_checksum))
+
+    def test_open_windows_v1(self):
         # Arrange
         # Act
         im = Image.open(TEST_FILE)
 
         # Assert
-        self.assertEqual(im.size, (128, 128))
+        self.assert_image_equal(im, hopper("1"))
-        self.assert_image_equal(im, hopper("1"), 4)
+        self.assertIsInstance(im, MspImagePlugin.MspImageFile)
+
+    def _assert_file_image_equal(self, source_path, target_path):
+        with Image.open(source_path) as im:
+            target = Image.open(target_path)
+            self.assert_image_equal(im, target)
+
+    @unittest.skipIf(not os.path.exists(EXTRA_DIR),
+                     "Extra image files not installed")
+    def test_open_windows_v2(self):
+
+        files = (os.path.join(EXTRA_DIR, f) for f in os.listdir(EXTRA_DIR)
+                 if os.path.splitext(f)[1] == '.msp')
+        for path in files:
+            self._assert_file_image_equal(path,
+                                          path.replace('.msp','.png'))
+
+    @unittest.skipIf(not os.path.exists(YA_EXTRA_DIR),
+                     "Even More Extra image files not installed")
+    def test_msp_v2(self):
+        for f in os.listdir(YA_EXTRA_DIR):
+            if not '.MSP' in f: continue
+            path = os.path.join(YA_EXTRA_DIR, f)
+            self._assert_file_image_equal(path,
+                                          path.replace('.MSP','.png'))
+
 
     def test_cannot_save_wrong_mode(self):
         # Arrange

Tests/test_image.py

@@ -375,5 +375,30 @@ class TestImage(PillowTestCase):
             self.assert_image_equal(im, target)
 
 
+class MockEncoder(object):pass
+
+def mock_encode(*args):
+    encoder = MockEncoder()
+    encoder.args = args
+    return encoder
+
+class TestRegistry(PillowTestCase):
+
+    def test_encode_registry(self):
+
+        Image.register_encoder('MOCK', mock_encode)
+        self.assert_('MOCK' in Image.ENCODERS)
+
+        enc = Image._getencoder('RGB', 'MOCK', ('args',), extra=('extra',))
+
+        self.assertIsInstance(enc, MockEncoder)
+        self.assertEqual(enc.args, ('RGB', 'args', 'extra'))
+
+    def test_encode_registry_fail(self):
+        self.assertRaises(IOError, lambda: Image._getencoder('RGB',
+                                                             'DoesNotExist',
+                                                             ('args',),
+                                                             extra=('extra',)))
+
 if __name__ == '__main__':
     unittest.main()

Tests/test_imagefile.py

@@ -135,5 +135,84 @@ class TestImageFile(PillowTestCase):
         finally:
             ImageFile.LOAD_TRUNCATED_IMAGES = False
 
+
+class MockPyDecoder(ImageFile.PyDecoder):
+    def decode(self, buffer):
+        #eof
+        return (-1, 0)
+
+xoff, yoff, xsize, ysize = 10, 20, 100, 100
+class MockImageFile(ImageFile.ImageFile):
+    def _open(self):
+        self.rawmode = 'RGBA'
+        self.mode = 'RGBA'
+        self.size = (200, 200)
+        self.tile = [("MOCK", (xoff, yoff, xoff+xsize, yoff+ysize), 32, None)]
+
+class TestPyDecoder(PillowTestCase):
+
+    def get_decoder(self):
+        decoder = MockPyDecoder(None)
+
+        def closure(mode, *args):
+            decoder.__init__(mode, *args)
+            return decoder
+
+        Image.register_decoder('MOCK', closure)
+        return decoder
+
+    def test_setimage(self):
+        buf = BytesIO(b'\x00'*255)
+
+        im = MockImageFile(buf)
+        d = self.get_decoder()
+
+        im.load()
+
+        self.assertEqual(d.state.xoff, xoff)
+        self.assertEqual(d.state.yoff, yoff)
+        self.assertEqual(d.state.xsize, xsize)
+        self.assertEqual(d.state.ysize, ysize)
+
+        self.assertRaises(ValueError, lambda: d.set_as_raw(b'\x00'))
+
+    def test_extents_none(self):
+        buf = BytesIO(b'\x00'*255)
+
+        im = MockImageFile(buf)
+        im.tile = [("MOCK", None, 32, None)]
+        d = self.get_decoder()
+
+        im.load()
+
+        self.assertEqual(d.state.xoff, 0)
+        self.assertEqual(d.state.yoff, 0)
+        self.assertEqual(d.state.xsize, 200)
+        self.assertEqual(d.state.ysize, 200)
+
+    def test_negsize(self):
+        buf = BytesIO(b'\x00'*255)
+
+        im = MockImageFile(buf)
+        im.tile = [("MOCK", (xoff, yoff, -10, yoff+ysize), 32, None)]
+        d = self.get_decoder()
+
+        self.assertRaises(ValueError, im.load)
+
+        im.tile = [("MOCK", (xoff, yoff, xoff+xsize, -10), 32, None)]
+        self.assertRaises(ValueError, im.load)
+
+    def test_oversize(self):
+        buf = BytesIO(b'\x00'*255)
+
+        im = MockImageFile(buf)
+        im.tile = [("MOCK", (xoff, yoff, xoff+xsize + 100, yoff+ysize), 32, None)]
+        d = self.get_decoder()
+
+        self.assertRaises(ValueError, im.load)
+
+        im.tile = [("MOCK", (xoff, yoff, xoff+xsize, yoff+ysize + 100), 32, None)]
+        self.assertRaises(ValueError, im.load)
+
 if __name__ == '__main__':
     unittest.main()

_imaging.c

@@ -3290,7 +3290,6 @@ extern PyObject* PyImaging_JpegDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_Jpeg2KDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_TiffLzwDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_LibTiffDecoderNew(PyObject* self, PyObject* args);
-extern PyObject* PyImaging_MspDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_PackbitsDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_PcdDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_PcxDecoderNew(PyObject* self, PyObject* args);
@@ -3368,7 +3367,6 @@ static PyMethodDef functions[] = {
     {"libtiff_decoder", (PyCFunction)PyImaging_LibTiffDecoderNew, 1},
     {"libtiff_encoder", (PyCFunction)PyImaging_LibTiffEncoderNew, 1},
 #endif
-    {"msp_decoder", (PyCFunction)PyImaging_MspDecoderNew, 1},
     {"packbits_decoder", (PyCFunction)PyImaging_PackbitsDecoderNew, 1},
     {"pcd_decoder", (PyCFunction)PyImaging_PcdDecoderNew, 1},
     {"pcx_decoder", (PyCFunction)PyImaging_PcxDecoderNew, 1},

decode.c

@@ -557,27 +557,6 @@ PyImaging_LibTiffDecoderNew(PyObject* self, PyObject* args)
 
 #endif
 
-/* -------------------------------------------------------------------- */
-/* MSP                                                                  */
-/* -------------------------------------------------------------------- */
-
-PyObject*
-PyImaging_MspDecoderNew(PyObject* self, PyObject* args)
-{
-    ImagingDecoderObject* decoder;
-
-    decoder = PyImaging_DecoderNew(0);
-    if (decoder == NULL)
-        return NULL;
-
-    if (get_unpacker(decoder, "1", "1") < 0)
-        return NULL;
-
-    decoder->decode = ImagingMspDecode;
-
-    return (PyObject*) decoder;
-}
-
 
 /* -------------------------------------------------------------------- */
 /* PackBits                                                             */

depends/install_extra_test_images.sh

@@ -1,11 +1,9 @@
 #!/bin/bash
 # install extra test images
 
-if [ ! -f test_images.tar.gz ]; then
-    wget -O 'test_images.tar.gz' 'https://github.com/python-pillow/pillow-depends/blob/master/test_images.tar.gz?raw=true'
-fi
-
 rm -r test_images
-tar -xvzf test_images.tar.gz
+
+# Use SVN to just fetch a single git subdirectory
+svn checkout https://github.com/python-pillow/pillow-depends/trunk/test_images
 
 cp -r test_images/* ../Tests/images

docs/example/DdsImagePlugin.py

@@ -0,0 +1,286 @@
+"""
+A Pillow loader for .dds files (S3TC-compressed aka DXTC)
+Jerome Leclanche <jerome@leclan.ch>
+
+Documentation:
+  http://oss.sgi.com/projects/ogl-sample/registry/EXT/texture_compression_s3tc.txt
+
+The contents of this file are hereby released in the public domain (CC0)
+Full text of the CC0 license:
+  https://creativecommons.org/publicdomain/zero/1.0/
+"""
+
+import struct
+from io import BytesIO
+from PIL import Image, ImageFile
+
+
+# Magic ("DDS ")
+DDS_MAGIC = 0x20534444
+
+# DDS flags
+DDSD_CAPS = 0x1
+DDSD_HEIGHT = 0x2
+DDSD_WIDTH = 0x4
+DDSD_PITCH = 0x8
+DDSD_PIXELFORMAT = 0x1000
+DDSD_MIPMAPCOUNT = 0x20000
+DDSD_LINEARSIZE = 0x80000
+DDSD_DEPTH = 0x800000
+
+# DDS caps
+DDSCAPS_COMPLEX = 0x8
+DDSCAPS_TEXTURE = 0x1000
+DDSCAPS_MIPMAP = 0x400000
+
+DDSCAPS2_CUBEMAP = 0x200
+DDSCAPS2_CUBEMAP_POSITIVEX = 0x400
+DDSCAPS2_CUBEMAP_NEGATIVEX = 0x800
+DDSCAPS2_CUBEMAP_POSITIVEY = 0x1000
+DDSCAPS2_CUBEMAP_NEGATIVEY = 0x2000
+DDSCAPS2_CUBEMAP_POSITIVEZ = 0x4000
+DDSCAPS2_CUBEMAP_NEGATIVEZ = 0x8000
+DDSCAPS2_VOLUME = 0x200000
+
+# Pixel Format
+DDPF_ALPHAPIXELS = 0x1
+DDPF_ALPHA = 0x2
+DDPF_FOURCC = 0x4
+DDPF_PALETTEINDEXED8 = 0x20
+DDPF_RGB = 0x40
+DDPF_LUMINANCE = 0x20000
+
+
+# dds.h
+
+DDS_FOURCC = DDPF_FOURCC
+DDS_RGB = DDPF_RGB
+DDS_RGBA = DDPF_RGB | DDPF_ALPHAPIXELS
+DDS_LUMINANCE = DDPF_LUMINANCE
+DDS_LUMINANCEA = DDPF_LUMINANCE | DDPF_ALPHAPIXELS
+DDS_ALPHA = DDPF_ALPHA
+DDS_PAL8 = DDPF_PALETTEINDEXED8
+
+DDS_HEADER_FLAGS_TEXTURE = (DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH |
+                            DDSD_PIXELFORMAT)
+DDS_HEADER_FLAGS_MIPMAP = DDSD_MIPMAPCOUNT
+DDS_HEADER_FLAGS_VOLUME = DDSD_DEPTH
+DDS_HEADER_FLAGS_PITCH = DDSD_PITCH
+DDS_HEADER_FLAGS_LINEARSIZE = DDSD_LINEARSIZE
+
+DDS_HEIGHT = DDSD_HEIGHT
+DDS_WIDTH = DDSD_WIDTH
+
+DDS_SURFACE_FLAGS_TEXTURE = DDSCAPS_TEXTURE
+DDS_SURFACE_FLAGS_MIPMAP = DDSCAPS_COMPLEX | DDSCAPS_MIPMAP
+DDS_SURFACE_FLAGS_CUBEMAP = DDSCAPS_COMPLEX
+
+DDS_CUBEMAP_POSITIVEX = DDSCAPS2_CUBEMAP | DDSCAPS2_CUBEMAP_POSITIVEX
+DDS_CUBEMAP_NEGATIVEX = DDSCAPS2_CUBEMAP | DDSCAPS2_CUBEMAP_NEGATIVEX
+DDS_CUBEMAP_POSITIVEY = DDSCAPS2_CUBEMAP | DDSCAPS2_CUBEMAP_POSITIVEY
+DDS_CUBEMAP_NEGATIVEY = DDSCAPS2_CUBEMAP | DDSCAPS2_CUBEMAP_NEGATIVEY
+DDS_CUBEMAP_POSITIVEZ = DDSCAPS2_CUBEMAP | DDSCAPS2_CUBEMAP_POSITIVEZ
+DDS_CUBEMAP_NEGATIVEZ = DDSCAPS2_CUBEMAP | DDSCAPS2_CUBEMAP_NEGATIVEZ
+
+
+# DXT1
+DXT1_FOURCC = 0x31545844
+
+# DXT3
+DXT3_FOURCC = 0x33545844
+
+# DXT5
+DXT5_FOURCC = 0x35545844
+
+
+def _decode565(bits):
+    a = ((bits >> 11) & 0x1f) << 3
+    b = ((bits >> 5) & 0x3f) << 2
+    c = (bits & 0x1f) << 3
+    return a, b, c
+
+
+def _c2a(a, b):
+    return (2 * a + b) // 3
+
+
+def _c2b(a, b):
+    return (a + b) // 2
+
+
+def _c3(a, b):
+    return (2 * b + a) // 3
+
+
+def _dxt1(data, width, height):
+    # TODO implement this function as pixel format in decode.c
+    ret = bytearray(4 * width * height)
+
+    for y in range(0, height, 4):
+        for x in range(0, width, 4):
+            color0, color1, bits = struct.unpack("<HHI", data.read(8))
+
+            r0, g0, b0 = _decode565(color0)
+            r1, g1, b1 = _decode565(color1)
+
+            # Decode this block into 4x4 pixels
+            for j in range(4):
+                for i in range(4):
+                    # get next control op and generate a pixel
+                    control = bits & 3
+                    bits = bits >> 2
+                    if control == 0:
+                        r, g, b = r0, g0, b0
+                    elif control == 1:
+                        r, g, b = r1, g1, b1
+                    elif control == 2:
+                        if color0 > color1:
+                            r, g, b = _c2a(r0, r1), _c2a(g0, g1), _c2a(b0, b1)
+                        else:
+                            r, g, b = _c2b(r0, r1), _c2b(g0, g1), _c2b(b0, b1)
+                    elif control == 3:
+                        if color0 > color1:
+                            r, g, b = _c3(r0, r1), _c3(g0, g1), _c3(b0, b1)
+                        else:
+                            r, g, b = 0, 0, 0
+
+                    idx = 4 * ((y + j) * width + (x + i))
+                    ret[idx:idx+4] = struct.pack('4B', r, g, b, 255)
+
+    return bytes(ret)
+
+
+def _dxtc_alpha(a0, a1, ac0, ac1, ai):
+    if ai <= 12:
+        ac = (ac0 >> ai) & 7
+    elif ai == 15:
+        ac = (ac0 >> 15) | ((ac1 << 1) & 6)
+    else:
+        ac = (ac1 >> (ai - 16)) & 7
+
+    if ac == 0:
+        alpha = a0
+    elif ac == 1:
+        alpha = a1
+    elif a0 > a1:
+        alpha = ((8 - ac) * a0 + (ac - 1) * a1) // 7
+    elif ac == 6:
+        alpha = 0
+    elif ac == 7:
+        alpha = 0xff
+    else:
+        alpha = ((6 - ac) * a0 + (ac - 1) * a1) // 5
+
+    return alpha
+
+
+def _dxt5(data, width, height):
+    # TODO implement this function as pixel format in decode.c
+    ret = bytearray(4 * width * height)
+
+    for y in range(0, height, 4):
+        for x in range(0, width, 4):
+            a0, a1, ac0, ac1, c0, c1, code = struct.unpack("<2BHI2HI",
+                                                           data.read(16))
+
+            r0, g0, b0 = _decode565(c0)
+            r1, g1, b1 = _decode565(c1)
+
+            for j in range(4):
+                for i in range(4):
+                    ai = 3 * (4 * j + i)
+                    alpha = _dxtc_alpha(a0, a1, ac0, ac1, ai)
+
+                    cc = (code >> 2 * (4 * j + i)) & 3
+                    if cc == 0:
+                        r, g, b = r0, g0, b0
+                    elif cc == 1:
+                        r, g, b = r1, g1, b1
+                    elif cc == 2:
+                        r, g, b = _c2a(r0, r1), _c2a(g0, g1), _c2a(b0, b1)
+                    elif cc == 3:
+                        r, g, b = _c3(r0, r1), _c3(g0, g1), _c3(b0, b1)
+
+                    idx = 4 * ((y + j) * width + (x + i))
+                    ret[idx:idx+4] = struct.pack('4B', r, g, b, alpha)
+
+    return bytes(ret)
+
+
+class DdsImageFile(ImageFile.ImageFile):
+    format = "DDS"
+    format_description = "DirectDraw Surface"
+
+    def _open(self):
+        magic, header_size = struct.unpack("<II", self.fp.read(8))
+        if header_size != 124:
+            raise IOError("Unsupported header size %r" % (header_size))
+        header_bytes = self.fp.read(header_size - 4)
+        if len(header_bytes) != 120:
+            raise IOError("Incomplete header: %s bytes" % len(header_bytes))
+        header = BytesIO(header_bytes)
+
+        flags, height, width = struct.unpack("<3I", header.read(12))
+        self.size = (width, height)
+        self.mode = "RGBA"
+
+        pitch, depth, mipmaps = struct.unpack("<3I", header.read(12))
+        reserved = struct.unpack("<11I", header.read(44))
+
+        # pixel format
+        pfsize, pfflags = struct.unpack("<2I", header.read(8))
+        fourcc = header.read(4)
+        bitcount, rmask, gmask, bmask, amask = struct.unpack("<5I",
+                                                             header.read(20))
+
+
+        if fourcc == b"DXT1":
+            self.decoder = "DXT1"
+            codec = _dxt1
+        elif fourcc == b"DXT5":
+            self.decoder = "DXT5"
+            codec = _dxt5
+        else:
+            raise NotImplementedError("Unimplemented pixel format %r" %
+                                      (fourcc))
+
+        self.tile = [
+            (self.decoder, (0, 0) + self.size, 0, (self.mode, 0, 1))
+        ]
+
+        
+    def load_seek(self, pos):
+        pass
+
+
+class DXT1Decoder(ImageFile.PyDecoder):
+    _pulls_fd = True
+
+    def decode(self, buffer):
+        try:
+            self.set_as_raw(_dxt1(self.fd, self.state.xsize, self.state.ysize))
+        except struct.error:
+            raise IOError("Truncated DDS file")
+         
+        return 0,0
+
+class DXT5Decoder(ImageFile.PyDecoder):
+    _pulls_fd = True
+
+    def decode(self, buffer):
+        try:
+            self.set_as_raw(_dxt5(self.fd, self.state.xsize, self.state.ysize))
+        except struct.error:
+            raise IOError("Truncated DDS file")
+        return 0,0
+
+Image.register_decoder('DXT1', DXT1Decoder)
+Image.register_decoder('DXT5', DXT5Decoder)
+
+
+def _validate(prefix):
+    return prefix[:4] == b"DDS "
+
+
+Image.register_open(DdsImageFile.format, DdsImageFile, _validate)
+Image.register_extension(DdsImageFile.format, ".dds")

docs/handbook/writing-your-own-file-decoder.rst

@@ -9,7 +9,10 @@ itself. Such plug-ins usually have names like
 :file:`XxxImagePlugin.py`, where ``Xxx`` is a unique format name
 (usually an abbreviation).
 
-.. warning:: Pillow >= 2.1.0 no longer automatically imports any file in the Python path with a name ending in :file:`ImagePlugin.py`.  You will need to import your image plugin manually.
+.. warning:: Pillow >= 2.1.0 no longer automatically imports any file
+             in the Python path with a name ending in
+             :file:`ImagePlugin.py`.  You will need to import your
+             image plugin manually.
 
 Pillow decodes files in 2 stages:
 
@@ -23,21 +26,19 @@ Pillow decodes files in 2 stages:
    called, which sets up a decoder for each tile and feeds the data to
    it.
 
-A decoder plug-in should contain a decoder class, based on the
+An image plug-in should contain a format handler derived from the
-:py:class:`PIL.ImageFile.ImageFile` base class. This class should provide an
+:py:class:`PIL.ImageFile.ImageFile` base class. This class should
-:py:meth:`_open` method, which reads the file header and sets up at least the
+provide an :py:meth:`_open` method, which reads the file header and
-:py:attr:`~PIL.Image.Image.mode` and :py:attr:`~PIL.Image.Image.size`
+sets up at least the :py:attr:`~PIL.Image.Image.mode` and
-attributes. To be able to load the file, the method must also create a list of
+:py:attr:`~PIL.Image.Image.size` attributes. To be able to load the
-:py:attr:`tile` descriptors. The class must be explicitly registered, via a
+file, the method must also create a list of :py:attr:`tile`
-call to the :py:mod:`~PIL.Image` module.
+descriptors, which contain a decoder name, extents of the tile, and
+any decoder-specific data. The format handler class must be explicitly
+registered, via a call to the :py:mod:`~PIL.Image` module.
 
-For performance reasons, it is important that the :py:meth:`_open` method
+.. note:: For performance reasons, it is important that the
-quickly rejects files that do not have the appropriate contents.
+  :py:meth:`_open` method quickly rejects files that do not have the
-
+  appropriate contents.
-The ``raw`` decoder is useful for uncompressed image formats, but many
-formats require more control of the decoding context, either with a
-decoder written in ``C`` or by linking in an external library to do
-the decoding. (Examples of this include PNG, Tiff, and Jpeg support)
 
 Example
 -------
@@ -94,12 +95,12 @@ true color.
 The format handler must always set the
 :py:attr:`~PIL.Image.Image.size` and :py:attr:`~PIL.Image.Image.mode`
 attributes. If these are not set, the file cannot be opened. To
-simplify the decoder, the calling code considers exceptions like
+simplify the plugin, the calling code considers exceptions like
 :py:exc:`SyntaxError`, :py:exc:`KeyError`, :py:exc:`IndexError`,
 :py:exc:`EOFError` and :py:exc:`struct.error` as a failure to identify
 the file.
 
-Note that the decoder must be explicitly registered using
+Note that the image plugin must be explicitly registered using
 :py:func:`PIL.Image.register_open`. Although not required, it is also a good
 idea to register any extensions used by this format.
 
@@ -137,6 +138,9 @@ The fields are used as follows:
 Note that the :py:attr:`tile` attribute contains a list of tile descriptors,
 not just a single descriptor.
 
+Decoders
+========
+
 The raw decoder
 ---------------
 
@@ -304,13 +308,14 @@ The fields are used as follows:
 
 .. _file-decoders:
 
-Writing Your Own File Decoder
+Writing Your Own File Decoder in C
-=============================
+==================================
 
 There are 3 stages in a file decoder's lifetime:
 
-1. Setup: Pillow looks for a function named ``[decodername]_decoder``
+1. Setup: Pillow looks for a function in the decoder registry, falling
-   on the internal core image object.  That function is called with the ``args`` tuple
+   back to a function named ``[decodername]_decoder`` on the internal
+   core image object.  That function is called with the ``args`` tuple
    from the ``tile`` setup in the ``_open`` method.
 
 2. Decoding: The decoder's decode function is repeatedly called with
@@ -349,9 +354,6 @@ interest in this object are:
   member is an opaque struct that can be used by the decoder to store
   any format specific state or options.
 
-**handles_eof**
-  UNDONE, set if your code handles EOF errors.
-
 **pulls_fd**
   **EXPERIMENTAL** -- **WARNING**, interface may change. If set to 1,
   ``state->fd`` will be a pointer to the Python file like object.  The
@@ -391,3 +393,25 @@ The cleanup function is called after the decoder returns a negative
 value, or if there is a read error from the file. This function should
 free any allocated memory and release any resources from external
 libraries.
+
+.. _file-decoders-py:
+
+Writing Your Own File Decoder in Python
+=======================================
+
+Python file decoders should derive from
+:py:class:`PIL.ImageFile.PyDecoder` and should at least override the
+decode method. File decoders should be registered using
+:py:meth:`PIL.Image.register_decoder`. As in the C implementation of
+the file decoders, there are three stages in the lifetime of a
+Python-based file decoder:
+
+1. Setup: Pillow looks for the decoder in the registry, then
+   instantiates the class.
+
+2. Decoding: The decoder instance's ``decode`` method is repeatedly
+   called with a buffer of data to be interpreted.
+
+3. Cleanup: The decoder instance's ``cleanup`` method is called.
+
+

docs/reference/ImageFile.rst

@@ -19,7 +19,7 @@ Example: Parse an image
 
     from PIL import ImageFile
 
-    fp = open("lena.pgm", "rb")
+    fp = open("hopper.pgm", "rb")
 
     p = ImageFile.Parser()
 
@@ -39,3 +39,9 @@ Example: Parse an image
 
 .. autoclass:: PIL.ImageFile.Parser()
     :members:
+
+:py:class:`~PIL.ImageFile.PyDecoder`
+------------------------------------
+
+.. autoclass:: PIL.ImageFile.PyDecoder()
+    :members:

libImaging/MspDecode.c

@@ -1,91 +0,0 @@
-/*
- * The Python Imaging Library.
- * $Id$
- *
- * decoder for MSP version 2 data.
- *
- * history:
- *	97-01-03 fl	Created
- *
- * Copyright (c) Fredrik Lundh 1997.
- * Copyright (c) Secret Labs AB 1997.
- *
- * See the README file for information on usage and redistribution.
- */
-
-
-#include "Imaging.h"
-
-
-int
-ImagingMspDecode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes)
-{
-    int n;
-    UINT8* ptr;
-
-    ptr = buf;
-
-    for (;;) {
-
-	if (bytes < 1)
-	    return ptr - buf;
-
-	if (ptr[0] == 0) {
-
-	    /* Run (3 bytes block) */
-	    if (bytes < 3)
-		break;
-
-	    n = ptr[1];
-
-	    if (state->x + n > state->bytes) {
-		state->errcode = IMAGING_CODEC_OVERRUN;
-		return -1;
-	    }
-
-	    memset(state->buffer + state->x, ptr[2], n);
-
-	    ptr += 3;
-	    bytes -= 3;
-
-	} else {
-
-	    /* Literal (1+n bytes block) */
-	    n = ptr[0];
-
-	    if (bytes < 1 + n)
-		break;
-
-	    if (state->x + n > state->bytes) {
-		state->errcode = IMAGING_CODEC_OVERRUN;
-		return -1;
-	    }
-
-	    memcpy(state->buffer + state->x, ptr + 1, n);
-
-	    ptr += 1 + n;
-	    bytes -= 1 + n;
-
-	}
-
-	state->x += n;
-
-	if (state->x >= state->bytes) {
-
-	    /* Got a full line, unpack it */
-	    state->shuffle((UINT8*) im->image[state->y + state->yoff] +
-			   state->xoff * im->pixelsize, state->buffer,
-			   state->xsize);
-
-	    state->x = 0;
-
-	    if (++state->y >= state->ysize) {
-		/* End of file (errcode = 0) */
-		return -1;
-	    }
-	}
-
-    }
-
-    return ptr - buf;
-}

setup.py

@@ -32,7 +32,7 @@ _LIB_IMAGING = (
     "Draw", "Effects", "EpsEncode", "File", "Fill", "Filter", "FliDecode",
     "Geometry", "GetBBox", "GifDecode", "GifEncode", "HexDecode", "Histo",
     "JpegDecode", "JpegEncode", "LzwDecode", "Matrix", "ModeFilter",
-    "MspDecode", "Negative", "Offset", "Pack", "PackDecode", "Palette", "Paste",
+    "Negative", "Offset", "Pack", "PackDecode", "Palette", "Paste",
     "Quant", "QuantOctree", "QuantHash", "QuantHeap", "PcdDecode", "PcxDecode",
     "PcxEncode", "Point", "RankFilter", "RawDecode", "RawEncode", "Storage",
     "SunRleDecode", "TgaRleDecode", "Unpack", "UnpackYCC", "UnsharpMask",