Merge pull request #2769 from wiredfool/pr_2638

Update to #2638, add 16bit/rle support to SgiImageFile

MANIFEST.in

@@ -21,6 +21,7 @@ prune docs/_static
 
 # build/src control detritus
 exclude .coveragerc
+exclude codecov.yml
 exclude .editorconfig
 exclude .landscape.yaml
 exclude .travis

PIL/SgiImagePlugin.py

@@ -9,6 +9,7 @@
 #
 #
 # History:
+# 2017-22-07 mb   Add RLE decompression
 # 2016-16-10 mb   Add save method without compression
 # 1995-09-10 fl   Created
 #
@@ -22,9 +23,11 @@
 
 
 from . import Image, ImageFile
-from ._binary import i8, o8, i16be as i16
+from ._binary import i8, o8, i16be as i16, o16be as o16
 import struct
 import os
+import sys
+
 
 __version__ = "0.3"
 
@@ -33,9 +36,20 @@ def _accept(prefix):
     return len(prefix) >= 2 and i16(prefix) == 474
 
 
+MODES = {
+    (1, 1, 1): "L",
+    (1, 2, 1): "L",
+    (2, 1, 1): "L;16B",
+    (2, 2, 1): "L;16B",
+    (1, 3, 3): "RGB",
+    (2, 3, 3): "RGB;16B",
+    (1, 3, 4): "RGBA",
+    (2, 3, 4): "RGBA;16B"
+}
+
+
 ##
 # Image plugin for SGI images.
-
 class SgiImageFile(ImageFile.ImageFile):
 
     format = "SGI"
@@ -44,54 +58,89 @@ class SgiImageFile(ImageFile.ImageFile):
     def _open(self):
 
         # HEAD
-        s = self.fp.read(512)
+        headlen = 512
+        s = self.fp.read(headlen)
+
+        # magic number : 474
         if i16(s) != 474:
             raise ValueError("Not an SGI image file")
 
-        # relevant header entries
+        # compression : verbatim or RLE
         compression = i8(s[2])
 
-        # bytes, dimension, zsize
-        layout = i8(s[3]), i16(s[4:]), i16(s[10:])
+        # bpc : 1 or 2 bytes (8bits or 16bits)
+        bpc = i8(s[3])
 
-        # determine mode from bytes/zsize
-        if layout == (1, 2, 1) or layout == (1, 1, 1):
-            self.mode = "L"
-        elif layout == (1, 3, 3):
-            self.mode = "RGB"
-        elif layout == (1, 3, 4):
-            self.mode = "RGBA"
-        else:
+        # dimension : 1, 2 or 3 (depending on xsize, ysize and zsize)
+        dimension = i16(s[4:])
+
+        # xsize : width
+        xsize = i16(s[6:])
+
+        # ysize : height
+        ysize = i16(s[8:])
+
+        # zsize : channels count
+        zsize = i16(s[10:])
+
+        # layout
+        layout = bpc, dimension, zsize
+
+        # determine mode from bits/zsize
+        rawmode = ""
+        try:
+            rawmode = MODES[layout]
+        except KeyError:
+            pass
+
+        if rawmode == "":
             raise ValueError("Unsupported SGI image mode")
 
-        # size
-        self.size = i16(s[6:]), i16(s[8:])
+        self.size = xsize, ysize
+        self.mode = rawmode.split(";")[0]
+
+        # orientation -1 : scanlines begins at the bottom-left corner
+        orientation = -1
 
         # decoder info
         if compression == 0:
-            offset = 512
-            pagesize = self.size[0]*self.size[1]*layout[0]
-            self.tile = []
-            for layer in self.mode:
-                self.tile.append(
-                    ("raw", (0, 0)+self.size, offset, (layer, 0, -1)))
-                offset = offset + pagesize
+            pagesize = xsize * ysize * bpc
+            if bpc == 2:
+                self.tile = [("SGI16", (0, 0) + self.size,
+                              headlen, (self.mode, 0, orientation))]
+            else:
+                self.tile = []
+                offset = headlen
+                for layer in self.mode:
+                    self.tile.append(
+                        ("raw", (0, 0) + self.size,
+                            offset, (layer, 0, orientation)))
+                    offset += pagesize
         elif compression == 1:
-            raise ValueError("SGI RLE encoding not supported")
+            self.tile = [("sgi_rle", (0, 0) + self.size,
+                          headlen, (rawmode, orientation, bpc))]
 
 
 def _save(im, fp, filename):
     if im.mode != "RGB" and im.mode != "RGBA" and im.mode != "L":
         raise ValueError("Unsupported SGI image mode")
 
+    # Get the keyword arguments
+    info = im.encoderinfo
+
+    # Byte-per-pixel precision, 1 = 8bits per pixel
+    bpc = info.get("bpc", 1)
+
+    if bpc not in (1, 2):
+        raise ValueError("Unsupported number of bytes per pixel")
+
     # Flip the image, since the origin of SGI file is the bottom-left corner
-    im = im.transpose(Image.FLIP_TOP_BOTTOM)
+    orientation = -1
     # Define the file as SGI File Format
     magicNumber = 474
     # Run-Length Encoding Compression - Unsupported at this time
     rle = 0
-    # Byte-per-pixel precision, 1 = 8bits per pixel
-    bpc = 1
+
     # Number of dimensions (x,y,z)
     dim = 3
     # X Dimension = width / Y Dimension = height
@@ -102,8 +151,10 @@ def _save(im, fp, filename):
         dim = 2
     # Z Dimension: Number of channels
     z = len(im.mode)
+
     if dim == 1 or dim == 2:
         z = 1
+
     # assert we've got the right number of bands.
     if len(im.getbands()) != z:
         raise ValueError("incorrect number of bands in SGI write: %s vs %s" %
@@ -128,23 +179,43 @@ def _save(im, fp, filename):
     fp.write(struct.pack('>H', z))
     fp.write(struct.pack('>l', pinmin))
     fp.write(struct.pack('>l', pinmax))
-
     fp.write(struct.pack('4s', b''))  # dummy
     fp.write(struct.pack('79s', imgName))  # truncates to 79 chars
     fp.write(struct.pack('s', b''))  # force null byte after imgname
     fp.write(struct.pack('>l', colormap))
-
     fp.write(struct.pack('404s', b''))  # dummy
 
+    rawmode = 'L'
+    if bpc == 2:
+        rawmode = 'L;16B'
+
     for channel in im.split():
-        fp.write(channel.tobytes())
+        fp.write(channel.tobytes('raw', rawmode, 0, orientation))
 
     fp.close()
 
+class SGI16Decoder(ImageFile.PyDecoder):
+    _pulls_fd = True
+
+    def decode(self, buffer):
+        rawmode, stride, orientation = self.args
+        pagesize = self.state.xsize * self.state.ysize
+        zsize = len(self.mode)
+        self.fd.seek(512)
+
+        for band in range(zsize):
+            channel = Image.new('L', (self.state.xsize, self.state.ysize))
+            channel.frombytes(self.fd.read(2 * pagesize), 'raw',
+                              'L;16B', stride, orientation)
+            self.im.putband(channel.im, band)
+
+        return -1, 0
 
 #
 # registry
 
+
+Image.register_decoder("SGI16", SGI16Decoder)
 Image.register_open(SgiImageFile.format, SgiImageFile, _accept)
 Image.register_save(SgiImageFile.format, _save)
 Image.register_mime(SgiImageFile.format, "image/sgi")

Tests/test_file_sgi.py

@@ -13,6 +13,12 @@ class TestFileSgi(PillowTestCase):
         im = Image.open(test_file)
         self.assert_image_equal(im, hopper())
 
+    def test_rgb16(self):
+        test_file = "Tests/images/hopper16.rgb"
+
+        im = Image.open(test_file)
+        self.assert_image_equal(im, hopper())
+
     def test_l(self):
         # Created with ImageMagick
         # convert hopper.ppm -monochrome -compress None sgi:hopper.bw
@@ -31,12 +37,20 @@ class TestFileSgi(PillowTestCase):
         self.assert_image_equal(im, target)
 
     def test_rle(self):
+        # Created with ImageMagick:
         # convert hopper.ppm  hopper.sgi
-        # We don't support RLE compression, this should throw a value error
         test_file = "Tests/images/hopper.sgi"
 
-        with self.assertRaises(ValueError):
-            Image.open(test_file)
+        im = Image.open(test_file)
+        target = Image.open('Tests/images/hopper.rgb')
+        self.assert_image_equal(im, target)
+
+    def test_rle16(self):
+        test_file = "Tests/images/tv16.sgi"
+
+        im = Image.open(test_file)
+        target = Image.open('Tests/images/tv.rgb')
+        self.assert_image_equal(im, target)
 
     def test_invalid_file(self):
         invalid_file = "Tests/images/flower.jpg"
@@ -57,19 +71,22 @@ class TestFileSgi(PillowTestCase):
         # Test 1 dimension for an L mode image
         roundtrip(Image.new('L', (10, 1)))
 
+    def test_write16(self):
+        test_file = "Tests/images/hopper16.rgb"
+
+        im = Image.open(test_file)
+        out = self.tempfile('temp.sgi')
+        im.save(out, format='sgi', bpc=2)
+
+        reloaded = Image.open(out)
+        self.assert_image_equal(im, reloaded)
+       
     def test_unsupported_mode(self):
         im = hopper('LA')
         out = self.tempfile('temp.sgi')
 
         self.assertRaises(ValueError, im.save, out, format='sgi')
 
-    def test_incorrect_number_of_bands(self):
-        im = hopper('YCbCr')
-        im.mode = 'RGB'
-        out = self.tempfile('temp.sgi')
-
-        self.assertRaises(ValueError, im.save, out, format='sgi')
-
 
 if __name__ == '__main__':
     unittest.main()

Tests/test_lib_pack.py

@@ -38,6 +38,8 @@ class TestLibPack(PillowTestCase):
 
     def test_L(self):
         self.assert_pack("L", "L", 1, 1,2,3,4)
+        self.assert_pack("L", "L;16", b'\x00\xc6\x00\xaf', 198, 175)
+        self.assert_pack("L", "L;16B", b'\xc6\x00\xaf\x00', 198, 175)
 
     def test_LA(self):
         self.assert_pack("LA", "LA", 2, (1,2), (3,4), (5,6))
@@ -221,6 +223,9 @@ class TestLibUnpack(PillowTestCase):
         self.assert_unpack("L", "L;R", 1, 128, 64, 192, 32)
         self.assert_unpack("L", "L;16", 2, 2, 4, 6, 8)
         self.assert_unpack("L", "L;16B", 2, 1, 3, 5, 7)
+        self.assert_unpack("L", "L;16", b'\x00\xc6\x00\xaf', 198, 175) 
+        self.assert_unpack("L", "L;16B", b'\xc6\x00\xaf\x00', 198, 175)
+
 
     def test_LA(self):
         self.assert_unpack("LA", "LA", 2, (1, 2), (3, 4), (5, 6))

_imaging.c

@@ -3496,6 +3496,7 @@ extern PyObject* PyImaging_PackbitsDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_PcdDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_PcxDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_RawDecoderNew(PyObject* self, PyObject* args);
+extern PyObject* PyImaging_SgiRleDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_SunRleDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_TgaRleDecoderNew(PyObject* self, PyObject* args);
 extern PyObject* PyImaging_XbmDecoderNew(PyObject* self, PyObject* args);
@@ -3573,6 +3574,7 @@ static PyMethodDef functions[] = {
     {"pcx_encoder", (PyCFunction)PyImaging_PcxEncoderNew, 1},
     {"raw_decoder", (PyCFunction)PyImaging_RawDecoderNew, 1},
     {"raw_encoder", (PyCFunction)PyImaging_RawEncoderNew, 1},
+    {"sgi_rle_decoder", (PyCFunction)PyImaging_SgiRleDecoderNew, 1},
     {"sun_rle_decoder", (PyCFunction)PyImaging_SunRleDecoderNew, 1},
     {"tga_rle_decoder", (PyCFunction)PyImaging_TgaRleDecoderNew, 1},
     {"xbm_decoder", (PyCFunction)PyImaging_XbmDecoderNew, 1},

decode.c

@@ -38,6 +38,7 @@
 #include "Lzw.h"
 #include "Raw.h"
 #include "Bit.h"
+#include "Sgi.h"
 
 
 /* -------------------------------------------------------------------- */
@@ -671,6 +672,39 @@ PyImaging_RawDecoderNew(PyObject* self, PyObject* args)
 }
 
 
+/* -------------------------------------------------------------------- */
+/* SGI RLE                                                              */
+/* -------------------------------------------------------------------- */
+
+PyObject*
+PyImaging_SgiRleDecoderNew(PyObject* self, PyObject* args)
+{
+    ImagingDecoderObject* decoder;
+
+    char* mode;
+    char* rawmode;
+    int ystep = 1;
+    int bpc = 1;
+    if (!PyArg_ParseTuple(args, "ss|ii", &mode, &rawmode, &ystep, &bpc))
+        return NULL;
+
+    decoder = PyImaging_DecoderNew(sizeof(SGISTATE));
+    if (decoder == NULL)
+        return NULL;
+
+    if (get_unpacker(decoder, mode, rawmode) < 0)
+        return NULL;
+
+    decoder->pulls_fd = 1;
+    decoder->decode = ImagingSgiRleDecode;
+    decoder->state.ystep = ystep;
+
+    ((SGISTATE*)decoder->state.context)->bpc = bpc;
+
+    return (PyObject*) decoder;
+}
+
+
 /* -------------------------------------------------------------------- */
 /* SUN RLE                                                              */
 /* -------------------------------------------------------------------- */

libImaging/Imaging.h

@@ -466,6 +466,9 @@ extern int ImagingRawDecode(Imaging im, ImagingCodecState state,
                             UINT8* buffer, int bytes);
 extern int ImagingRawEncode(Imaging im, ImagingCodecState state,
                             UINT8* buffer, int bytes);
+extern int ImagingSgiRleDecode(Imaging im, ImagingCodecState state,
+                               UINT8* buffer, int bytes);
+extern int ImagingSgiRleDecodeCleanup(ImagingCodecState state);
 extern int ImagingSunRleDecode(Imaging im, ImagingCodecState state,
                                UINT8* buffer, int bytes);
 extern int ImagingTgaRleDecode(Imaging im, ImagingCodecState state,

libImaging/Pack.c

@@ -197,6 +197,31 @@ packP2(UINT8* out, const UINT8* in, int pixels)
     }
 }
 
+static void
+packL16(UINT8* out, const UINT8* in, int pixels)
+{
+    int i;
+    /* L -> L;16, e.g: \xff77 -> \x00\xff\x00\x77 */
+    for (i = 0; i < pixels; i++) {
+        out[0] = 0;
+        out[1] = in[i];
+        out += 2;
+    }
+}
+
+static void
+packL16B(UINT8* out, const UINT8* in, int pixels)
+{
+    int i;
+    /* L -> L;16B, e.g: \xff77 -> \xff\x00\x77\x00 */
+    for (i = 0; i < pixels; i++) {
+        out[0] = in[i];
+        out[1] = 0;
+        out += 2;
+    }
+}
+
+
 static void
 packLA(UINT8* out, const UINT8* in, int pixels)
 {
@@ -512,6 +537,8 @@ static struct {
 
     /* greyscale */
     {"L",       "L",            8,      copy1},
+    {"L",       "L;16",         16,     packL16},
+    {"L",       "L;16B",        16,     packL16B},
 
     /* greyscale w. alpha */
     {"LA",      "LA",           16,     packLA},

libImaging/Sgi.h

@@ -0,0 +1,40 @@
+/* Sgi.h */
+
+typedef struct {
+
+    /* CONFIGURATION */
+
+    /* Number of bytes per channel per pixel */
+    int bpc;
+
+    /* RLE offsets table */
+    UINT32 *starttab;
+
+    /* RLE lengths table */
+    UINT32 *lengthtab;
+
+    /* current row offset */
+    UINT32 rleoffset;
+
+    /* current row length */
+    UINT32 rlelength;
+
+    /* RLE table size */
+    int tablen;
+
+    /* RLE table index */
+    int tabindex;
+
+    /* buffer index */
+    int bufindex;
+
+    /* current row index */
+    int rowno;
+
+    /* current channel index */
+    int channo;
+
+    /* image data size from file descriptor */
+    long bufsize;
+
+} SGISTATE;

libImaging/SgiRleDecode.c

@@ -0,0 +1,188 @@
+/*
+ * The Python Imaging Library.
+ * $Id$
+ *
+ * decoder for Sgi RLE data.
+ *
+ * history:
+ * 2017-07-28 mb    fixed for images larger than 64KB
+ * 2017-07-20 mb	created
+ *
+ * Copyright (c) Mickael Bonfill 2017.
+ *
+ * See the README file for information on usage and redistribution.
+ */
+
+#include "Imaging.h"
+#include "Sgi.h"
+
+#define SGI_HEADER_SIZE 512
+#define RLE_COPY_FLAG 0x80
+#define RLE_MAX_RUN 0x7f
+
+static void read4B(UINT32* dest, UINT8* buf)
+{
+    *dest = (UINT32)((buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3]);
+}
+
+static int expandrow(UINT8* dest, UINT8* src, int n, int z)
+{
+    UINT8 pixel, count;
+    
+    for (;n > 0; n--)
+    {
+        pixel = *src++;
+        if (n == 1 && pixel != 0)
+            return n;
+        count = pixel & RLE_MAX_RUN;
+        if (!count)
+            return count;
+        if (pixel & RLE_COPY_FLAG) {
+            while(count--) {
+                *dest = *src++;
+                dest += z;
+            }
+                
+        }
+        else {
+            pixel = *src++;
+            while (count--) {
+                *dest = pixel;
+                dest += z;
+            }
+        }
+        
+    }
+    return 0;
+}
+
+static int expandrow2(UINT16* dest, UINT16* src, int n, int z)
+{
+    UINT8 pixel, count;
+
+   
+    for (;n > 0; n--)
+    {
+        pixel = ((UINT8*)src)[1];
+        ++src;
+        if (n == 1 && pixel != 0)
+            return n;
+        count = pixel & RLE_MAX_RUN;
+        if (!count)
+            return count;
+        if (pixel & RLE_COPY_FLAG) {
+            while(count--) {
+                *dest = *src++;
+                dest += z;
+            }
+        }
+        else {
+            while (count--) {
+                *dest = *src;
+                dest += z;
+            }
+            ++src;
+        }
+    }
+    return 0;
+}
+
+
+int
+ImagingSgiRleDecode(Imaging im, ImagingCodecState state,
+		    UINT8* buf, int bytes)
+{
+    UINT8 *ptr;
+    SGISTATE *c;
+    int err = 0;
+
+    /* Get all data from File descriptor */    
+    c = (SGISTATE*)state->context;
+    _imaging_seek_pyFd(state->fd, 0L, SEEK_END);
+    c->bufsize = _imaging_tell_pyFd(state->fd);
+    c->bufsize -= SGI_HEADER_SIZE;
+    ptr = malloc(sizeof(UINT8) * c->bufsize);
+    if (!ptr) {
+        return IMAGING_CODEC_MEMORY;
+    }
+    _imaging_seek_pyFd(state->fd, SGI_HEADER_SIZE, SEEK_SET);
+    _imaging_read_pyFd(state->fd, (char*)ptr, c->bufsize);
+
+
+    /* decoder initialization */
+    state->count = 0;
+    state->y = 0;
+    if (state->ystep < 0) {
+        state->y = im->ysize - 1;
+    } else {
+        state->ystep = 1;
+    }
+
+    if (im->xsize > INT_MAX / im->bands ||
+        im->ysize > INT_MAX / im->bands) {
+        err = IMAGING_CODEC_MEMORY;
+        goto sgi_finish_decode;
+    }
+
+    /* Allocate memory for RLE tables and rows */
+    free(state->buffer);
+    state->buffer = NULL;
+    /* malloc overflow check above */
+    state->buffer = calloc(im->xsize * im->bands, sizeof(UINT8) * 2);
+    c->tablen = im->bands * im->ysize;
+    c->starttab = calloc(c->tablen, sizeof(UINT32));
+    c->lengthtab = calloc(c->tablen, sizeof(UINT32));
+    if (!state->buffer ||
+        !c->starttab ||
+        !c->lengthtab) {
+        err = IMAGING_CODEC_MEMORY;
+        goto sgi_finish_decode;
+    }
+    /* populate offsets table */
+    for (c->tabindex = 0, c->bufindex = 0; c->tabindex < c->tablen; c->tabindex++, c->bufindex+=4)
+        read4B(&c->starttab[c->tabindex], &ptr[c->bufindex]);
+    /* populate lengths table */
+    for (c->tabindex = 0, c->bufindex = c->tablen * sizeof(UINT32); c->tabindex < c->tablen; c->tabindex++, c->bufindex+=4)
+        read4B(&c->lengthtab[c->tabindex], &ptr[c->bufindex]);
+
+    state->count += c->tablen * sizeof(UINT32) * 2;
+
+    /* read compressed rows */
+    for (c->rowno = 0; c->rowno < im->ysize; c->rowno++, state->y += state->ystep)
+    {
+        for (c->channo = 0; c->channo < im->bands; c->channo++)
+        {
+            c->rleoffset = c->starttab[c->rowno + c->channo * im->ysize];
+            c->rlelength = c->lengthtab[c->rowno + c->channo * im->ysize];
+            c->rleoffset -= SGI_HEADER_SIZE;
+            
+            /* row decompression */
+            if (c->bpc ==1) {
+                if(expandrow(&state->buffer[c->channo], &ptr[c->rleoffset], c->rlelength, im->bands))
+                    goto sgi_finish_decode;
+            }
+            else {
+                if(expandrow2((UINT16*)&state->buffer[c->channo * 2], (UINT16*)&ptr[c->rleoffset], c->rlelength, im->bands))
+                    goto sgi_finish_decode;
+            }
+            
+            state->count += c->rlelength;
+        }
+        
+        /* store decompressed data in image */
+        state->shuffle((UINT8*)im->image[state->y], state->buffer, im->xsize);
+        
+    }
+
+    c->bufsize++;
+
+sgi_finish_decode: ;
+    
+    free(c->starttab);
+    free(c->lengthtab);
+    free(ptr);
+    if (err != 0){
+        return err;
+    }
+    return state->count - c->bufsize;
+}

setup.py

@@ -35,8 +35,8 @@ _LIB_IMAGING = (
     "Negative", "Offset", "Pack", "PackDecode", "Palette", "Paste",
     "Quant", "QuantOctree", "QuantHash", "QuantHeap", "PcdDecode", "PcxDecode",
     "PcxEncode", "Point", "RankFilter", "RawDecode", "RawEncode", "Storage",
-    "SunRleDecode", "TgaRleDecode", "Unpack", "UnpackYCC", "UnsharpMask",
-    "XbmDecode", "XbmEncode", "ZipDecode", "ZipEncode", "TiffDecode",
+    "SgiRleDecode", "SunRleDecode", "TgaRleDecode", "Unpack", "UnpackYCC",
+    "UnsharpMask", "XbmDecode", "XbmEncode", "ZipDecode", "ZipEncode", "TiffDecode",
     "Jpeg2KDecode", "Jpeg2KEncode", "BoxBlur", "QuantPngQuant", "codec_fd")
 
 DEBUG = False