Merge pull request #3227 from kkopachev/master

Adding support to reading tiled and YcbCr jpeg tiffs through libtiff

Tests/test_file_libtiff.py

@@ -633,6 +633,43 @@ class TestFileLibTiff(LibTiffTestCase):
         im2 = hopper()
         self.assert_image_similar(im, im2, 5)
 
+    def test_strip_cmyk_jpeg(self):
+        infile = "Tests/images/tiff_strip_cmyk_jpeg.tif"
+        im = Image.open(infile)
+
+        self.assert_image_similar_tofile(im, "Tests/images/pil_sample_cmyk.jpg", 0.5)
+
+    def test_strip_ycbcr_jpeg_2x2_sampling(self):
+        infile = "Tests/images/tiff_strip_ycbcr_jpeg_2x2_sampling.tif"
+        im = Image.open(infile)
+
+        self.assert_image_similar_tofile(im, "Tests/images/flower.jpg", 0.5)
+
+    def test_strip_ycbcr_jpeg_1x1_sampling(self):
+        infile = "Tests/images/tiff_strip_ycbcr_jpeg_1x1_sampling.tif"
+        im = Image.open(infile)
+
+        self.assert_image_equal_tofile(im, "Tests/images/flower2.jpg")
+
+    def test_tiled_cmyk_jpeg(self):
+        infile = "Tests/images/tiff_tiled_cmyk_jpeg.tif"
+        im = Image.open(infile)
+
+        self.assert_image_similar_tofile(im, "Tests/images/pil_sample_cmyk.jpg", 0.5)
+
+    def test_tiled_ycbcr_jpeg_1x1_sampling(self):
+        infile = "Tests/images/tiff_tiled_ycbcr_jpeg_1x1_sampling.tif"
+        im = Image.open(infile)
+
+        self.assert_image_equal_tofile(im, "Tests/images/flower2.jpg")
+
+    def test_tiled_ycbcr_jpeg_2x2_sampling(self):
+        infile = "Tests/images/tiff_tiled_ycbcr_jpeg_2x2_sampling.tif"
+        im = Image.open(infile)
+
+        self.assert_image_similar_tofile(im, "Tests/images/flower.jpg", 0.5)
+
+
 
 if __name__ == '__main__':
     unittest.main()

Tests/test_file_tiff.py

@@ -413,6 +413,34 @@ class TestFileTiff(PillowTestCase):
 
         self.assert_image_equal(im, reloaded)
 
+    def test_strip_raw(self):
+        infile = "Tests/images/tiff_strip_raw.tif"
+        im = Image.open(infile)
+
+        self.assert_image_equal_tofile(im, "Tests/images/tiff_adobe_deflate.png")
+
+    def test_strip_planar_raw(self):
+        # gdal_translate -of GTiff -co INTERLEAVE=BAND tiff_strip_raw.tif tiff_strip_planar_raw.tiff
+        infile = "Tests/images/tiff_strip_planar_raw.tif"
+        im = Image.open(infile)
+
+        self.assert_image_equal_tofile(im, "Tests/images/tiff_adobe_deflate.png")
+
+    def test_strip_planar_raw_with_overviews(self):
+        # gdaladdo tiff_strip_planar_raw2.tif 2 4 8 16
+        infile = "Tests/images/tiff_strip_planar_raw_with_overviews.tif"
+        im = Image.open(infile)
+
+        self.assert_image_equal_tofile(im, "Tests/images/tiff_adobe_deflate.png")
+
+    def test_tiled_planar_raw(self):
+        # gdal_translate -of GTiff -co TILED=YES -co BLOCKXSIZE=32 -co BLOCKYSIZE=32 -co INTERLEAVE=BAND \
+        # tiff_tiled_raw.tif tiff_tiled_planar_raw.tiff
+        infile = "Tests/images/tiff_tiled_planar_raw.tif"
+        im = Image.open(infile)
+
+        self.assert_image_equal_tofile(im, "Tests/images/tiff_adobe_deflate.png")
+
     def test_tiff_save_all(self):
         import io
         import os

Tests/test_lib_pack.py

@@ -478,10 +478,6 @@ class TestLibUnpack(PillowTestCase):
             "YCbCr", "YCbCrK", 4, (1, 2, 3), (5, 6, 7), (9, 10, 11))
         self.assert_unpack(
             "YCbCr", "YCbCrX", 4, (1, 2, 3), (5, 6, 7), (9, 10, 11))
-        self.assert_unpack(
-            "YCbCr", "YCbCrXX", 5, (1, 2, 3), (6, 7, 8), (11, 12, 13))
-        self.assert_unpack(
-            "YCbCr", "YCbCrXXX", 6, (1, 2, 3), (7, 8, 9), (13, 14, 15))
 
     def test_LAB(self):
         self.assert_unpack(

src/PIL/TiffImagePlugin.py

@@ -257,14 +257,10 @@ OPEN_INFO = {
     (II, 5, (1,), 1, (8, 8, 8, 8, 8, 8), (0, 0)): ("CMYK", "CMYKXX"),
     (MM, 5, (1,), 1, (8, 8, 8, 8, 8, 8), (0, 0)): ("CMYK", "CMYKXX"),
 
-    (II, 6, (1,), 1, (8, 8, 8), ()): ("YCbCr", "YCbCr"),
+    # JPEG compressed images handled by LibTiff and auto-converted to RGB
-    (MM, 6, (1,), 1, (8, 8, 8), ()): ("YCbCr", "YCbCr"),
+    # Minimal Baseline TIFF requires YCbCr images to have 3 SamplesPerPixel
-    (II, 6, (1,), 1, (8, 8, 8, 8), (0,)): ("YCbCr", "YCbCrX"),
+    (II, 6, (1,), 1, (8, 8, 8), ()): ("RGB", "RGB"),
-    (MM, 6, (1,), 1, (8, 8, 8, 8), (0,)): ("YCbCr", "YCbCrX"),
+    (MM, 6, (1,), 1, (8, 8, 8), ()): ("RGB", "RGB"),
-    (II, 6, (1,), 1, (8, 8, 8, 8, 8), (0, 0)): ("YCbCr", "YCbCrXX"),
-    (MM, 6, (1,), 1, (8, 8, 8, 8, 8), (0, 0)): ("YCbCr", "YCbCrXX"),
-    (II, 6, (1,), 1, (8, 8, 8, 8, 8, 8), (0, 0, 0)): ("YCbCr", "YCbCrXXX"),
-    (MM, 6, (1,), 1, (8, 8, 8, 8, 8, 8), (0, 0, 0)): ("YCbCr", "YCbCrXXX"),
 
     (II, 8, (1,), 1, (8, 8, 8), ()): ("LAB", "LAB"),
     (MM, 8, (1,), 1, (8, 8, 8), ()): ("LAB", "LAB"),
@@ -1053,20 +1049,6 @@ class TiffImageFile(ImageFile.ImageFile):
         "Return the current frame number"
         return self.__frame
 
-    def _decoder(self, rawmode, layer, tile=None):
-        "Setup decoder contexts"
-
-        args = None
-        if rawmode == "RGB" and self._planar_configuration == 2:
-            rawmode = rawmode[layer]
-        compression = self._compression
-        if compression == "raw":
-            args = (rawmode, 0, 1)
-        elif compression == "packbits":
-            args = rawmode
-
-        return args
-
     def load(self):
         if self.use_load_libtiff:
             return self._load_libtiff()
@@ -1187,6 +1169,7 @@ class TiffImageFile(ImageFile.ImageFile):
             print("- photometric_interpretation:", photo)
             print("- planar_configuration:", self._planar_configuration)
             print("- fill_order:", fillorder)
+            print("- YCbCr subsampling:", self.tag.get(530))
 
         # size
         xsize = self.tag_v2.get(IMAGEWIDTH)
@@ -1258,94 +1241,84 @@ class TiffImageFile(ImageFile.ImageFile):
         # build tile descriptors
         x = y = layer = 0
         self.tile = []
-        self.use_load_libtiff = False
+        self.use_load_libtiff = READ_LIBTIFF or self._compression != 'raw'
-        if STRIPOFFSETS in self.tag_v2:
+        if self.use_load_libtiff:
+            # Decoder expects entire file as one tile.
+            # There's a buffer size limit in load (64k)
+            # so large g4 images will fail if we use that
+            # function.
+            #
+            # Setup the one tile for the whole image, then
+            # use the _load_libtiff function.
+
+            # libtiff handles the fillmode for us, so 1;IR should
+            # actually be 1;I. Including the R double reverses the
+            # bits, so stripes of the image are reversed.  See
+            # https://github.com/python-pillow/Pillow/issues/279
+            if fillorder == 2:
+                # Replace fillorder with fillorder=1
+                key = key[:3] + (1,) + key[4:]
+                if DEBUG:
+                    print("format key:", key)
+                # this should always work, since all the
+                # fillorder==2 modes have a corresponding
+                # fillorder=1 mode
+                self.mode, rawmode = OPEN_INFO[key]
+            # libtiff always returns the bytes in native order.
+            # we're expecting image byte order. So, if the rawmode
+            # contains I;16, we need to convert from native to image
+            # byte order.
+            if rawmode == 'I;16':
+                rawmode = 'I;16N'
+            if ';16B' in rawmode:
+                rawmode = rawmode.replace(';16B', ';16N')
+            if ';16L' in rawmode:
+                rawmode = rawmode.replace(';16L', ';16N')
+
+            # Offset in the tile tuple is 0, we go from 0,0 to
+            # w,h, and we only do this once -- eds
+            a = (rawmode, self._compression, False)
+            self.tile.append(
+                (self._compression,
+                 (0, 0, xsize, ysize),
+                 0, a))
+
+        elif STRIPOFFSETS in self.tag_v2 or TILEOFFSETS in self.tag_v2:
             # striped image
-            offsets = self.tag_v2[STRIPOFFSETS]
+            if STRIPOFFSETS in self.tag_v2:
-            h = self.tag_v2.get(ROWSPERSTRIP, ysize)
+                offsets = self.tag_v2[STRIPOFFSETS]
-            w = self.size[0]
+                h = self.tag_v2.get(ROWSPERSTRIP, ysize)
-            if READ_LIBTIFF or self._compression != 'raw':
+                w = self.size[0]
-                # Decoder expects entire file as one tile.
-                # There's a buffer size limit in load (64k)
-                # so large g4 images will fail if we use that
-                # function.
-                #
-                # Setup the one tile for the whole image, then
-                # use the _load_libtiff function.
-
-                self.use_load_libtiff = True
-
-                # libtiff handles the fillmode for us, so 1;IR should
-                # actually be 1;I. Including the R double reverses the
-                # bits, so stripes of the image are reversed.  See
-                # https://github.com/python-pillow/Pillow/issues/279
-                if fillorder == 2:
-                    key = (
-                        self.tag_v2.prefix, photo, sampleFormat, 1,
-                        self.tag_v2.get(BITSPERSAMPLE, (1,)),
-                        self.tag_v2.get(EXTRASAMPLES, ())
-                        )
-                    if DEBUG:
-                        print("format key:", key)
-                    # this should always work, since all the
-                    # fillorder==2 modes have a corresponding
-                    # fillorder=1 mode
-                    self.mode, rawmode = OPEN_INFO[key]
-                # libtiff always returns the bytes in native order.
-                # we're expecting image byte order. So, if the rawmode
-                # contains I;16, we need to convert from native to image
-                # byte order.
-                if rawmode == 'I;16':
-                    rawmode = 'I;16N'
-                if ';16B' in rawmode:
-                    rawmode = rawmode.replace(';16B', ';16N')
-                if ';16L' in rawmode:
-                    rawmode = rawmode.replace(';16L', ';16N')
-
-                # Offset in the tile tuple is 0, we go from 0,0 to
-                # w,h, and we only do this once -- eds
-                a = (rawmode, self._compression, False)
-                self.tile.append(
-                    (self._compression,
-                     (0, 0, w, ysize),
-                     0, a))
-                a = None
-
             else:
-                for i, offset in enumerate(offsets):
+                # tiled image
-                    a = self._decoder(rawmode, layer, i)
+                offsets = self.tag_v2[TILEOFFSETS]
-                    self.tile.append(
+                w = self.tag_v2.get(322)
-                        (self._compression,
+                h = self.tag_v2.get(323)
-                            (0, min(y, ysize), w, min(y+h, ysize)),
+
-                            offset, a))
+            for offset in offsets:
-                    if DEBUG:
+                if x + w > xsize:
-                        print("tiles: ", self.tile)
+                    stride = w * sum(bps_tuple) / 8  # bytes per line
-                    y = y + h
+                else:
-                    if y >= self.size[1]:
+                    stride = 0
-                        x = y = 0
+
-                        layer += 1
+                tile_rawmode = rawmode
-                    a = None
+                if self._planar_configuration == 2:
-        elif TILEOFFSETS in self.tag_v2:
+                    # each band on it's own layer
-            # tiled image
+                    tile_rawmode = rawmode[layer]
-            w = self.tag_v2.get(322)
+                    # adjust stride width accordingly
-            h = self.tag_v2.get(323)
+                    stride /= bps_count
-            a = None
+
-            for o in self.tag_v2[TILEOFFSETS]:
+                a = (tile_rawmode, int(stride), 1)
-                if not a:
-                    a = self._decoder(rawmode, layer)
-                # FIXME: this doesn't work if the image size
-                # is not a multiple of the tile size...
                 self.tile.append(
                     (self._compression,
-                        (x, y, x+w, y+h),
+                     (x, y, min(x+w, xsize), min(y+h, ysize)),
-                        o, a))
+                     offset, a))
                 x = x + w
                 if x >= self.size[0]:
                     x, y = 0, y + h
                     if y >= self.size[1]:
                         x = y = 0
                         layer += 1
-                        a = None
         else:
             if DEBUG:
                 print("- unsupported data organization")

src/PIL/TiffTags.py

@@ -438,7 +438,7 @@ LIBTIFF_CORE.remove(301)  # Array of short, crashes
 LIBTIFF_CORE.remove(532)  # Array of long, crashes
 
 LIBTIFF_CORE.remove(255)  # We don't have support for subfiletypes
-LIBTIFF_CORE.remove(322)  # We don't have support for tiled images in libtiff
+LIBTIFF_CORE.remove(322)  # We don't have support for writing tiled images with libtiff
 LIBTIFF_CORE.remove(323)  # Tiled images
 LIBTIFF_CORE.remove(333)  # Ink Names either
 

src/libImaging/TiffDecode.c

@@ -55,7 +55,7 @@ tsize_t _tiffWriteProc(thandle_t hdata, tdata_t buf, tsize_t size) {
 
 	to_write = min(size, state->size - (tsize_t)state->loc);
 	if (state->flrealloc && size>to_write) {
-		tdata_t new;
+		tdata_t new_data;
 		tsize_t newsize=state->size;
 		while (newsize < (size + state->size)) {
             if (newsize > INT_MAX - 64*1024){
@@ -66,12 +66,12 @@ tsize_t _tiffWriteProc(thandle_t hdata, tdata_t buf, tsize_t size) {
 		}
 		TRACE(("Reallocing in write to %d bytes\n", (int)newsize));
         /* malloc check ok, overflow checked above */
-		new = realloc(state->data, newsize);
+		new_data = realloc(state->data, newsize);
-		if (!new) {
+		if (!new_data) {
 			// fail out
 			return 0;
 		}
-		state->data = new;
+		state->data = new_data;
 		state->size = newsize;
 		to_write = size;
 	}
@@ -173,7 +173,6 @@ int ImagingLibTiffDecode(Imaging im, ImagingCodecState state, UINT8* buffer, int
 	char *filename = "tempfile.tif";
 	char *mode = "r";
 	TIFF *tiff;
-	tsize_t size;
 
 
 	/* buffer is the encoded file, bytes is the length of the encoded file */
@@ -235,36 +234,97 @@ int ImagingLibTiffDecode(Imaging im, ImagingCodecState state, UINT8* buffer, int
 		}
 	}
 
-	size = TIFFScanlineSize(tiff);
+    TIFFSetField(tiff, TIFFTAG_JPEGCOLORMODE, JPEGCOLORMODE_RGB);
-	TRACE(("ScanlineSize: %d \n", size));
-	if (size > state->bytes) {
-		TRACE(("Error, scanline size > buffer size\n"));
-		state->errcode = IMAGING_CODEC_BROKEN;
-		TIFFClose(tiff);
-		return -1;
-	}
 
-	// Have to do this row by row and shove stuff into the buffer that way,
+    if (TIFFIsTiled(tiff)) {
-	// with shuffle.  (or, just alloc a buffer myself, then figure out how to get it
+        uint32 x, y, tile_y;
-	// back in. Can't use read encoded stripe.
+        uint32 tileWidth, tileLength;
+        UINT8 *new_data;
 
-	// This thing pretty much requires that I have the whole image in one shot.
+        state->bytes = TIFFTileSize(tiff);
-	// Perhaps a stub version would work better???
-	while(state->y < state->ysize){
-		if (TIFFReadScanline(tiff, (tdata_t)state->buffer, (uint32)state->y, 0) == -1) {
-			TRACE(("Decode Error, row %d\n", state->y));
-			state->errcode = IMAGING_CODEC_BROKEN;
-			TIFFClose(tiff);
-			return -1;
-		}
-		/* TRACE(("Decoded row %d \n", state->y)); */
-		state->shuffle((UINT8*) im->image[state->y + state->yoff] +
-					       state->xoff * im->pixelsize,
-					   state->buffer,
-					   state->xsize);
 
-		state->y++;
+        /* overflow check for malloc */
-	}
+        if (state->bytes > INT_MAX - 1) {
+            state->errcode = IMAGING_CODEC_MEMORY;
+            TIFFClose(tiff);
+            return -1;
+        }
+
+        /* realloc to fit whole tile */
+        new_data = realloc (state->buffer, state->bytes);
+        if (!new_data) {
+            state->errcode = IMAGING_CODEC_MEMORY;
+            TIFFClose(tiff);
+            return -1;
+        }
+
+        state->buffer = new_data;
+
+        TRACE(("TIFFTileSize: %d\n", state->bytes));
+
+        TIFFGetField(tiff, TIFFTAG_TILEWIDTH, &tileWidth);
+        TIFFGetField(tiff, TIFFTAG_TILELENGTH, &tileLength);
+
+        for (y = state->yoff; y < state->ysize; y += tileLength) {
+            for (x = state->xoff; x < state->xsize; x += tileWidth) {
+                if (TIFFReadTile(tiff, (tdata_t)state->buffer, x, y, 0, 0) == -1) {
+                    TRACE(("Decode Error, Tile at %dx%d\n", x, y));
+                    state->errcode = IMAGING_CODEC_BROKEN;
+                    TIFFClose(tiff);
+                    return -1;
+                }
+
+                TRACE(("Read tile at %dx%d; \n\n", x, y));
+
+                // iterate over each line in the tile and stuff data into image
+                for (tile_y = 0; tile_y < min(tileLength, state->ysize - y); tile_y++) {
+
+                    TRACE(("Writing tile data at %dx%d using tilwWidth: %d; \n", tile_y + y, x, min(tileWidth, state->xsize - x)));
+
+                    // UINT8 * bbb = state->buffer + tile_y * (state->bytes / tileLength);
+                    // TRACE(("chars: %x%x%x%x\n", ((UINT8 *)bbb)[0], ((UINT8 *)bbb)[1], ((UINT8 *)bbb)[2], ((UINT8 *)bbb)[3]));
+
+                    state->shuffle((UINT8*) im->image[tile_y + y] + x * im->pixelsize,
+                       state->buffer + tile_y * (state->bytes / tileLength),
+                       min(tileWidth, state->xsize - x)
+                    );
+                }
+            }
+        }
+    } else {
+        tsize_t size;
+
+        size = TIFFScanlineSize(tiff);
+        TRACE(("ScanlineSize: %lu \n", size));
+        if (size > state->bytes) {
+            TRACE(("Error, scanline size > buffer size\n"));
+            state->errcode = IMAGING_CODEC_BROKEN;
+            TIFFClose(tiff);
+            return -1;
+        }
+
+        // Have to do this row by row and shove stuff into the buffer that way,
+        // with shuffle.  (or, just alloc a buffer myself, then figure out how to get it
+        // back in. Can't use read encoded stripe.
+
+        // This thing pretty much requires that I have the whole image in one shot.
+        // Perhaps a stub version would work better???
+        while(state->y < state->ysize){
+            if (TIFFReadScanline(tiff, (tdata_t)state->buffer, (uint32)state->y, 0) == -1) {
+                TRACE(("Decode Error, row %d\n", state->y));
+                state->errcode = IMAGING_CODEC_BROKEN;
+                TIFFClose(tiff);
+                return -1;
+            }
+            /* TRACE(("Decoded row %d \n", state->y)); */
+            state->shuffle((UINT8*) im->image[state->y + state->yoff] +
+                               state->xoff * im->pixelsize,
+                           state->buffer,
+                           state->xsize);
+
+            state->y++;
+        }
+    }
 
 	TIFFClose(tiff);
 	TRACE(("Done Decoding, Returning \n"));

src/libImaging/Unpack.c

@@ -1429,8 +1429,6 @@ static struct {
     {"YCbCr",   "YCbCr",        24,     ImagingUnpackRGB},
     {"YCbCr",   "YCbCr;L",      24,     unpackRGBL},
     {"YCbCr",   "YCbCrX",       32,     copy4},
-    {"YCbCr",   "YCbCrXX",      40,     copy4skip1},
-    {"YCbCr",   "YCbCrXXX",     48,     copy4skip2},
     {"YCbCr",   "YCbCrK",       32,     copy4},
 
     /* LAB Color */

winbuild/nmake.opt

@@ -55,18 +55,16 @@ LOGLUV_SUPPORT	= 1
 # Uncomment and edit following lines to enable JPEG support.
 #
 JPEG_SUPPORT	= 1
-JPEGDIR 	= $(BUILD)\jpeg-9c
+JPEG_INCLUDE	= -I$(INCLIB)
-JPEG_INCLUDE	= -I$(JPEGDIR)
+JPEG_LIB 	= $(INCLIB)/libjpeg.lib
-JPEG_LIB 	= $(JPEGDIR)/libjpeg.lib
 
 #
 # Uncomment and edit following lines to enable ZIP support
 # (required for Deflate compression and Pixar log-format)
 #
 ZIP_SUPPORT	= 1
-ZLIBDIR 	= $(BUILD)\zlib-1.2.8
+ZLIB_INCLUDE	= -I$(INCLIB)
-ZLIB_INCLUDE	= -I$(ZLIBDIR)
+ZLIB_LIB 	= $(INCLIB)/zlib.lib
-ZLIB_LIB 	= $(ZLIBDIR)/zlib.lib
 
 #
 # Uncomment and edit following lines to enable ISO JBIG support