From add2746ac61980421455dc722d0edede2478cbfa Mon Sep 17 00:00:00 2001 From: Konstantin Kopachev Date: Wed, 28 Feb 2018 22:15:58 -0800 Subject: [PATCH 1/3] Certain corrupted jpegs can result in no data read On truncated jpeg, decoder can suspend waiting for additional bytes in buffer. For some input files, decoder suspends on jpeg_start_decompress stage. If at this point file reader reaches EOF, py code never gets back to jpeg decoder and we end up with no bytes to result image. This leaves us with some amount of potentially useful bytes undecoded and thrown away. Libjpeg docs suggest that in such situation, more appropriate would be to add EOI marker to the end of buffer, which will allows decoder to finish. https://github.com/libjpeg-turbo/libjpeg-turbo/blob/0dd9a2c1fd6c/libjpeg.txt#L1803-L1809 Docs also mention that adding EOI markers is what non-suspending code does anyway. --- src/PIL/ImageFile.py | 69 ++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/src/PIL/ImageFile.py b/src/PIL/ImageFile.py index 458857f41..7dbea67f3 100644 --- a/src/PIL/ImageFile.py +++ b/src/PIL/ImageFile.py @@ -202,44 +202,39 @@ class ImageFile(Image.Image): for decoder_name, extents, offset, args in self.tile: decoder = Image._getdecoder(self.mode, decoder_name, args, self.decoderconfig) - seek(offset) - decoder.setimage(self.im, extents) - if decoder.pulls_fd: - decoder.setfd(self.fp) - status, err_code = decoder.decode(b"") - else: - b = prefix - while True: - try: - s = read(self.decodermaxblock) - except (IndexError, struct.error): # truncated png/gif - if LOAD_TRUNCATED_IMAGES: + try: + seek(offset) + decoder.setimage(self.im, extents) + if decoder.pulls_fd: + decoder.setfd(self.fp) + status, err_code = decoder.decode(b"") + else: + b = prefix + while True: + try: + s = read(self.decodermaxblock) + except (IndexError, struct.error): # truncated png/gif + if LOAD_TRUNCATED_IMAGES: + break + else: + raise IOError("image file is truncated") + + if not s: # truncated jpeg + if LOAD_TRUNCATED_IMAGES: + s = b"\xFF\xD9" # Pretend file is finished adding EOI marker + else: + self.tile = [] + raise IOError("image file is truncated " + "(%d bytes not processed)" % len(b)) + + b = b + s + n, err_code = decoder.decode(b) + if n < 0: break - else: - raise IOError("image file is truncated") - - if not s: # truncated jpeg - self.tile = [] - - # JpegDecode needs to clean things up here either way - # If we don't destroy the decompressor, - # we have a memory leak. - decoder.cleanup() - - if LOAD_TRUNCATED_IMAGES: - break - else: - raise IOError("image file is truncated " - "(%d bytes not processed)" % len(b)) - - b = b + s - n, err_code = decoder.decode(b) - if n < 0: - break - b = b[n:] - - # Need to cleanup here to prevent leaks in PyPy - decoder.cleanup() + b = b[n:] + finally: + # Need to cleanup here to prevent leaks + decoder.cleanup() self.tile = [] self.readonly = readonly From 1e9e64c8b05727ddcb9b2428f4796743a1ce1e1c Mon Sep 17 00:00:00 2001 From: Konstantin Kopachev Date: Mon, 5 Mar 2018 21:05:56 -0800 Subject: [PATCH 2/3] Move jpeg-specific eof-processing to jpeg plugin --- src/PIL/ImageFile.py | 2 +- src/PIL/JpegImagePlugin.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/PIL/ImageFile.py b/src/PIL/ImageFile.py index 7dbea67f3..39ad1f093 100644 --- a/src/PIL/ImageFile.py +++ b/src/PIL/ImageFile.py @@ -221,7 +221,7 @@ class ImageFile(Image.Image): if not s: # truncated jpeg if LOAD_TRUNCATED_IMAGES: - s = b"\xFF\xD9" # Pretend file is finished adding EOI marker + break else: self.tile = [] raise IOError("image file is truncated " diff --git a/src/PIL/JpegImagePlugin.py b/src/PIL/JpegImagePlugin.py index 9a190fb4a..6a18fff19 100644 --- a/src/PIL/JpegImagePlugin.py +++ b/src/PIL/JpegImagePlugin.py @@ -353,6 +353,21 @@ class JpegImageFile(ImageFile.ImageFile): else: raise SyntaxError("no marker found") + def load_read(self, read_bytes): + """ + internal: read more image data + For premature EOF and LOAD_TRUNCATED_IMAGES adds EOI marker + so libjpeg can finish decoding + """ + s = self.fp.read(read_bytes) + + if not s and ImageFile.LOAD_TRUNCATED_IMAGES: + # Premature EOF. + # Pretend file is finished adding EOI marker + return b"\xFF\xD9" + + return s + def draft(self, mode, size): if len(self.tile) != 1: From 5269bbc268da758998acb7ba8604ca02eef2b7a0 Mon Sep 17 00:00:00 2001 From: Konstantin Kopachev Date: Wed, 7 Mar 2018 21:31:51 -0800 Subject: [PATCH 3/3] Add truncated jpeg tests --- Tests/images/truncated_jpeg.jpg | Bin 0 -> 6698 bytes Tests/test_file_jpeg.py | 16 ++++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 Tests/images/truncated_jpeg.jpg diff --git a/Tests/images/truncated_jpeg.jpg b/Tests/images/truncated_jpeg.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f4fec450df95ab99ae051c23d0fc51362c7cef17 GIT binary patch literal 6698 zcmb7IbySqyyL~5s85m%Ol4j@uRJt36M(L1HkdT%^Me0KsI+Vtd?i2(BMWsbrS{g|S zl@3X{`2Ftp-L>xh>pthtXRWi&e)qHAefGJSxR?i!+Ui>B00;yCpvwhZOarO_jF^~& zm;ZcShywtlKoAt@q8(rW00`*M zY5%9d5D=8;PYd+FTOcq5xNJ=U2nYxOBO%C3kfYx87VTHd2Szl~mnzfu-od?ztEp78 z^W4o$rfHK5b>5vkhmXy|lGfvAiZ$2cwECak^k=yqnBw+wP5h>x`)&Jm8T(Ix8GS%N zI_940i(YfC_YUaUM4zQWTTI7`SH4Y^6TPOx1wnT_dnS^NB7}$TjrnM+X{a*LI()bn z@X_eOzV@jwN1`XUf(6mZQV?rHQ2ahWbv7;jyocy4?h*zE5&(lQ5B_KUr$Ar;1)-v5 z=YVnwizusH;(=X$0T2c__IeJ(499g`D8gKr&+ zG`yM9F@)QwG&D=txNB!h-!@?r`qV7mxh%B`Zf!a{TMgkaV%KrBGw3jdI+=@EzUhW6 z!%nVlEc#t3wJQU8^x=huB}zWOnI=bhr;xnsUf1-=N+VnwHSOV1abrx)b3kyEcF7@f z{JNwa?mm3-$-0HvHi_P?QG)9vp;u{?F!%H5r+J)e{PHnEkHS@}T5FfCX7TgElS>-- zPyh%5fq;pK{+YDP5kf)Og{hPcL~u|J+uUtx!-UqMzuCAPNktGn>8=VcrYKwdxh6Un z`IJ9&GFLYF2R-R#4+pl?v*(S<$rmP@p<^ z&)EF<9S1Ao1%`e`Ka;s8 zxL;*WP48{a-qk8j6;F$wq#2p`X04%PC*O2u&Eu;~dK@uTQP(>O%yjozvQ~lg?Q^pC zHSeE0|Kgr`79KiIld5HS%`H4_Vs3gsHp6^D+_5l~yVRtoxcc)Fh$!+8DPRaOG3dXc{2_&kU6?~z#K0DU3(g(drdBb$o6yS1 zC0f1XmALTVVj)%pzwZ0>y)DgJ*JJsHP1Gn0VG^b3)E+*y{w)uFUKGJ-fhN`1&_E3o zCaG!g(i?3bP!c?182lmtnB2S5*7c*Co+uXN@0=?4xG87G^w=G{ytfF3S|9(Fv^@X| z$Ku(>HxKm0lo~$w6g_8HMP?u$S@{$@2}mF|B5Xl*#V%X+C-IogQj9rn8WMz(Q$&h`pHY}kmzGB_Vs>2uaT zIF5tP&kyy3GkI+soE;C7$c`DN^roJbCKR+q2f69{`dll!_G_yw0#c+@bXM_MC%qtZ zmL_Lh`U0qQ4WB7w5T+J=IF`iZWll%)^OaAG6i>9OvTUg45l)E2X~f>Aj^f*S3lp+8 zru@B~?R{FKN%t}|WgC(YOfH(ub$k0ZG2>1Dc+c0{x~*oscYTZhVoJ8rl345^@OseL zxjM6jf_}ab_HUy|kmRcWW<}+>e(pbO*kV7^p06b2d)n!j*PnR~j+)b)uqHvyj|67z z7Uit=uPX2jlliA7GmfQOntgM|m9zGYv5s&&ETOz_q467o3NT+PPUQZKm?8gS*!yAS zV5MTpCE6_}4M9eQE;rJbW-3XQ7%~%ggh{N_Nl$~UW4?!mO#Mk)x09^4V1&E2QzhD*xTLrtA)&WhD}9Qz z`1=HF>7cuBO!8RTK|*llGsHN*|Y{+oJoAd9_-Tm&( zVA5mju1rloN&cz<-j-g_3EvZqMUMr4MInf}HtRm`^_adG|KKeb@6Q+8^3F(TCnk?; zL~(!L$fSesabpqz8xm_rL>bi*Xd7Rr_q1C{d^E1SjC@pI_E}S>iY7R1^w6SK`3@WSlPf97o1R?+qylpu=9@%DzZmA zZJtnk@OZ+!2t3vzjSf`!<%<>qAsZTP&UfOUrx8tsoZ`%f%c9CrkopUN>Ldtf4sPry z$f}mme-uqWX|EUZRp?tc_V`C+WWYf8x7;BIaVRc2GyW5Fb@&O}W8GTw;I;rxZjs?U z2ab0$BxhZGm|WvoRWsJfp&fU&_NK?#&&xR~S#=S`e)dw8YZriYM`dJNI;FqPQNJ@S z>p%r=kB2>e80MQlGM=H9hO{+YiTj)GW2gQN8Gv_{+|aM5ya;2klfbYLc`XnRc1z z#HP-dC!Y7YJm3OpQ7b%;2Yg*b1G@f7%D)pk#uz@uamM3+($&I`kvz7Dxd0}h^>vx3 zUd#RN0_xjsR0)gJ)2Or*6#oa#kFpm)6;?Tqbz%juy#R!#<@DLN&+adG(0&a99c?~P zOHQq3@{!#AirK2N;hZ-1|GC9r70aqQ)=V9U0@ukBiJ^_6XU)en>qs~U6%H%}=()co z8eQ9M_VFD2Na60SI8L2@CO}t^zM7OhRZql~O!>g*mCKm;`;kzaP}>!~psa6Y=n zueA1JkuTM8F2CkFR6Y5XBWPb8;v%V19^swGwK^J>JM?>=PFs@6t}+*Z*kk)gSbg8M zd=68T@|>eE?^P27e^7keAd_BGJYxaAWKvk(c!xep0BN53eDk2YLlqWJC&!PDGT#xD(gEO6Nhc^-TzR1MIR@=UzC?i_b>Inb2{C7}doA}xRVXpDi4`%R{dn3^lC6!C&Q ztNaDv%cIY+##0>mExH%y$+IPvZC(wFa-i0U%vA82S0m$V?!y(w)+gNJ$Cb$4zY~jm z6CNdDZFN82IXc!U6e*KXAGq6}phWfJ`6Is54Q~~f`tIEUCS9uBk6AGN`N{1A`gZJx zO!&(B1z?qv^i7v1uo3qy+%(MOliSGC$QRU+&0D6@9U`Dmd}D@j`OSqFXgQN5gDE;$ z2j9{cD@!anNA!@!x)xjpMPYrI#k22N;icZlSSY1l#N? z^rN>$mfQ)*TG?$gQd<#hpph-tye7U z0$h2xLgKcY9?=Ft^sDc%ES*3jGuQ2NHV&_CCX{MaaMu3giU;{7?*B*$JZqiv$1m&iAnzM z{*{pBy&y8c9yN3?eSg?yjSfpq*O9s6$>%QR^n~@@yD6;llaY2E z1wH5+OoP0f|2Jp&ep&CY-=0#CXs_dw{Us6xX?@J0`gw*-*l}IE-<{%mwuyh0f?s(l8+}tRbI~?p4)4c%$xBi`SU30Y_5lcdP^+M@7myef%U ze!!{o2S(Kg=G>ICHc;E;B*}6pw6OQbWl)zc1A<nmYOq>Cd^TVd)pW zrZ%Vx$;gW@0L2FN94b%8B@f(uCzI1CX30Z~vy!K4bnqSj2K|e!d-B#UQesoS*cxAg zWxJ>=hwjo7dMeqi3R1M)979~BWQG+36;q9WV&KEFg2AZvi4SJuHPBur5(S~-E;~Xu zQMK&Sas$rOg$7O=^Sw_v!={o~U$dmeTii{0BK)m69wMmIk3PFs5U->9^LH(u2<7f! zFoz?rbLg94H16G~<>;5aLejgO@*e|#Fx=}?!L!v2h_3wQ%8k-pwOd{F%i^Szw8e2o z=f7C>W2N#G^|){Pq<1lAE1%mX+P+pVLnlpi;fqkQ;Adn%!(Jx9qZL5xSN{Md(4A9s zHRh1vc&S6ow2h%5p@Xw;Rn7^fxq5n1CrvOx^)L;Lamtj~ifF}((%#%l+vm}lPk++6 z=vc(X&AZyk6y1Nn)h6ip1n-^S>E}h}*2Q(654W^DdHh_o9D<~51hHl} zinlNxtL{n`ve*4JblNNf*CG3fI)7H1ZFhL^>XLD;zZr)>FB$(QTmKK^xkJ^h3)}x@ zAf$br^L-{Lb@H6l5TnioAeGa5Q)(o)z$}nq{Ruk+NHG=KlI|bguOC8i2~)SuMY~qe z3fpvSY>7*;q@?1y@%{>#CRW(6q+nQC+H&dWgoEINuRV@$N1=)B*h$bw@pdj7%~6Y7 zCWghL72Twy0!vKSr-y`?4=>;JoBt$T{u*$I4)}qf#IhE$&N_q=QUx5F8KfdKmC|Li z2x+%7YkyEjEQ!1mZM11rPB6nZrdpg}c$E`R^VG+Wa57=(UMKaR)yO-XJiLovQqL=Bi312k3pyl zOf%kvJa)tC;v(f2_oJK2UftbjT#ve1`CI>Jh6QXi+u&}00aO&YL(wF;k zTCbDvE&@`!29IlA96k}2{bHC)j!v}QGWb=;5c}PPG_82#Ou@M|xHk4org|nqCvaa$ zD1X-W(23AtWyVNA-Rl#-ZnizV_jdJUR6OAQpZLy(ywvo+^@r?#wVe>$`tRKl2{{YE z>9=^Ilf`Km#`UF$j2wyG#VAE~0DXL0tLI35o5Y<5QZ#Ea4C;>>2Mf5kTgFo+-nK;e zmK0iqnF=yMI+kfo8S$tHTT3T3miLS%pb$D43q3)3X>7;)$k7ABXc;GjlQqMSG6%x50$A~YR{TK1tIlW{(@lH z12we7(AWz-#eF0Lfu16Cc%Y^+%Nm-S}Cl&2N@)63QOr1Cc zpKQt1)uS#Ny`Av9@>lu#8avJMLg%(lPyr;I;NU|XRl=%pllF0VNKrPSGWfuLVi*&Y zYob5jRyxg&z!2YhpJno+?dFeJD%jmuST!L@@A?FO10`(`jd##7odasM*B! zmuDd|c}i`MB7#GyO@dj1YAFT-xR-@fqd#u1KwMe91_hk8D!a&TZZ=tWp%~P)f z6C%c-8_32?La_0E&LoghQdN7JaDHZcwK7xVSZ<`fL4kOXp14T(w8|ArH0bHW`J2R~ zL`X*T+^=t|k)(L#WLi3y{Nc;_64N|R?c7JYY<#vo`ZyVJl=ympaXxRF5PwiT4zN>LUf1?h- z$%C~d&QQk23689|8*eQ6Wyd0720pV2(|O@ZZh8zN-QC!w0hHB7*ON-0}U6r@ab#iZf= z1u#HwyPTK;sz8}~+WsWl=4!Qk^n2aPd~M~970hWqct-j0<+EjT9D&AVn0l#$;_^j; zj4Ow+wIl#Px)9$fM+8??hWc#Ue9*ROkRe~!#-xy2zvG<$!r5jTcjvvh+msTJ-- z5O_wKQ>b_Ye3uj*5X{H2ri;C%JO3ePLG21#zMK{Z*Crh~Te6Vimk#j{!8*U=T}BO0 zT;+~BTDjHXm=b}HjZgSPZ+QauHc+sV3O2?LdN9HnXK%7#F&RuUu%Fu8> z)E;(XO7xakum|QtPW15djm*g(6!6d?;7DGr{xKO@yAm-IYE%uLT2FgpsI{FFAKX21 zYH65Y*m%INBvOTyL_osnvk-3KQ!Bz_1--1= zE@w97jIUM~z}wP3!pNq2g11Mr1k-FvY83D}5i&HSM|pOkgj-u$zbTX2ypxl@1#&;M{pYfC9iJ`A1w#iVMNK(rSAaptn zj6**Uy-zVVw2%T-uFxb=fb~mN!5fu@T>=K@k~9`QXPD?qorrDI^*OK7g^=r2JLrlU zvXC$!vjw(;M4tctK9&!PaR)TiZ;uY(u*< z0FqDSX|lzr$DjpmWMWgevW;SD5Ylv6^QE>$?(C{)-%D<5TIgflSE}(?p%dQGyXFA{ z(3^=VsB54G`rn;Ba-R&7!2~Fp*dSl7DI|KNraW$|_taonBdLpjT_E~ZhhHpV