Replace SIZE_MAX with type specific _MAX

decode.c

@@ -189,7 +189,7 @@ _setimage(ImagingDecoderObject* decoder, PyObject* args)
     /* Allocate memory buffer (if bits field is set) */
     if (state->bits > 0) {
         if (!state->bytes) {
-            if (state->xsize > ((SIZE_MAX / state->bits)-7)){
+            if (state->xsize > ((INT_MAX / state->bits)-7)){
                 return PyErr_NoMemory();
             }
             state->bytes = (state->bits * state->xsize+7)/8;

encode.c

@@ -234,7 +234,7 @@ _setimage(ImagingEncoderObject* encoder, PyObject* args)
 
     /* Allocate memory buffer (if bits field is set) */
     if (state->bits > 0) {
-        if (state->xsize > ((SIZE_MAX / state->bits)-7)) {
+        if (state->xsize > ((INT_MAX / state->bits)-7)) {
             return PyErr_NoMemory();
         }
         state->bytes = (state->bits * state->xsize+7)/8;

libImaging/Draw.c

@@ -937,7 +937,7 @@ allocate(ImagingOutline outline, int extra)
             /* malloc check ok, uses calloc for overflow */
             e = calloc(outline->size, sizeof(Edge));
         } else {
-            if (outline->size > SIZE_MAX / sizeof(Edge)) {
+            if (outline->size > INT_MAX / sizeof(Edge)) {
                 return NULL;
             }
             /* malloc check ok, overflow checked above */

libImaging/Quant.c

@@ -1092,6 +1092,10 @@ k_means(Pixel *pixelData,
    uint32_t **avgDistSortKey;
    int changes;
    int built=0;
+
+   if (nPaletteEntries > UINT32_MAX / (sizeof(uint32_t))) {
+       return 0;
+   }
    /* malloc check ok, using calloc */
    if (!(count=calloc(nPaletteEntries, sizeof(uint32_t)))) {
       return 0;
@@ -1107,7 +1111,7 @@ k_means(Pixel *pixelData,
    }
    
    /* this is enough of a check, since the multiplication n*size is done above */
-   if (nPaletteEntries > SIZE_MAX / (nPaletteEntries * sizeof(uint32_t))) {
+   if (nPaletteEntries > UINT32_MAX / (nPaletteEntries * sizeof(uint32_t))) {
        goto error_1;
    }
    /* malloc check ok, using calloc, checking n*n above */
@@ -1266,8 +1270,8 @@ quantize(Pixel *pixelData,
    qp=calloc(nPixels, sizeof(uint32_t));
    if (!qp) { goto error_4; }
 
-   if ((nPaletteEntries > SIZE_MAX / nPaletteEntries ) ||
-       (nPaletteEntries > SIZE_MAX / (nPaletteEntries * sizeof(uint32_t)))) {
+   if ((nPaletteEntries > UINT32_MAX / nPaletteEntries ) ||
+       (nPaletteEntries > UINT32_MAX / (nPaletteEntries * sizeof(uint32_t)))) {
        goto error_5;
    }
    /* malloc check ok, using calloc for overflow, check of n*n above */
@@ -1445,8 +1449,8 @@ quantize2(Pixel *pixelData,
    qp=calloc(nPixels, sizeof(uint32_t));
    if (!qp) { goto error_1; }
 
-   if ((nQuantPixels > SIZE_MAX / nQuantPixels ) ||
-       (nQuantPixels > SIZE_MAX / (nQuantPixels * sizeof(uint32_t)))) {
+   if ((nQuantPixels > UINT32_MAX / nQuantPixels ) ||
+       (nQuantPixels > UINT32_MAX / (nQuantPixels * sizeof(uint32_t)))) {
        goto error_2;
    }
 
@@ -1515,8 +1519,8 @@ ImagingQuantize(Imaging im, int colors, int mode, int kmeans)
     if (!strcmp(im->mode, "RGBA") && mode != 2 && mode != 3)
        return ImagingError_ModeError();
 
-    if ((im->xsize > SIZE_MAX / im->ysize) ||
-        (im->xsize > SIZE_MAX / (im->ysize * sizeof(Pixel)))) {
+    if ((im->xsize > INT_MAX / im->ysize) ||
+        (im->xsize > INT_MAX / (im->ysize * sizeof(Pixel)))) {
         return ImagingError_MemoryError();
     }
     /* malloc check ok, using calloc for final overflow, x*y above */

libImaging/QuantHeap.c

@@ -20,6 +20,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <math.h>
+#include <limits.h>
 
 #include "QuantHeap.h"
 
@@ -47,7 +48,7 @@ static int _heap_grow(Heap *h,int newsize) {
    void *newheap;
    if (!newsize) newsize=h->heapsize<<1;
    if (newsize<h->heapsize) return 0;
-   if (newsize > ((int)SIZE_MAX) / sizeof(void *)){
+   if (newsize > INT_MAX / sizeof(void *)){
        return 0;
    }
    /* malloc check ok, using calloc for overflow, also checking 

libImaging/QuantOctree.c

@@ -26,6 +26,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <limits.h>
 
 #include "QuantOctree.h"
 
@@ -62,6 +63,12 @@ new_color_cube(int r, int g, int b, int a) {
    cube->bBits = MAX(b, 0);
    cube->aBits = MAX(a, 0);
 
+   /* overflow check for size multiplication below */
+   if (cube->rBits + cube->gBits + cube->bBits + cube->aBits > 31) {
+       free(cube);
+       return NULL;
+   }
+
    /* the width of the cube for each dimension */
    cube->rWidth = 1<<cube->rBits;
    cube->gWidth = 1<<cube->gBits;
@@ -77,6 +84,7 @@ new_color_cube(int r, int g, int b, int a) {
 
    /* the number of color buckets */
    cube->size = cube->rWidth * cube->gWidth * cube->bWidth * cube->aWidth;
+   /* malloc check ok, overflow checked above */
    cube->buckets = calloc(cube->size, sizeof(struct _ColorBucket));
 
    if (!cube->buckets) {
@@ -155,7 +163,7 @@ compare_bucket_count(const ColorBucket a, const ColorBucket b) {
 static ColorBucket
 create_sorted_color_palette(const ColorCube cube) {
    ColorBucket buckets;
-   if (cube->size > SIZE_MAX / sizeof(struct _ColorBucket)) {
+   if (cube->size > LONG_MAX / sizeof(struct _ColorBucket)) {
        return NULL;
    }
    /* malloc check ok, calloc + overflow check above for memcpy */
@@ -285,8 +293,8 @@ void add_lookup_buckets(ColorCube cube, ColorBucket palette, long nColors, long
 ColorBucket
 combined_palette(ColorBucket bucketsA, long nBucketsA, ColorBucket bucketsB, long nBucketsB) {
    ColorBucket result;
-   if (nBucketsA > SIZE_MAX - nBucketsB ||
-       (nBucketsA+nBucketsB) > SIZE_MAX / sizeof(struct _ColorBucket)) {
+   if (nBucketsA > LONG_MAX - nBucketsB ||
+       (nBucketsA+nBucketsB) > LONG_MAX / sizeof(struct _ColorBucket)) {
        return NULL;
    }
    /* malloc check ok, overflow check above */

libImaging/RankFilter.c

@@ -61,8 +61,8 @@ ImagingRankFilter(Imaging im, int size, int rank)
         return (Imaging) ImagingError_ValueError("bad filter size");
     
     /* malloc check ok, for overflow in the define below */
-    if (size > SIZE_MAX / size || 
-        size > SIZE_MAX / (size * sizeof(FLOAT32))) {
+    if (size > INT_MAX / size || 
+        size > INT_MAX / (size * sizeof(FLOAT32))) {
         return (Imaging) ImagingError_ValueError("filter size too large");
     }
 

libImaging/ZipDecode.c

@@ -57,7 +57,7 @@ ImagingZipDecode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes)
             context->prefix = 1; /* PNG */
 
         /* overflow check for malloc */
-        if (state->bytes > SIZE_MAX - 1) {
+        if (state->bytes > INT_MAX - 1) {
             state->errcode = IMAGING_CODEC_MEMORY;
             return -1;
         }

libImaging/ZipEncode.c

@@ -38,7 +38,7 @@ ImagingZipEncode(Imaging im, ImagingCodecState state, UINT8* buf, int bytes)
         /* Valid modes are ZIP_PNG, ZIP_PNG_PALETTE, and ZIP_TIFF */
 
         /* overflow check for malloc */
-        if (state->bytes > SIZE_MAX - 1) {
+        if (state->bytes > INT_MAX - 1) {
             state->errcode = IMAGING_CODEC_MEMORY;
             return -1;
         }