normalize coefficients in normalize_coeffs_8bpc

increase precision of negative filter lobes. Add test

Tests/test_image_resample.py

@@ -320,5 +320,22 @@ class CoreResamplePassesTest(PillowTestCase):
         self.assertEqual(Image.core.getcount(), count + 2)
 
 
+class CoreResampleCoefficientsTest(PillowTestCase):
+    def test_reduce(self):
+        test_color = 254
+        # print ''
+
+        for size in range(400000, 400010, 2):
+            # print '\r', size,
+            i = Image.new('L', (size, 1), 0)
+            draw = ImageDraw.Draw(i)
+            draw.rectangle((0, 0, i.size[0] // 2 - 1, 0), test_color)
+
+            px = i.resize((5, i.size[1]), Image.BICUBIC).load()
+            if px[2, 0] != test_color // 2:
+                self.assertEqual(test_color // 2, px[2, 0])
+                # print '\r>', size, test_color // 2, px[2, 0]
+
+
 if __name__ == '__main__':
     unittest.main()

libImaging/Resample.c

@@ -206,6 +206,32 @@ precompute_coeffs(int inSize, int outSize, struct filter *filterp,
 }
 
 
+int
+normalize_coeffs_8bpc(int outSize, int kmax, double *prekk, INT32 **kkp)
+{
+    int x;
+    INT32 *kk;
+
+    /* malloc check ok, overflow checked in precompute_coeffs */
+    kk = malloc(outSize * kmax * sizeof(INT32));
+    if ( ! kk) {
+        return 0;
+    }
+
+    for (x = 0; x < outSize * kmax; x++) {
+        if (prekk[x] < 0) {
+            kk[x] = (int) (-0.5 + prekk[x] * (1 << PRECISION_BITS));
+        } else {
+            kk[x] = (int) (0.5 + prekk[x] * (1 << PRECISION_BITS));
+        }
+    }
+
+    *kkp = kk;
+    return kmax;
+}
+
+
+
 Imaging
 ImagingResampleHorizontal_8bpc(Imaging imIn, int xsize, struct filter *filterp)
 {
@@ -214,27 +240,21 @@ ImagingResampleHorizontal_8bpc(Imaging imIn, int xsize, struct filter *filterp)
     int ss0, ss1, ss2, ss3;
     int xx, yy, x, kmax, xmin, xmax;
     int *xbounds;
-    int *k, *kk;
+    INT32 *k, *kk;
     double *prekk;
 
     kmax = precompute_coeffs(imIn->xsize, xsize, filterp, &xbounds, &prekk);
     if ( ! kmax) {
         return (Imaging) ImagingError_MemoryError();
     }
-    
+
-    kk = malloc(xsize * kmax * sizeof(int));
+    kmax = normalize_coeffs_8bpc(xsize, kmax, prekk, &kk);
-    if ( ! kk) {
+    free(prekk);    
+    if ( ! kmax) {
         free(xbounds);
-        free(prekk);
         return (Imaging) ImagingError_MemoryError();
     }
 
-    for (x = 0; x < xsize * kmax; x++) {
-        kk[x] = (int) (0.5 + prekk[x] * (1 << PRECISION_BITS));
-    }
-
-    free(prekk);
-
     imOut = ImagingNew(imIn->mode, xsize, imIn->ysize);
     if ( ! imOut) {
         free(kk);
@@ -325,27 +345,21 @@ ImagingResampleVertical_8bpc(Imaging imIn, int ysize, struct filter *filterp)
     int ss0, ss1, ss2, ss3;
     int xx, yy, y, kmax, ymin, ymax;
     int *xbounds;
-    int *k, *kk;
+    INT32 *k, *kk;
     double *prekk;
 
     kmax = precompute_coeffs(imIn->ysize, ysize, filterp, &xbounds, &prekk);
     if ( ! kmax) {
         return (Imaging) ImagingError_MemoryError();
     }
-    
+
-    kk = malloc(ysize * kmax * sizeof(int));
+    kmax = normalize_coeffs_8bpc(ysize, kmax, prekk, &kk);
-    if ( ! kk) {
+    free(prekk);    
+    if ( ! kmax) {
         free(xbounds);
-        free(prekk);
         return (Imaging) ImagingError_MemoryError();
     }
 
-    for (y = 0; y < ysize * kmax; y++) {
-        kk[y] = (int) (0.5 + prekk[y] * (1 << PRECISION_BITS));
-    }
-
-    free(prekk);
-
     imOut = ImagingNew(imIn->mode, imIn->xsize, ysize);
     if ( ! imOut) {
         free(kk);